From d1419b7db3c4388a746e577e9792921ae5e7d2d0 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 18 Oct 2026 10:08:37 +0300 Subject: [PATCH 01/47] fix: Keep working-state reporting alive through provider turns --- docs/turn-outcome.md | 22 +- docs/websocket-events.md | 9 +- src/band/adapters/opencode/adapter.py | 53 ++-- src/band/runtime/execution.py | 125 ++++++++-- src/band/runtime/tools/agent.py | 4 +- src/band/runtime/working_state.py | 1 + tests/runtime/activityownership/__init__.py | 1 + tests/runtime/activityownership/conftest.py | 56 +++++ tests/runtime/activityownership/kit.py | 17 ++ .../activityownership/test_lifetime.py | 198 +++++++++++++++ .../activityownership/test_opencode.py | 77 ++++++ .../runtime/executionworkingstate/__init__.py | 1 + .../runtime/executionworkingstate/conftest.py | 38 +++ tests/runtime/executionworkingstate/kit.py | 52 ++++ .../test_sessionconfigguards.py | 56 +++++ .../executionworkingstate/test_wiring.py | 104 ++++++++ tests/runtime/test_execution_working_state.py | 233 ------------------ tests/runtime/test_working_state.py | 203 --------------- tests/runtime/workingstate/__init__.py | 1 + tests/runtime/workingstate/kit.py | 28 +++ .../runtime/workingstate/test_cancellation.py | 47 ++++ .../workingstate/test_failureswallowed.py | 22 ++ tests/runtime/workingstate/test_keepalive.py | 32 +++ .../runtime/workingstate/test_maxduration.py | 29 +++ tests/runtime/workingstate/test_ordering.py | 25 ++ tests/runtime/workingstate/test_startstop.py | 52 ++++ tests/runtime/workingstate/test_validation.py | 12 + 27 files changed, 1019 insertions(+), 479 deletions(-) create mode 100644 tests/runtime/activityownership/__init__.py create mode 100644 tests/runtime/activityownership/conftest.py create mode 100644 tests/runtime/activityownership/kit.py create mode 100644 tests/runtime/activityownership/test_lifetime.py create mode 100644 tests/runtime/activityownership/test_opencode.py create mode 100644 tests/runtime/executionworkingstate/__init__.py create mode 100644 tests/runtime/executionworkingstate/conftest.py create mode 100644 tests/runtime/executionworkingstate/kit.py create mode 100644 tests/runtime/executionworkingstate/test_sessionconfigguards.py create mode 100644 tests/runtime/executionworkingstate/test_wiring.py delete mode 100644 tests/runtime/test_execution_working_state.py delete mode 100644 tests/runtime/test_working_state.py create mode 100644 tests/runtime/workingstate/__init__.py create mode 100644 tests/runtime/workingstate/kit.py create mode 100644 tests/runtime/workingstate/test_cancellation.py create mode 100644 tests/runtime/workingstate/test_failureswallowed.py create mode 100644 tests/runtime/workingstate/test_keepalive.py create mode 100644 tests/runtime/workingstate/test_maxduration.py create mode 100644 tests/runtime/workingstate/test_ordering.py create mode 100644 tests/runtime/workingstate/test_startstop.py create mode 100644 tests/runtime/workingstate/test_validation.py diff --git a/docs/turn-outcome.md b/docs/turn-outcome.md index 7402a79a1..6951fad99 100644 --- a/docs/turn-outcome.md +++ b/docs/turn-outcome.md @@ -374,7 +374,7 @@ refusal instead of returning `DONE`; a drain records only accepted messages. ## Cancellation ownership -Each runtime handler and SDK-managed Codex/Claude worker has one task owner. +Each runtime handler and SDK-managed Claude, Codex or OpenCode worker has one task owner. The originating turn scope coordinates termination and fixes one absolute cleanup deadline using the existing one-second grace. Repeated controls still notify the adapter and update the unsettled delivery outcome, but never cancel its cleanup @@ -384,9 +384,9 @@ A stopped tool or auxiliary claim first revokes that scope's posting permission and raises cancellation in its caller. Bound workers and the handler get the same cleanup grace to unwind cooperatively; an unfinished task receives cancellation once at expiry if none was delivered, even when nothing waits for it. Native -controls, heartbeat refusal, watchdog expiry and shutdown cancel a running turn -immediately; a worker already released for approval is left to the adapter's -room cleanup. Provider execution and recovery timeouts retain their adapter +STOP and INTERRUPT controls, heartbeat refusal and watchdog expiry cancel the +originating turn, including a worker released for approval. Leave and shutdown +clear reporting while released worker cleanup remains with adapter residency. Provider execution and recovery timeouts retain their adapter policy and coordinate with that same owner. Unfinished tasks remain owned after grace expires. They cannot post late replies, @@ -396,3 +396,17 @@ a later existing host cleanup retry completes retirement. Worker completion alon does not readmit the room. Successful approval release retains its original tools and judgment until the worker actually finishes; `Turn.detach()` is still only an outcome-ledger operation. + +## Working-state lifetime + +The activity owner is the originating turn scope, independently of the current +message delivery. A registered provider worker keeps its reporter alive after +the handler settles, including while waiting for human approval. Later approval +replies and busy deliveries do not replace that owner or restart its duration +cap. Reporting ends when both the handler and its workers finish. + +An echoed heartbeat STOP can cancel that detached provider and notify its adapter +once. It does not rewrite the original delivery's settled acknowledgement. +Explicit interruption invalidates its activity; leave and shutdown clear the +indicator while adapter residency retains responsibility for released workers. +The authoritative idle report finishes before a replacement reports working. diff --git a/docs/websocket-events.md b/docs/websocket-events.md index f233da3fb..f623f8cd4 100644 --- a/docs/websocket-events.md +++ b/docs/websocket-events.md @@ -49,7 +49,10 @@ While working-state reporting is active, its existing activity keep-alive also recovers a missed STOP: the platform answers a `working: true` report with `working: false` when the room execution is stopped. The runtime cancels the originating turn through the same platform-stop path as a refused claim or post, -leaving its message PROCESSING for later replay. Responses from before PLAY, +leaving an unsettled message PROCESSING for later replay. A provider worker +released for approval retains its originating activity scope across later +deliveries; a late heartbeat STOP cancels that provider without changing the +original message's settled acknowledgement. Responses from before PLAY, a replacement turn, or shutdown cannot stop newer work. Request failures never count as STOP, and the public activity-report boolean still means request success. @@ -94,7 +97,9 @@ never interprets approval syntax or drains model context for a control. Room admission and control revisions belong to the execution context; a turn scope owns termination intent and delivery disposition. Each concrete task has -one cancellation owner, including explicitly bound Codex/Claude workers. +one cancellation owner, including explicitly bound Claude, Codex and OpenCode +workers. Activity follows that scope until its handler and registered workers +finish, independently of current delivery admission. Shutdown takes precedence; otherwise the latest accepted native STOP/INTERRUPT before settlement determines delivery. An observed REST refusal cannot replace an accepted native disposition. PLAY advances admission without reviving an diff --git a/src/band/adapters/opencode/adapter.py b/src/band/adapters/opencode/adapter.py index 8de9c2381..a9f826d1d 100644 --- a/src/band/adapters/opencode/adapter.py +++ b/src/band/adapters/opencode/adapter.py @@ -18,6 +18,7 @@ from band.adapters.opencode.approvals import ApprovalPorts, RoomApprovals from band.adapters.opencode.config import OpencodeAdapterConfig +from band.client.streaming import ControlMode from band.converters.opencode import OpencodeHistoryConverter from band.core.delivery import relay_reply from band.core.exceptions import BandConnectionError, InputNotAccepted @@ -28,6 +29,11 @@ ) from band.core.simple_adapter import SimpleAdapter from band.core.turn import judge_detached_turn +from band.core.turn_lifecycle import ( + CYCLE_CANCEL_GRACE_SECONDS, + TurnTask, + bind_turn_task, +) from band.core.types import ( AdapterFeatures, Capability, @@ -99,6 +105,8 @@ class TurnState: assistant_part_types: dict[str, str] = field(default_factory=dict) reported_tool_calls: set[str] = field(default_factory=set) reported_tool_results: set[str] = field(default_factory=set) + worker: TurnTask | None = None + abort_started: bool = False last_error_message: str | None = None # Per-assistant-message usage for the current turn (last-write-wins per id, # since message.updated streams repeatedly). Summed across messages at turn @@ -532,6 +540,7 @@ async def on_message( turn, ) ) + turn.worker = bind_turn_task(tools, turn_task) # Register the watcher only while this turn is still current; a # superseded turn's task must not clobber (or be cancelled through) # the next turn's ambient pointer. @@ -547,21 +556,12 @@ async def on_message( # Nothing awaited here re-raises asyncio.TimeoutError, so on_message has no # timeout handler of its own. except asyncio.CancelledError: - # The runtime interrupts a turn by cancelling this coroutine, but the - # watcher runs detached. Left alone it outlives the interrupt: it - # posts the very reply the user stopped, and holds the room's busy - # guard until turn_timeout_s. Drop the turn state even if a second - # cancellation (e.g. a double-tap stop) cuts off the abandon/abort - # awaits below -- otherwise the room stays wedged on a turn whose - # state never clears. - if turn_future is not None: - assert turn is not None + if turn is not None: try: - already_aborted = await turn.approvals.abandon() + await turn.approvals.abandon() + await self._abort_turn(turn, "interrupted") finally: self._clear_turn_state(room_state, expected_turn=turn) - if not already_aborted: - await self._abort_turn(turn, "interrupted") raise except httpx.HTTPStatusError as exc: logger.exception("OpenCode request failed for room %s", room_id) @@ -1029,6 +1029,9 @@ async def _watch_turn_completion( ) try: await self._await_turn(turn) + except asyncio.CancelledError: + await self._abort_turn(turn, "interrupted") + raise except TimeoutError: logger.warning( "OpenCode turn timed out for room %s (session=%s)", @@ -1075,10 +1078,25 @@ async def _watch_turn_completion( expected_task=asyncio.current_task(), ) + async def on_interrupt(self, room_id: str, mode: ControlMode) -> None: + state = self._rooms.get(room_id) + if state is None or state.turn is None: + return + turn = state.turn + if turn.worker is not None: + turn.worker.terminate() + await turn.worker.drain() + else: + await self._abort_turn(turn, "interrupted") + async def _abort_turn(self, turn: TurnState, reason: str) -> None: """Best-effort: tell OpenCode to stop working on this room's session.""" + if turn.abort_started: + return + turn.abort_started = True try: - await turn.client.abort_session(turn.session_id) + async with asyncio.timeout(CYCLE_CANCEL_GRACE_SECONDS): + await turn.client.abort_session(turn.session_id) except Exception: logger.exception( "Failed to abort %s OpenCode session %s", @@ -1173,8 +1191,13 @@ def _clear_turn_state( return turn_task = room_state.turn_task - if turn_task is not None and turn_task is not expected_task: - turn_task.cancel() + if ( + turn_task is not None + and turn_task is not expected_task + and room_state.turn is not None + and room_state.turn.worker is not None + ): + room_state.turn.worker.terminate() if room_state.turn is not None: room_state.turn.approvals.cancel() diff --git a/src/band/runtime/execution.py b/src/band/runtime/execution.py index 6948d1133..701f25ded 100644 --- a/src/band/runtime/execution.py +++ b/src/band/runtime/execution.py @@ -413,6 +413,9 @@ def __init__( self._detached_cycle_tasks: dict[asyncio.Task[None], TurnTask] = {} self.current_scope: TurnScope | None = None + self._activity_owner: TurnScope | None = None + self._activity_stop_scope: TurnScope | None = None + self._activity_completion: asyncio.Task[None] | None = None self._control_revision = 0 self._on_platform_stop = on_platform_stop self._shutdown_task: asyncio.Task[bool] | None = None @@ -451,8 +454,10 @@ def is_stopped(self) -> bool: @property def adapter_stop_notified(self) -> bool: """Whether an observed STOP already notified the adapter for this turn.""" - scope = self.current_scope - return scope is not None and scope.observer_task is not None + return any( + scope is not None and scope.observer_task is not None + for scope in (self.current_scope, self._activity_stop_scope) + ) def _set_state(self, new_state: ExecutionState) -> None: """ @@ -629,6 +634,8 @@ async def _stop(self, timeout: float | None) -> bool: loop_owner.terminate() await self._drain_or_retain(loop_owner) self._process_loop_task = None + self._activity_owner = None + await self._join_activity_completion() await self._stop_retained_tasks(terminal_deadline()) await self._working_reporter.stop() return self._stop_graceful @@ -736,10 +743,13 @@ def _active_cycle_task(self) -> asyncio.Task[None] | None: def _cancel_cycle(self, kind: ControlMode) -> bool: scope = self._running_scope - if scope is None: - return False - scope.terminate(kind) - return True + activity = self._activity_owner + if activity is not None and activity is not scope: + activity.terminate(kind) + self._finish_activity(activity, aborted=True) + if scope is not None: + scope.terminate(kind) + return scope is not None or activity is not None def stop_room(self) -> None: """Pause triggers and abort the cycle, leaving it actionable for PLAY.""" @@ -767,7 +777,11 @@ def _reset_resync_interval(self) -> None: self._queue_wait.reschedule(self._resync_schedule.due_at) def owns_stop_observation(self, scope: TurnScope) -> bool: - return self._owns_control_revision(scope, scope.stop_revision) + return ( + scope + in (self.current_scope, self._activity_owner, self._activity_stop_scope) + and scope.stop_revision == self._control_revision + ) def _owns_control_revision(self, scope: TurnScope, revision: int | None) -> bool: return self.current_scope is scope and revision == self._control_revision @@ -784,8 +798,10 @@ def observe_platform_stop( scope.control_revision if control_revision is None else control_revision ) scope.terminate(ControlMode.STOP, cooperative=cooperative, observed=True) - if self.current_scope is not scope: - return + if scope is self._activity_owner: + self._activity_stop_scope = scope + if scope.settled: + self._finish_activity(scope, aborted=True) if ( not self._closing and self.owns_stop_observation(scope) @@ -1842,14 +1858,19 @@ async def _report_working_state(self, working: bool) -> bool: working, timeout_seconds=self.config.working_request_timeout_seconds, on_stopped=partial( - self._observe_activity_stop, self.current_scope, self._control_revision + self._observe_activity_stop, + self._activity_owner or self.current_scope, + self._control_revision, ), ) def _observe_activity_stop(self, scope: TurnScope | None, revision: int) -> None: if ( scope is None - or not self._owns_control_revision(scope, revision) + or not ( + scope in (self._activity_owner, self.current_scope) + and revision == self._control_revision + ) or scope.stop_observed or self._closing or scope.kind is not None @@ -1863,8 +1884,9 @@ def _observe_activity_stop(self, scope: TurnScope | None, revision: int) -> None async def _execute_message_cycle( self, event: PlatformEvent, message_id: str | None ) -> None: - await self._working_reporter.start() scope = self.current_scope + if scope is not None: + await self._acquire_activity(scope) if scope is not None and scope.terminating: raise asyncio.CancelledError self._resync_handled_work = True @@ -1872,6 +1894,73 @@ async def _execute_message_cycle( with _observe_phase("handler", self.room_id, message_id): await self._on_execute(self, event) + async def _join_activity_completion(self) -> None: + task = self._activity_completion + if task is not None: + await asyncio.shield(task) + if self._activity_completion is task: + self._activity_completion = None + + async def _acquire_activity(self, scope: TurnScope) -> None: + await self._join_activity_completion() + await self._start_activity(scope) + + async def _start_activity(self, scope: TurnScope) -> None: + previous = self._activity_owner + if previous is not None and previous is not scope: + if not previous.aborted and any( + not worker.task.done() for worker in previous.workers + ): + return + self._finish_activity(previous, aborted=True) + await self._join_activity_completion() + if scope.aborted or self._closing: + return + if self._activity_owner is not scope: + self._activity_stop_scope = None + self._activity_owner = scope + await self._working_reporter.start() + + def register_turn_task(self, scope: TurnScope, owner: TurnTask) -> None: + """Bind completion to the same owner that delivers worker cancellation.""" + scope.bind(owner) + owner.task.add_done_callback(lambda _: self._finish_activity(scope)) + if scope is self.current_scope and self._activity_owner is None: + previous = self._activity_completion + + async def acquire() -> None: + if previous is not None: + await asyncio.shield(previous) + if scope is self.current_scope: + await self._start_activity(scope) + + self._retain_activity_completion(asyncio.create_task(acquire())) + + def _retain_activity_completion(self, task: asyncio.Task[None]) -> None: + self._activity_completion = task + task.add_done_callback(self._activity_completed) + + def _activity_completed(self, task: asyncio.Task[None]) -> None: + task_failure(task) + if self._activity_completion is task: + self._activity_completion = None + if self._activity_owner is not None: + self._finish_activity(self._activity_owner) + + def _finish_activity(self, scope: TurnScope, *, aborted: bool = False) -> None: + if self._activity_owner is not scope: + return + if not aborted and ( + not scope.settled or any(not owner.task.done() for owner in scope.workers) + ): + return + self._activity_owner = None + self._working_reporter.defer_false_report() + if self._activity_completion is None: + self._retain_activity_completion( + asyncio.create_task(self._working_reporter.stop()) + ) + def _retain_detached_cycle_task( self, task: asyncio.Task[None], *, owner: TurnTask | None = None ) -> None: @@ -1957,15 +2046,11 @@ async def invoke() -> None: finally: wake.cancel() await asyncio.gather(wake, return_exceptions=True) - if isinstance(event, MessageEvent) and handler is None: - # stop() joins this cycle inside the settlement budget, which is - # shorter than the activity POST. A closing execution finishes - # working:false after that join. - if self._closing: - self._working_reporter.defer_false_report() - else: - await self._working_reporter.stop() scope.settled = True + if isinstance(event, MessageEvent) and handler is None: + self._finish_activity(scope, aborted=scope.terminating) + if not self._closing: + await self._join_activity_completion() if scope.shutdown: raise asyncio.CancelledError if scope.kind is not None: diff --git a/src/band/runtime/tools/agent.py b/src/band/runtime/tools/agent.py index deeaa144e..6951d5492 100644 --- a/src/band/runtime/tools/agent.py +++ b/src/band/runtime/tools/agent.py @@ -388,8 +388,8 @@ def from_context( return tools def register_turn_task(self, owner: TurnTask) -> None: - if self._scope is not None: - self._scope.bind(owner) + if self._scope is not None and self._ctx is not None: + self._ctx.register_turn_task(self._scope, owner) def _abort_stopped_post(self, error: RoomExecutionStoppedError) -> None: if self._ctx is not None and self._scope is not None: diff --git a/src/band/runtime/working_state.py b/src/band/runtime/working_state.py index f9402c389..4d842c7fe 100644 --- a/src/band/runtime/working_state.py +++ b/src/band/runtime/working_state.py @@ -74,6 +74,7 @@ async def start(self) -> None: # Re-entrant start while active is a no-op: the signal is already # alive and the keep-alive task is already running. return + await self._finish_false_send() self._active = True self._started_at = asyncio.get_running_loop().time() await self._safe_report(True) diff --git a/tests/runtime/activityownership/__init__.py b/tests/runtime/activityownership/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/runtime/activityownership/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/runtime/activityownership/conftest.py b/tests/runtime/activityownership/conftest.py new file mode 100644 index 000000000..916e0ae7b --- /dev/null +++ b/tests/runtime/activityownership/conftest.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from collections.abc import AsyncIterator, Awaitable, Callable +from unittest.mock import AsyncMock + +import pytest + +from band.platform.event import PlatformEvent +from band.platform.link import BandLink +from band.runtime.execution import ExecutionContext +from band.runtime.runtime import AgentRuntime +from band.runtime.sessionconfig import SessionConfig +from tests.runtime.activityownership.kit import ActivityPlatform +from tests.runtime.helpers import ( + AGENT_ID, + ROOM_ID, + LifecyclePlatform, + rest_client_over, +) + + +@pytest.fixture +async def peer() -> AsyncIterator[LifecyclePlatform]: + yield ActivityPlatform() + + +@pytest.fixture +async def execution( + peer: LifecyclePlatform, +) -> AsyncIterator[Callable[..., ExecutionContext]]: + contexts: list[ExecutionContext] = [] + async with rest_client_over(peer.answer) as rest: + + def create( + handler: Callable[[ExecutionContext, PlatformEvent], Awaitable[None]], + **settings: object, + ) -> ExecutionContext: + link = BandLink(agent_id=AGENT_ID, api_key="test-key") + link.rest = rest + runtime = AgentRuntime( + link=link, agent_id=AGENT_ID, on_execute=handler, on_control=AsyncMock() + ) + ctx = ExecutionContext( + ROOM_ID, + link, + handler, + agent_id=AGENT_ID, + on_platform_stop=runtime._on_platform_stop, + config=SessionConfig(enable_context_hydration=False, **settings), + ) + contexts.append(ctx) + return ctx + + yield create + for ctx in contexts: + await ctx.stop() diff --git a/tests/runtime/activityownership/kit.py b/tests/runtime/activityownership/kit.py new file mode 100644 index 000000000..d43a5b921 --- /dev/null +++ b/tests/runtime/activityownership/kit.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import httpx + +from tests.runtime.helpers import ALICE, LifecyclePlatform, page + + +class ActivityPlatform(LifecyclePlatform): + def answer(self, request: httpx.Request) -> httpx.Response: + tail = request.url.path.rsplit("/", 1)[-1] + if tail == "participants": + return page([{**ALICE, "role": "member", "status": "active"}]) + if request.method == "POST" and tail == "messages": + return httpx.Response( + 201, json={"data": {"id": "m-1", "success": True, "recipients": []}} + ) + return super().answer(request) diff --git a/tests/runtime/activityownership/test_lifetime.py b/tests/runtime/activityownership/test_lifetime.py new file mode 100644 index 000000000..1dcf0f921 --- /dev/null +++ b/tests/runtime/activityownership/test_lifetime.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Callable + +import pytest + +from band.client.streaming import ControlMode +from band.core.turn_lifecycle import TurnTask, bind_turn_task +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from tests.conftest import make_message_event +from tests.runtime.helpers import IDLE, KEEP_ALIVE, LifecyclePlatform + + +class TestProviderLifetime: + @pytest.mark.looptime + async def test_detached_worker_keeps_heartbeat_across_busy_delivery( + self, execution: Callable[..., ExecutionContext], peer: LifecyclePlatform + ) -> None: + release = asyncio.Event() + workers: list[TurnTask] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + if workers: + return + tools = AgentTools.from_context(ctx) + workers.append(bind_turn_task(tools, asyncio.create_task(release.wait()))) + + ctx = execution(handler) + await ctx._process_event(make_message_event(msg_id="original")) + await asyncio.sleep(3.1) + await ctx._process_event(make_message_event(msg_id="busy")) + assert peer.activity_reports == [KEEP_ALIVE, KEEP_ALIVE] + assert peer.marked("processed") == ["original", "busy"] + release.set() + await workers[0].task + await asyncio.sleep(0) + await ctx._join_activity_completion() + assert peer.activity_reports[-1] == IDLE + + async def test_worker_completion_does_not_clear_running_handler( + self, execution: Callable[..., ExecutionContext], peer: LifecyclePlatform + ) -> None: + finished, release = asyncio.Event(), asyncio.Event() + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + worker = asyncio.create_task(asyncio.sleep(0)) + bind_turn_task(AgentTools.from_context(ctx), worker) + await worker + finished.set() + await release.wait() + + ctx = execution(handler) + turn = asyncio.create_task(ctx._process_event(make_message_event())) + await finished.wait() + assert peer.activity_reports == [KEEP_ALIVE] + release.set() + await turn + assert peer.activity_reports == [KEEP_ALIVE, IDLE] + + @pytest.mark.parametrize("mode", [ControlMode.STOP, ControlMode.INTERRUPT]) + async def test_detached_control_cancels_worker_preserving_settled_ack( + self, + execution: Callable[..., ExecutionContext], + peer: LifecyclePlatform, + mode: ControlMode, + ) -> None: + workers: list[TurnTask] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + workers.append( + bind_turn_task( + AgentTools.from_context(ctx), + asyncio.create_task(asyncio.Event().wait()), + ) + ) + + ctx = execution(handler) + await ctx._process_event(make_message_event(msg_id="original")) + assert ctx.interrupt(kind=mode) + await asyncio.gather(workers[0].task, return_exceptions=True) + await ctx._join_activity_completion() + assert workers[0].task.cancelled() + assert peer.marked("processed") == ["original"] + assert peer.activity_reports == [KEEP_ALIVE, IDLE] + + @pytest.mark.looptime + async def test_detached_heartbeat_stop_preserves_ack( + self, execution: Callable[..., ExecutionContext], peer: LifecyclePlatform + ) -> None: + workers: list[TurnTask] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + if not workers: + workers.append( + bind_turn_task( + AgentTools.from_context(ctx), + asyncio.create_task(asyncio.Event().wait()), + ) + ) + + ctx = execution(handler) + await ctx._process_event(make_message_event(msg_id="original")) + await ctx._process_event(make_message_event(msg_id="busy")) + peer.stopped = True + await asyncio.sleep(3.1) + assert workers[0].task.cancelled() + assert peer.marked("processed") == ["original", "busy"] + assert peer.activity_reports[-1] == IDLE + + @pytest.mark.looptime + async def test_busy_delivery_keeps_original_cap_deadline( + self, execution: Callable[..., ExecutionContext], peer: LifecyclePlatform + ) -> None: + release = asyncio.Event() + workers: list[TurnTask] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + if not workers: + workers.append( + bind_turn_task( + AgentTools.from_context(ctx), + asyncio.create_task(release.wait()), + ) + ) + + ctx = execution(handler, max_working_state_seconds=4.0) + await ctx._process_event(make_message_event(msg_id="original")) + await asyncio.sleep(3.1) + await ctx._process_event(make_message_event(msg_id="busy")) + await asyncio.sleep(6) + assert peer.activity_reports == [KEEP_ALIVE, KEEP_ALIVE] + assert not workers[0].task.done() + release.set() + await workers[0].task + await asyncio.sleep(0) + await ctx._join_activity_completion() + assert peer.activity_reports[-1] == IDLE + + async def test_shutdown_clears_reporting_without_cancelling_detached_worker( + self, execution: Callable[..., ExecutionContext], peer: LifecyclePlatform + ) -> None: + release = asyncio.Event() + workers: list[TurnTask] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + workers.append( + bind_turn_task( + AgentTools.from_context(ctx), asyncio.create_task(release.wait()) + ) + ) + + ctx = execution(handler) + await ctx._process_event(make_message_event()) + await ctx.stop() + assert peer.activity_reports == [KEEP_ALIVE, IDLE] + assert not workers[0].task.done() + release.set() + await workers[0].task + assert peer.activity_reports == [KEEP_ALIVE, IDLE] + + async def test_new_worker_admitted_after_previous_completion_gets_own_indicator( + self, execution: Callable[..., ExecutionContext], peer: LifecyclePlatform + ) -> None: + releases = [asyncio.Event(), asyncio.Event()] + entered, admitted = asyncio.Event(), asyncio.Event() + workers: list[TurnTask] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + index = len(workers) + if index: + entered.set() + await admitted.wait() + workers.append( + bind_turn_task( + AgentTools.from_context(ctx), + asyncio.create_task(releases[index].wait()), + ) + ) + + ctx = execution(handler) + await ctx._process_event(make_message_event(msg_id="original")) + turn = asyncio.create_task(ctx._process_event(make_message_event(msg_id="new"))) + await entered.wait() + releases[0].set() + await workers[0].task + await asyncio.sleep(0) + await ctx._join_activity_completion() + admitted.set() + await turn + assert peer.activity_reports == [KEEP_ALIVE, IDLE, KEEP_ALIVE] + releases[1].set() + await workers[1].task + await asyncio.sleep(0) + await ctx._join_activity_completion() + assert peer.activity_reports == [KEEP_ALIVE, IDLE, KEEP_ALIVE, IDLE] diff --git a/tests/runtime/activityownership/test_opencode.py b/tests/runtime/activityownership/test_opencode.py new file mode 100644 index 000000000..7fb26af66 --- /dev/null +++ b/tests/runtime/activityownership/test_opencode.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Callable +from unittest.mock import patch + +import pytest + +from band.adapters.opencode import OpencodeAdapter +from band.client.streaming import ControlMode +from band.integrations.opencode.types import OpencodeSessionState +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from tests.adapters.opencode.helpers import ( + FakeOpencodeClient, + event_permission, + make_platform_message, +) +from tests.conftest import make_message_event +from tests.mcpbackends import backends_created_by +from tests.runtime.helpers import IDLE, KEEP_ALIVE, ROOM_ID, LifecyclePlatform + + +class TestOpenCodeLifetime: + @pytest.mark.parametrize("mode", [ControlMode.STOP, ControlMode.INTERRUPT]) + async def test_parked_native_turn_dual_dispatch_aborts_remote_once( + self, + execution: Callable[..., ExecutionContext], + peer: LifecyclePlatform, + mode: ControlMode, + ) -> None: + client = FakeOpencodeClient( + prompt_event_sequences=[[event_permission("sess-1", "permission-1")]] + ) + adapter = OpencodeAdapter(client_factory=lambda _: client, emit=()) + aborting, release, cleaned = (asyncio.Event() for _ in range(3)) + original_abort = client.abort_session + + async def abort(session_id: str) -> None: + aborting.set() + await release.wait() + await original_abort(session_id) + cleaned.set() + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + await adapter.on_message( + make_platform_message(room_id=ROOM_ID, sender_id="user-456"), + AgentTools.from_context(ctx), + OpencodeSessionState(), + None, + None, + is_session_bootstrap=True, + room_id=ROOM_ID, + ) + + with backends_created_by(), patch.object(client, "abort_session", abort): + await adapter.on_started("Test", "Test") + ctx = execution(handler) + try: + await ctx._process_event(make_message_event(msg_id="original")) + assert peer.activity_reports == [KEEP_ALIVE] + assert peer.marked("processed") == ["original"] + assert ctx.interrupt(kind=mode) + notify = asyncio.create_task(adapter.on_interrupt(ROOM_ID, mode)) + await aborting.wait() + assert not cleaned.is_set() + release.set() + await notify + await ctx._join_activity_completion() + assert client.aborted_sessions == ["sess-1"] + assert cleaned.is_set() + assert peer.activity_reports == [KEEP_ALIVE, IDLE] + assert peer.marked("processed") == ["original"] + finally: + release.set() + await adapter.on_cleanup(ROOM_ID) diff --git a/tests/runtime/executionworkingstate/__init__.py b/tests/runtime/executionworkingstate/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/runtime/executionworkingstate/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/runtime/executionworkingstate/conftest.py b/tests/runtime/executionworkingstate/conftest.py new file mode 100644 index 000000000..67bd0bbf7 --- /dev/null +++ b/tests/runtime/executionworkingstate/conftest.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + + +@pytest.fixture +def mock_link(): + link = MagicMock() + link.agent_id = "agent-123" + link.rest = MagicMock() + + participant1 = MagicMock() + participant1.id = "user-1" + participant1.name = "User One" + participant1.type = "User" + link.rest.agent_api_participants = MagicMock() + link.rest.agent_api_participants.list_agent_chat_participants = AsyncMock( + return_value=MagicMock(data=[participant1]) + ) + link.rest.agent_api_context = MagicMock() + link.rest.agent_api_context.get_agent_chat_context = AsyncMock( + return_value=MagicMock(data=[]) + ) + + link.mark_processing = AsyncMock(return_value=True) + link.mark_processed = AsyncMock(return_value=True) + link.mark_failed = AsyncMock(return_value=True) + link.get_next_message = AsyncMock(return_value=None) + link.get_stale_processing_messages = AsyncMock(return_value=[]) + link.report_activity = AsyncMock(return_value=True) + return link + + +@pytest.fixture +def mock_handler(): + return AsyncMock() diff --git a/tests/runtime/executionworkingstate/kit.py b/tests/runtime/executionworkingstate/kit.py new file mode 100644 index 000000000..0be9f6f5f --- /dev/null +++ b/tests/runtime/executionworkingstate/kit.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from unittest.mock import AsyncMock, MagicMock + +from band.runtime.messages import PlatformMessage +from band.runtime.sessionconfig import SessionConfig + + +def _working_values(link) -> list[bool]: + """Sequence of `working` booleans passed to link.report_activity.""" + return [ + c.args[1] if len(c.args) > 1 else c.kwargs["working"] + for c in link.report_activity.call_args_list + ] + + +_FAST = SessionConfig(working_keep_alive_seconds=4.0) + + +def _backlog_message( + room_id: str = "room-123", msg_id: str = "msg-b1" +) -> PlatformMessage: + return PlatformMessage( + id=msg_id, + room_id=room_id, + content="hi", + sender_id="user-1", + sender_type="User", + sender_name="User One", + message_type="text", + metadata={}, + created_at=datetime.now(UTC), + ) + + +def _independent_link(): + link = MagicMock() + link.agent_id = "agent-123" + link.rest = MagicMock() + p = MagicMock(id="user-1", name="User One", type="User") + link.rest.agent_api_participants.list_agent_chat_participants = AsyncMock( + return_value=MagicMock(data=[p]) + ) + link.rest.agent_api_context.get_agent_chat_context = AsyncMock( + return_value=MagicMock(data=[]) + ) + link.mark_processing = AsyncMock(return_value=True) + link.mark_processed = AsyncMock(return_value=True) + link.mark_failed = AsyncMock(return_value=True) + link.report_activity = AsyncMock(return_value=True) + return link diff --git a/tests/runtime/executionworkingstate/test_sessionconfigguards.py b/tests/runtime/executionworkingstate/test_sessionconfigguards.py new file mode 100644 index 000000000..8431784e0 --- /dev/null +++ b/tests/runtime/executionworkingstate/test_sessionconfigguards.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import pytest + +from band.runtime.sessionconfig import SessionConfig + + +class TestSessionConfigGuards: + def test_defaults_are_valid(self): + cfg = SessionConfig() + assert cfg.enable_working_state is True + assert cfg.working_keep_alive_seconds == 3.0 + assert cfg.working_request_timeout_seconds == 2 + assert cfg.max_working_state_seconds is None + + def test_cadence_must_be_below_ttl_half(self): + with pytest.raises(ValueError): + SessionConfig(working_keep_alive_seconds=5.0) # >= 10/2 + + def test_cadence_must_be_positive(self): + with pytest.raises(ValueError): + SessionConfig(working_keep_alive_seconds=0) + + def test_timeout_must_be_below_cadence(self): + with pytest.raises(ValueError): + SessionConfig( + working_keep_alive_seconds=3.0, working_request_timeout_seconds=3 + ) + + def test_timeout_must_be_positive(self): + with pytest.raises(ValueError): + SessionConfig(working_request_timeout_seconds=0) + + def test_max_duration_if_set_must_be_positive(self): + with pytest.raises(ValueError): + SessionConfig(max_working_state_seconds=0) + + def test_max_cycle_seconds_if_set_must_be_positive(self): + with pytest.raises(ValueError, match="max_cycle_seconds"): + SessionConfig(max_cycle_seconds=0) + with pytest.raises(ValueError, match="max_cycle_seconds"): + SessionConfig(max_cycle_seconds=-1.0) + + def test_disabled_skips_guards(self): + # When disabled, odd values must not raise. + cfg = SessionConfig( + enable_working_state=False, working_keep_alive_seconds=100.0 + ) + assert cfg.enable_working_state is False + + def test_max_cycle_seconds_guard_is_independent_of_working_state(self): + # max_cycle_seconds is a hang-killer unrelated to working-state + # reporting, so its guard must still fire even when working-state + # reporting itself is disabled. + with pytest.raises(ValueError, match="max_cycle_seconds"): + SessionConfig(enable_working_state=False, max_cycle_seconds=0) diff --git a/tests/runtime/executionworkingstate/test_wiring.py b/tests/runtime/executionworkingstate/test_wiring.py new file mode 100644 index 000000000..3ca462dfd --- /dev/null +++ b/tests/runtime/executionworkingstate/test_wiring.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.platform.event import ReconnectedEvent +from band.runtime.execution import ExecutionContext +from band.runtime.sessionconfig import SessionConfig +from tests.conftest import ( + make_message_event, + make_participant_added_event, +) +from tests.runtime.executionworkingstate.kit import ( + _FAST, + _backlog_message, + _independent_link, + _working_values, +) + + +class TestWiring: + @pytest.mark.asyncio + async def test_message_event_reports_true_then_false(self, mock_link, mock_handler): + ctx = ExecutionContext("room-123", mock_link, mock_handler, config=_FAST) + + await ctx._process_event(make_message_event(room_id="room-123", msg_id="m1")) + + assert _working_values(mock_link) == [True, False] + for call in mock_link.report_activity.call_args_list: + assert call.args[0] == "room-123" + + @pytest.mark.asyncio + async def test_reports_false_even_when_handler_raises(self, mock_link): + handler = AsyncMock(side_effect=RuntimeError("boom")) + ctx = ExecutionContext("room-123", mock_link, handler, config=_FAST) + + await ctx._process_event(make_message_event(room_id="room-123", msg_id="m1")) + + assert _working_values(mock_link)[-1] is False + assert _working_values(mock_link).count(False) == 1 + + @pytest.mark.asyncio + async def test_participant_event_does_not_report(self, mock_link, mock_handler): + ctx = ExecutionContext("room-123", mock_link, mock_handler, config=_FAST) + + await ctx._process_event( + make_participant_added_event(room_id="room-123", participant_id="user-2") + ) + + mock_link.report_activity.assert_not_called() + + @pytest.mark.asyncio + async def test_reconnected_event_does_not_report(self, mock_link, mock_handler): + ctx = ExecutionContext("room-123", mock_link, mock_handler, config=_FAST) + + await ctx._process_event(ReconnectedEvent(room_id="room-123")) + + mock_link.report_activity.assert_not_called() + + @pytest.mark.asyncio + async def test_hub_room_does_not_report(self, mock_link, mock_handler): + ctx = ExecutionContext( + "hub-1", mock_link, mock_handler, config=_FAST, hub_room_id="hub-1" + ) + + await ctx._process_event(make_message_event(room_id="hub-1", msg_id="m1")) + + mock_link.report_activity.assert_not_called() + + @pytest.mark.asyncio + async def test_backlog_message_reports_true_then_false( + self, mock_link, mock_handler + ): + ctx = ExecutionContext("room-123", mock_link, mock_handler, config=_FAST) + + await ctx._process_backlog_message(_backlog_message()) + + assert _working_values(mock_link) == [True, False] + + @pytest.mark.asyncio + async def test_disabled_config_does_not_report(self, mock_link, mock_handler): + ctx = ExecutionContext( + "room-123", + mock_link, + mock_handler, + config=SessionConfig(enable_working_state=False), + ) + + await ctx._process_event(make_message_event(room_id="room-123", msg_id="m1")) + + mock_link.report_activity.assert_not_called() + + @pytest.mark.asyncio + async def test_rooms_are_isolated(self, mock_handler): + link_a = _independent_link() + link_b = _independent_link() + ctx_a = ExecutionContext("room-a", link_a, mock_handler, config=_FAST) + ExecutionContext("room-b", link_b, mock_handler, config=_FAST) + + await ctx_a._process_event(make_message_event(room_id="room-a", msg_id="m1")) + + link_a.report_activity.assert_called() + link_b.report_activity.assert_not_called() diff --git a/tests/runtime/test_execution_working_state.py b/tests/runtime/test_execution_working_state.py deleted file mode 100644 index fa39effdf..000000000 --- a/tests/runtime/test_execution_working_state.py +++ /dev/null @@ -1,233 +0,0 @@ -"""Integration tests: ExecutionContext wiring of the working-state reporter.""" - -from __future__ import annotations - -from datetime import UTC, datetime -from unittest.mock import AsyncMock, MagicMock - -import pytest - -from band.platform.event import ReconnectedEvent -from band.runtime.execution import ExecutionContext -from band.runtime.messages import PlatformMessage -from band.runtime.sessionconfig import SessionConfig -from tests.conftest import ( - make_message_event, - make_participant_added_event, -) - - -@pytest.fixture -def mock_link(): - link = MagicMock() - link.agent_id = "agent-123" - link.rest = MagicMock() - - participant1 = MagicMock() - participant1.id = "user-1" - participant1.name = "User One" - participant1.type = "User" - link.rest.agent_api_participants = MagicMock() - link.rest.agent_api_participants.list_agent_chat_participants = AsyncMock( - return_value=MagicMock(data=[participant1]) - ) - link.rest.agent_api_context = MagicMock() - link.rest.agent_api_context.get_agent_chat_context = AsyncMock( - return_value=MagicMock(data=[]) - ) - - link.mark_processing = AsyncMock(return_value=True) - link.mark_processed = AsyncMock(return_value=True) - link.mark_failed = AsyncMock(return_value=True) - link.get_next_message = AsyncMock(return_value=None) - link.get_stale_processing_messages = AsyncMock(return_value=[]) - link.report_activity = AsyncMock(return_value=True) - return link - - -def _working_values(link) -> list[bool]: - """Sequence of `working` booleans passed to link.report_activity.""" - return [ - c.args[1] if len(c.args) > 1 else c.kwargs["working"] - for c in link.report_activity.call_args_list - ] - - -# Keep cadence high enough that no keep-alive fires during a fast test cycle, -# but below the TTL/2 guard. -_FAST = SessionConfig(working_keep_alive_seconds=4.0) - - -def _backlog_message( - room_id: str = "room-123", msg_id: str = "msg-b1" -) -> PlatformMessage: - return PlatformMessage( - id=msg_id, - room_id=room_id, - content="hi", - sender_id="user-1", - sender_type="User", - sender_name="User One", - message_type="text", - metadata={}, - created_at=datetime.now(UTC), - ) - - -class TestWiring: - @pytest.mark.asyncio - async def test_message_event_reports_true_then_false(self, mock_link, mock_handler): - ctx = ExecutionContext("room-123", mock_link, mock_handler, config=_FAST) - - await ctx._process_event(make_message_event(room_id="room-123", msg_id="m1")) - - assert _working_values(mock_link) == [True, False] - for call in mock_link.report_activity.call_args_list: - assert call.args[0] == "room-123" - - @pytest.mark.asyncio - async def test_reports_false_even_when_handler_raises(self, mock_link): - handler = AsyncMock(side_effect=RuntimeError("boom")) - ctx = ExecutionContext("room-123", mock_link, handler, config=_FAST) - - await ctx._process_event(make_message_event(room_id="room-123", msg_id="m1")) - - assert _working_values(mock_link)[-1] is False - assert _working_values(mock_link).count(False) == 1 - - @pytest.mark.asyncio - async def test_participant_event_does_not_report(self, mock_link, mock_handler): - ctx = ExecutionContext("room-123", mock_link, mock_handler, config=_FAST) - - await ctx._process_event( - make_participant_added_event(room_id="room-123", participant_id="user-2") - ) - - mock_link.report_activity.assert_not_called() - - @pytest.mark.asyncio - async def test_reconnected_event_does_not_report(self, mock_link, mock_handler): - ctx = ExecutionContext("room-123", mock_link, mock_handler, config=_FAST) - - await ctx._process_event(ReconnectedEvent(room_id="room-123")) - - mock_link.report_activity.assert_not_called() - - @pytest.mark.asyncio - async def test_hub_room_does_not_report(self, mock_link, mock_handler): - ctx = ExecutionContext( - "hub-1", mock_link, mock_handler, config=_FAST, hub_room_id="hub-1" - ) - - await ctx._process_event(make_message_event(room_id="hub-1", msg_id="m1")) - - mock_link.report_activity.assert_not_called() - - @pytest.mark.asyncio - async def test_backlog_message_reports_true_then_false( - self, mock_link, mock_handler - ): - ctx = ExecutionContext("room-123", mock_link, mock_handler, config=_FAST) - - await ctx._process_backlog_message(_backlog_message()) - - assert _working_values(mock_link) == [True, False] - - @pytest.mark.asyncio - async def test_disabled_config_does_not_report(self, mock_link, mock_handler): - ctx = ExecutionContext( - "room-123", - mock_link, - mock_handler, - config=SessionConfig(enable_working_state=False), - ) - - await ctx._process_event(make_message_event(room_id="room-123", msg_id="m1")) - - mock_link.report_activity.assert_not_called() - - @pytest.mark.asyncio - async def test_rooms_are_isolated(self, mock_handler): - link_a = _independent_link() - link_b = _independent_link() - ctx_a = ExecutionContext("room-a", link_a, mock_handler, config=_FAST) - ExecutionContext("room-b", link_b, mock_handler, config=_FAST) - - await ctx_a._process_event(make_message_event(room_id="room-a", msg_id="m1")) - - link_a.report_activity.assert_called() - link_b.report_activity.assert_not_called() - - -def _independent_link(): - link = MagicMock() - link.agent_id = "agent-123" - link.rest = MagicMock() - p = MagicMock(id="user-1", name="User One", type="User") - link.rest.agent_api_participants.list_agent_chat_participants = AsyncMock( - return_value=MagicMock(data=[p]) - ) - link.rest.agent_api_context.get_agent_chat_context = AsyncMock( - return_value=MagicMock(data=[]) - ) - link.mark_processing = AsyncMock(return_value=True) - link.mark_processed = AsyncMock(return_value=True) - link.mark_failed = AsyncMock(return_value=True) - link.report_activity = AsyncMock(return_value=True) - return link - - -@pytest.fixture -def mock_handler(): - return AsyncMock() - - -class TestSessionConfigGuards: - def test_defaults_are_valid(self): - cfg = SessionConfig() - assert cfg.enable_working_state is True - assert cfg.working_keep_alive_seconds == 3.0 - assert cfg.working_request_timeout_seconds == 2 - assert cfg.max_working_state_seconds is None - - def test_cadence_must_be_below_ttl_half(self): - with pytest.raises(ValueError): - SessionConfig(working_keep_alive_seconds=5.0) # >= 10/2 - - def test_cadence_must_be_positive(self): - with pytest.raises(ValueError): - SessionConfig(working_keep_alive_seconds=0) - - def test_timeout_must_be_below_cadence(self): - with pytest.raises(ValueError): - SessionConfig( - working_keep_alive_seconds=3.0, working_request_timeout_seconds=3 - ) - - def test_timeout_must_be_positive(self): - with pytest.raises(ValueError): - SessionConfig(working_request_timeout_seconds=0) - - def test_max_duration_if_set_must_be_positive(self): - with pytest.raises(ValueError): - SessionConfig(max_working_state_seconds=0) - - def test_max_cycle_seconds_if_set_must_be_positive(self): - with pytest.raises(ValueError, match="max_cycle_seconds"): - SessionConfig(max_cycle_seconds=0) - with pytest.raises(ValueError, match="max_cycle_seconds"): - SessionConfig(max_cycle_seconds=-1.0) - - def test_disabled_skips_guards(self): - # When disabled, odd values must not raise. - cfg = SessionConfig( - enable_working_state=False, working_keep_alive_seconds=100.0 - ) - assert cfg.enable_working_state is False - - def test_max_cycle_seconds_guard_is_independent_of_working_state(self): - # max_cycle_seconds is a hang-killer unrelated to working-state - # reporting, so its guard must still fire even when working-state - # reporting itself is disabled. - with pytest.raises(ValueError, match="max_cycle_seconds"): - SessionConfig(enable_working_state=False, max_cycle_seconds=0) diff --git a/tests/runtime/test_working_state.py b/tests/runtime/test_working_state.py deleted file mode 100644 index 8b0a1bd02..000000000 --- a/tests/runtime/test_working_state.py +++ /dev/null @@ -1,203 +0,0 @@ -"""Tests for WorkingStateReporter (boolean working-state keep-alive).""" - -from __future__ import annotations - -import asyncio - -import pytest - -from band.runtime.working_state import WorkingStateReporter - - -async def _wait_for(predicate, timeout: float = 2.0) -> None: - """Poll until predicate() is truthy or raise on timeout (non-brittle wait).""" - loop = asyncio.get_running_loop() - deadline = loop.time() + timeout - while loop.time() < deadline: - if predicate(): - return - await asyncio.sleep(0.005) - raise AssertionError("condition not met within timeout") - - -class Recorder: - """Async report callback that records the sequence of working values.""" - - def __init__(self, *, fail: bool = False) -> None: - self.calls: list[bool] = [] - self.fail = fail - - async def __call__(self, working: bool) -> bool: - self.calls.append(working) - if self.fail: - raise RuntimeError("report failed") - return True - - -class TestStartStop: - @pytest.mark.asyncio - async def test_start_sends_true_immediately(self): - rec = Recorder() - reporter = WorkingStateReporter(rec, keep_alive_seconds=10.0) - - await reporter.start() - - assert rec.calls == [True] - await reporter.stop() - - @pytest.mark.asyncio - async def test_stop_sends_false(self): - rec = Recorder() - reporter = WorkingStateReporter(rec, keep_alive_seconds=10.0) - - await reporter.start() - await reporter.stop() - - assert rec.calls[0] is True - assert rec.calls[-1] is False - assert rec.calls.count(False) == 1 - - @pytest.mark.asyncio - async def test_stop_without_start_is_noop(self): - rec = Recorder() - reporter = WorkingStateReporter(rec, keep_alive_seconds=10.0) - - await reporter.stop() - - assert rec.calls == [] - - @pytest.mark.asyncio - async def test_disabled_reports_nothing(self): - rec = Recorder() - reporter = WorkingStateReporter(rec, keep_alive_seconds=0.01, enabled=False) - - await reporter.start() - await asyncio.sleep(0.05) - await reporter.stop() - - assert rec.calls == [] - - -class TestKeepAlive: - @pytest.mark.asyncio - async def test_refreshes_true_on_cadence(self): - rec = Recorder() - reporter = WorkingStateReporter(rec, keep_alive_seconds=0.02) - - await reporter.start() - # initial true + at least two refreshes - await _wait_for(lambda: rec.calls.count(True) >= 3) - await reporter.stop() - - assert rec.calls.count(True) >= 3 - assert rec.calls[-1] is False - - @pytest.mark.asyncio - async def test_reentrant_start_does_not_duplicate_keepalive(self): - rec = Recorder() - reporter = WorkingStateReporter(rec, keep_alive_seconds=10.0) - - await reporter.start() - await reporter.start() # already active -> no-op - - assert rec.calls == [True] # only one initial true, one task - await reporter.stop() - - -class TestOrdering: - @pytest.mark.asyncio - async def test_false_always_after_true_even_on_ultra_short_cycle(self): - rec = Recorder() - reporter = WorkingStateReporter(rec, keep_alive_seconds=10.0) - - # Ultra-short cycle: start then immediately stop. - await reporter.start() - await reporter.stop() - - # The lock must guarantee true is sent (and lands) before false. - assert rec.calls[0] is True - assert rec.calls[-1] is False - # No false appears before the first true. - first_false = rec.calls.index(False) - first_true = rec.calls.index(True) - assert first_true < first_false - - -class TestCancellation: - @pytest.mark.asyncio - async def test_false_sent_even_when_cycle_cancelled(self): - rec = Recorder() - reporter = WorkingStateReporter(rec, keep_alive_seconds=10.0) - - async def cycle(): - await reporter.start() - try: - await asyncio.sleep(100) # long-running reasoning - finally: - await reporter.stop() - - task = asyncio.create_task(cycle()) - await _wait_for(lambda: rec.calls == [True]) - - task.cancel() - with pytest.raises(asyncio.CancelledError): - await task - - # The shielded false-send must have completed despite the cancel. - await _wait_for(lambda: rec.calls and rec.calls[-1] is False) - assert rec.calls.count(False) == 1 - - def test_stop_sends_false_when_the_keepalive_loop_has_closed(self) -> None: - rec = Recorder() - reporter = WorkingStateReporter(rec, keep_alive_seconds=10.0) - with asyncio.Runner() as runner: - runner.run(reporter.start()) - - resumed = asyncio.new_event_loop() - try: - resumed.run_until_complete(reporter.stop()) - finally: - resumed.close() - - assert rec.calls == [True, False] - - -class TestFailureSwallowed: - @pytest.mark.asyncio - async def test_report_exception_does_not_propagate(self): - rec = Recorder(fail=True) - reporter = WorkingStateReporter(rec, keep_alive_seconds=0.02) - - # Neither start nor the keep-alive task nor stop may raise. - await reporter.start() - await asyncio.sleep(0.05) - await reporter.stop() - - assert rec.calls # attempts were made - - -class TestMaxDuration: - @pytest.mark.asyncio - async def test_keepalive_stops_after_max_but_stop_still_sends_false(self): - rec = Recorder() - reporter = WorkingStateReporter( - rec, keep_alive_seconds=0.02, max_working_state_seconds=0.05 - ) - - await reporter.start() - # Let the cap elapse and the keep-alive go quiet. - await asyncio.sleep(0.2) - count_after_cap = rec.calls.count(True) - await asyncio.sleep(0.1) - # No further true pings after the cap. - assert rec.calls.count(True) == count_after_cap - - # stop() still sends the authoritative false. - await reporter.stop() - assert rec.calls[-1] is False - - -class TestValidation: - def test_non_positive_cadence_rejected(self): - with pytest.raises(ValueError): - WorkingStateReporter(Recorder(), keep_alive_seconds=0) diff --git a/tests/runtime/workingstate/__init__.py b/tests/runtime/workingstate/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/runtime/workingstate/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/runtime/workingstate/kit.py b/tests/runtime/workingstate/kit.py new file mode 100644 index 000000000..ee553bf4c --- /dev/null +++ b/tests/runtime/workingstate/kit.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import asyncio + + +async def _wait_for(predicate, timeout: float = 2.0) -> None: + """Poll until predicate() is truthy or raise on timeout (non-brittle wait).""" + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + while loop.time() < deadline: + if predicate(): + return + await asyncio.sleep(0.005) + raise AssertionError("condition not met within timeout") + + +class Recorder: + """Async report callback that records the sequence of working values.""" + + def __init__(self, *, fail: bool = False) -> None: + self.calls: list[bool] = [] + self.fail = fail + + async def __call__(self, working: bool) -> bool: + self.calls.append(working) + if self.fail: + raise RuntimeError("report failed") + return True diff --git a/tests/runtime/workingstate/test_cancellation.py b/tests/runtime/workingstate/test_cancellation.py new file mode 100644 index 000000000..9cb55c023 --- /dev/null +++ b/tests/runtime/workingstate/test_cancellation.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from band.runtime.working_state import WorkingStateReporter +from tests.runtime.workingstate.kit import Recorder, _wait_for + + +class TestCancellation: + @pytest.mark.asyncio + async def test_false_sent_even_when_cycle_cancelled(self): + rec = Recorder() + reporter = WorkingStateReporter(rec, keep_alive_seconds=10.0) + + async def cycle(): + await reporter.start() + try: + await asyncio.sleep(100) # long-running reasoning + finally: + await reporter.stop() + + task = asyncio.create_task(cycle()) + await _wait_for(lambda: rec.calls == [True]) + + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + # The shielded false-send must have completed despite the cancel. + await _wait_for(lambda: rec.calls and rec.calls[-1] is False) + assert rec.calls.count(False) == 1 + + def test_stop_sends_false_when_the_keepalive_loop_has_closed(self) -> None: + rec = Recorder() + reporter = WorkingStateReporter(rec, keep_alive_seconds=10.0) + with asyncio.Runner() as runner: + runner.run(reporter.start()) + + resumed = asyncio.new_event_loop() + try: + resumed.run_until_complete(reporter.stop()) + finally: + resumed.close() + + assert rec.calls == [True, False] diff --git a/tests/runtime/workingstate/test_failureswallowed.py b/tests/runtime/workingstate/test_failureswallowed.py new file mode 100644 index 000000000..0dbe23eef --- /dev/null +++ b/tests/runtime/workingstate/test_failureswallowed.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from band.runtime.working_state import WorkingStateReporter +from tests.runtime.workingstate.kit import Recorder + + +class TestFailureSwallowed: + @pytest.mark.asyncio + async def test_report_exception_does_not_propagate(self): + rec = Recorder(fail=True) + reporter = WorkingStateReporter(rec, keep_alive_seconds=0.02) + + # Neither start nor the keep-alive task nor stop may raise. + await reporter.start() + await asyncio.sleep(0.05) + await reporter.stop() + + assert rec.calls # attempts were made diff --git a/tests/runtime/workingstate/test_keepalive.py b/tests/runtime/workingstate/test_keepalive.py new file mode 100644 index 000000000..ee7a06a58 --- /dev/null +++ b/tests/runtime/workingstate/test_keepalive.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import pytest + +from band.runtime.working_state import WorkingStateReporter +from tests.runtime.workingstate.kit import Recorder, _wait_for + + +class TestKeepAlive: + @pytest.mark.asyncio + async def test_refreshes_true_on_cadence(self): + rec = Recorder() + reporter = WorkingStateReporter(rec, keep_alive_seconds=0.02) + + await reporter.start() + # initial true + at least two refreshes + await _wait_for(lambda: rec.calls.count(True) >= 3) + await reporter.stop() + + assert rec.calls.count(True) >= 3 + assert rec.calls[-1] is False + + @pytest.mark.asyncio + async def test_reentrant_start_does_not_duplicate_keepalive(self): + rec = Recorder() + reporter = WorkingStateReporter(rec, keep_alive_seconds=10.0) + + await reporter.start() + await reporter.start() # already active -> no-op + + assert rec.calls == [True] # only one initial true, one task + await reporter.stop() diff --git a/tests/runtime/workingstate/test_maxduration.py b/tests/runtime/workingstate/test_maxduration.py new file mode 100644 index 000000000..b16c313f4 --- /dev/null +++ b/tests/runtime/workingstate/test_maxduration.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from band.runtime.working_state import WorkingStateReporter +from tests.runtime.workingstate.kit import Recorder + + +class TestMaxDuration: + @pytest.mark.asyncio + async def test_keepalive_stops_after_max_but_stop_still_sends_false(self): + rec = Recorder() + reporter = WorkingStateReporter( + rec, keep_alive_seconds=0.02, max_working_state_seconds=0.05 + ) + + await reporter.start() + # Let the cap elapse and the keep-alive go quiet. + await asyncio.sleep(0.2) + count_after_cap = rec.calls.count(True) + await asyncio.sleep(0.1) + # No further true pings after the cap. + assert rec.calls.count(True) == count_after_cap + + # stop() still sends the authoritative false. + await reporter.stop() + assert rec.calls[-1] is False diff --git a/tests/runtime/workingstate/test_ordering.py b/tests/runtime/workingstate/test_ordering.py new file mode 100644 index 000000000..d15ff8289 --- /dev/null +++ b/tests/runtime/workingstate/test_ordering.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import pytest + +from band.runtime.working_state import WorkingStateReporter +from tests.runtime.workingstate.kit import Recorder + + +class TestOrdering: + @pytest.mark.asyncio + async def test_false_always_after_true_even_on_ultra_short_cycle(self): + rec = Recorder() + reporter = WorkingStateReporter(rec, keep_alive_seconds=10.0) + + # Ultra-short cycle: start then immediately stop. + await reporter.start() + await reporter.stop() + + # The lock must guarantee true is sent (and lands) before false. + assert rec.calls[0] is True + assert rec.calls[-1] is False + # No false appears before the first true. + first_false = rec.calls.index(False) + first_true = rec.calls.index(True) + assert first_true < first_false diff --git a/tests/runtime/workingstate/test_startstop.py b/tests/runtime/workingstate/test_startstop.py new file mode 100644 index 000000000..0b9f2bfb5 --- /dev/null +++ b/tests/runtime/workingstate/test_startstop.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from band.runtime.working_state import WorkingStateReporter +from tests.runtime.workingstate.kit import Recorder + + +class TestStartStop: + @pytest.mark.asyncio + async def test_start_sends_true_immediately(self): + rec = Recorder() + reporter = WorkingStateReporter(rec, keep_alive_seconds=10.0) + + await reporter.start() + + assert rec.calls == [True] + await reporter.stop() + + @pytest.mark.asyncio + async def test_stop_sends_false(self): + rec = Recorder() + reporter = WorkingStateReporter(rec, keep_alive_seconds=10.0) + + await reporter.start() + await reporter.stop() + + assert rec.calls[0] is True + assert rec.calls[-1] is False + assert rec.calls.count(False) == 1 + + @pytest.mark.asyncio + async def test_stop_without_start_is_noop(self): + rec = Recorder() + reporter = WorkingStateReporter(rec, keep_alive_seconds=10.0) + + await reporter.stop() + + assert rec.calls == [] + + @pytest.mark.asyncio + async def test_disabled_reports_nothing(self): + rec = Recorder() + reporter = WorkingStateReporter(rec, keep_alive_seconds=0.01, enabled=False) + + await reporter.start() + await asyncio.sleep(0.05) + await reporter.stop() + + assert rec.calls == [] diff --git a/tests/runtime/workingstate/test_validation.py b/tests/runtime/workingstate/test_validation.py new file mode 100644 index 000000000..36ffbbc15 --- /dev/null +++ b/tests/runtime/workingstate/test_validation.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +import pytest + +from band.runtime.working_state import WorkingStateReporter +from tests.runtime.workingstate.kit import Recorder + + +class TestValidation: + def test_non_positive_cadence_rejected(self): + with pytest.raises(ValueError): + WorkingStateReporter(Recorder(), keep_alive_seconds=0) From 8ec26b33cf3ff15f27d6149daa27a0d4ea68c17d Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 18 Oct 2026 10:25:08 +0300 Subject: [PATCH 02/47] fix: Preserve standalone worker binding across tool wrappers --- src/band/runtime/tools/agent.py | 7 +++++-- tests/runtime/helpers.py | 9 ++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/band/runtime/tools/agent.py b/src/band/runtime/tools/agent.py index 6951d5492..88d53634b 100644 --- a/src/band/runtime/tools/agent.py +++ b/src/band/runtime/tools/agent.py @@ -388,8 +388,11 @@ def from_context( return tools def register_turn_task(self, owner: TurnTask) -> None: - if self._scope is not None and self._ctx is not None: - self._ctx.register_turn_task(self._scope, owner) + if self._scope is not None: + if self._ctx is None: + self._scope.bind(owner) + else: + self._ctx.register_turn_task(self._scope, owner) def _abort_stopped_post(self, error: RoomExecutionStoppedError) -> None: if self._ctx is not None and self._scope is not None: diff --git a/tests/runtime/helpers.py b/tests/runtime/helpers.py index 0a96e4340..0a04accde 100644 --- a/tests/runtime/helpers.py +++ b/tests/runtime/helpers.py @@ -296,13 +296,16 @@ class PostingPlatform(LifecyclePlatform): def answer(self, request: httpx.Request) -> httpx.Response: tail = request.url.path.rsplit("/", 1)[-1] if tail == "participants": - return page([ALICE]) + return page([{**ALICE, "role": "member", "status": "active"}]) if request.method == "POST" and tail == "events": return httpx.Response( - 201, json={"data": {"id": "e-1", "message_type": "event"}} + 201, + json={"data": {"id": "e-1", "message_type": "event", "success": True}}, ) if request.method == "POST" and tail == "messages": - return httpx.Response(201, json={"data": {"id": "m-1", "recipients": []}}) + return httpx.Response( + 201, json={"data": {"id": "m-1", "recipients": [], "success": True}} + ) return super().answer(request) From 1476457ca6f33d8eb0f43918727d9cf16c88455b Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 18 Oct 2026 11:09:08 +0300 Subject: [PATCH 03/47] test: Avoid the E2E peer fixture reservation in runtime tests --- tests/runtime/activityownership/conftest.py | 6 +-- .../activityownership/test_lifetime.py | 46 +++++++++---------- .../activityownership/test_opencode.py | 10 ++-- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/tests/runtime/activityownership/conftest.py b/tests/runtime/activityownership/conftest.py index 916e0ae7b..16bcf6bfc 100644 --- a/tests/runtime/activityownership/conftest.py +++ b/tests/runtime/activityownership/conftest.py @@ -20,16 +20,16 @@ @pytest.fixture -async def peer() -> AsyncIterator[LifecyclePlatform]: +async def activity_platform() -> AsyncIterator[LifecyclePlatform]: yield ActivityPlatform() @pytest.fixture async def execution( - peer: LifecyclePlatform, + activity_platform: LifecyclePlatform, ) -> AsyncIterator[Callable[..., ExecutionContext]]: contexts: list[ExecutionContext] = [] - async with rest_client_over(peer.answer) as rest: + async with rest_client_over(activity_platform.answer) as rest: def create( handler: Callable[[ExecutionContext, PlatformEvent], Awaitable[None]], diff --git a/tests/runtime/activityownership/test_lifetime.py b/tests/runtime/activityownership/test_lifetime.py index 1dcf0f921..c6d8adaf5 100644 --- a/tests/runtime/activityownership/test_lifetime.py +++ b/tests/runtime/activityownership/test_lifetime.py @@ -17,7 +17,7 @@ class TestProviderLifetime: @pytest.mark.looptime async def test_detached_worker_keeps_heartbeat_across_busy_delivery( - self, execution: Callable[..., ExecutionContext], peer: LifecyclePlatform + self, execution: Callable[..., ExecutionContext], activity_platform: LifecyclePlatform ) -> None: release = asyncio.Event() workers: list[TurnTask] = [] @@ -32,16 +32,16 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: await ctx._process_event(make_message_event(msg_id="original")) await asyncio.sleep(3.1) await ctx._process_event(make_message_event(msg_id="busy")) - assert peer.activity_reports == [KEEP_ALIVE, KEEP_ALIVE] - assert peer.marked("processed") == ["original", "busy"] + assert activity_platform.activity_reports == [KEEP_ALIVE, KEEP_ALIVE] + assert activity_platform.marked("processed") == ["original", "busy"] release.set() await workers[0].task await asyncio.sleep(0) await ctx._join_activity_completion() - assert peer.activity_reports[-1] == IDLE + assert activity_platform.activity_reports[-1] == IDLE async def test_worker_completion_does_not_clear_running_handler( - self, execution: Callable[..., ExecutionContext], peer: LifecyclePlatform + self, execution: Callable[..., ExecutionContext], activity_platform: LifecyclePlatform ) -> None: finished, release = asyncio.Event(), asyncio.Event() @@ -55,16 +55,16 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: ctx = execution(handler) turn = asyncio.create_task(ctx._process_event(make_message_event())) await finished.wait() - assert peer.activity_reports == [KEEP_ALIVE] + assert activity_platform.activity_reports == [KEEP_ALIVE] release.set() await turn - assert peer.activity_reports == [KEEP_ALIVE, IDLE] + assert activity_platform.activity_reports == [KEEP_ALIVE, IDLE] @pytest.mark.parametrize("mode", [ControlMode.STOP, ControlMode.INTERRUPT]) async def test_detached_control_cancels_worker_preserving_settled_ack( self, execution: Callable[..., ExecutionContext], - peer: LifecyclePlatform, + activity_platform: LifecyclePlatform, mode: ControlMode, ) -> None: workers: list[TurnTask] = [] @@ -83,12 +83,12 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: await asyncio.gather(workers[0].task, return_exceptions=True) await ctx._join_activity_completion() assert workers[0].task.cancelled() - assert peer.marked("processed") == ["original"] - assert peer.activity_reports == [KEEP_ALIVE, IDLE] + assert activity_platform.marked("processed") == ["original"] + assert activity_platform.activity_reports == [KEEP_ALIVE, IDLE] @pytest.mark.looptime async def test_detached_heartbeat_stop_preserves_ack( - self, execution: Callable[..., ExecutionContext], peer: LifecyclePlatform + self, execution: Callable[..., ExecutionContext], activity_platform: LifecyclePlatform ) -> None: workers: list[TurnTask] = [] @@ -104,15 +104,15 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: ctx = execution(handler) await ctx._process_event(make_message_event(msg_id="original")) await ctx._process_event(make_message_event(msg_id="busy")) - peer.stopped = True + activity_platform.stopped = True await asyncio.sleep(3.1) assert workers[0].task.cancelled() - assert peer.marked("processed") == ["original", "busy"] - assert peer.activity_reports[-1] == IDLE + assert activity_platform.marked("processed") == ["original", "busy"] + assert activity_platform.activity_reports[-1] == IDLE @pytest.mark.looptime async def test_busy_delivery_keeps_original_cap_deadline( - self, execution: Callable[..., ExecutionContext], peer: LifecyclePlatform + self, execution: Callable[..., ExecutionContext], activity_platform: LifecyclePlatform ) -> None: release = asyncio.Event() workers: list[TurnTask] = [] @@ -131,16 +131,16 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: await asyncio.sleep(3.1) await ctx._process_event(make_message_event(msg_id="busy")) await asyncio.sleep(6) - assert peer.activity_reports == [KEEP_ALIVE, KEEP_ALIVE] + assert activity_platform.activity_reports == [KEEP_ALIVE, KEEP_ALIVE] assert not workers[0].task.done() release.set() await workers[0].task await asyncio.sleep(0) await ctx._join_activity_completion() - assert peer.activity_reports[-1] == IDLE + assert activity_platform.activity_reports[-1] == IDLE async def test_shutdown_clears_reporting_without_cancelling_detached_worker( - self, execution: Callable[..., ExecutionContext], peer: LifecyclePlatform + self, execution: Callable[..., ExecutionContext], activity_platform: LifecyclePlatform ) -> None: release = asyncio.Event() workers: list[TurnTask] = [] @@ -155,14 +155,14 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: ctx = execution(handler) await ctx._process_event(make_message_event()) await ctx.stop() - assert peer.activity_reports == [KEEP_ALIVE, IDLE] + assert activity_platform.activity_reports == [KEEP_ALIVE, IDLE] assert not workers[0].task.done() release.set() await workers[0].task - assert peer.activity_reports == [KEEP_ALIVE, IDLE] + assert activity_platform.activity_reports == [KEEP_ALIVE, IDLE] async def test_new_worker_admitted_after_previous_completion_gets_own_indicator( - self, execution: Callable[..., ExecutionContext], peer: LifecyclePlatform + self, execution: Callable[..., ExecutionContext], activity_platform: LifecyclePlatform ) -> None: releases = [asyncio.Event(), asyncio.Event()] entered, admitted = asyncio.Event(), asyncio.Event() @@ -190,9 +190,9 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: await ctx._join_activity_completion() admitted.set() await turn - assert peer.activity_reports == [KEEP_ALIVE, IDLE, KEEP_ALIVE] + assert activity_platform.activity_reports == [KEEP_ALIVE, IDLE, KEEP_ALIVE] releases[1].set() await workers[1].task await asyncio.sleep(0) await ctx._join_activity_completion() - assert peer.activity_reports == [KEEP_ALIVE, IDLE, KEEP_ALIVE, IDLE] + assert activity_platform.activity_reports == [KEEP_ALIVE, IDLE, KEEP_ALIVE, IDLE] diff --git a/tests/runtime/activityownership/test_opencode.py b/tests/runtime/activityownership/test_opencode.py index 7fb26af66..7d8e09879 100644 --- a/tests/runtime/activityownership/test_opencode.py +++ b/tests/runtime/activityownership/test_opencode.py @@ -27,7 +27,7 @@ class TestOpenCodeLifetime: async def test_parked_native_turn_dual_dispatch_aborts_remote_once( self, execution: Callable[..., ExecutionContext], - peer: LifecyclePlatform, + activity_platform: LifecyclePlatform, mode: ControlMode, ) -> None: client = FakeOpencodeClient( @@ -59,8 +59,8 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: ctx = execution(handler) try: await ctx._process_event(make_message_event(msg_id="original")) - assert peer.activity_reports == [KEEP_ALIVE] - assert peer.marked("processed") == ["original"] + assert activity_platform.activity_reports == [KEEP_ALIVE] + assert activity_platform.marked("processed") == ["original"] assert ctx.interrupt(kind=mode) notify = asyncio.create_task(adapter.on_interrupt(ROOM_ID, mode)) await aborting.wait() @@ -70,8 +70,8 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: await ctx._join_activity_completion() assert client.aborted_sessions == ["sess-1"] assert cleaned.is_set() - assert peer.activity_reports == [KEEP_ALIVE, IDLE] - assert peer.marked("processed") == ["original"] + assert activity_platform.activity_reports == [KEEP_ALIVE, IDLE] + assert activity_platform.marked("processed") == ["original"] finally: release.set() await adapter.on_cleanup(ROOM_ID) From 8bfeac07a5dc34eaf450fd6b1b3f418a9f41c8e8 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 18 Oct 2026 11:11:32 +0300 Subject: [PATCH 04/47] style: Format the renamed activity platform fixture --- .../activityownership/test_lifetime.py | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/tests/runtime/activityownership/test_lifetime.py b/tests/runtime/activityownership/test_lifetime.py index c6d8adaf5..225251892 100644 --- a/tests/runtime/activityownership/test_lifetime.py +++ b/tests/runtime/activityownership/test_lifetime.py @@ -17,7 +17,9 @@ class TestProviderLifetime: @pytest.mark.looptime async def test_detached_worker_keeps_heartbeat_across_busy_delivery( - self, execution: Callable[..., ExecutionContext], activity_platform: LifecyclePlatform + self, + execution: Callable[..., ExecutionContext], + activity_platform: LifecyclePlatform, ) -> None: release = asyncio.Event() workers: list[TurnTask] = [] @@ -41,7 +43,9 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: assert activity_platform.activity_reports[-1] == IDLE async def test_worker_completion_does_not_clear_running_handler( - self, execution: Callable[..., ExecutionContext], activity_platform: LifecyclePlatform + self, + execution: Callable[..., ExecutionContext], + activity_platform: LifecyclePlatform, ) -> None: finished, release = asyncio.Event(), asyncio.Event() @@ -88,7 +92,9 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: @pytest.mark.looptime async def test_detached_heartbeat_stop_preserves_ack( - self, execution: Callable[..., ExecutionContext], activity_platform: LifecyclePlatform + self, + execution: Callable[..., ExecutionContext], + activity_platform: LifecyclePlatform, ) -> None: workers: list[TurnTask] = [] @@ -112,7 +118,9 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: @pytest.mark.looptime async def test_busy_delivery_keeps_original_cap_deadline( - self, execution: Callable[..., ExecutionContext], activity_platform: LifecyclePlatform + self, + execution: Callable[..., ExecutionContext], + activity_platform: LifecyclePlatform, ) -> None: release = asyncio.Event() workers: list[TurnTask] = [] @@ -140,7 +148,9 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: assert activity_platform.activity_reports[-1] == IDLE async def test_shutdown_clears_reporting_without_cancelling_detached_worker( - self, execution: Callable[..., ExecutionContext], activity_platform: LifecyclePlatform + self, + execution: Callable[..., ExecutionContext], + activity_platform: LifecyclePlatform, ) -> None: release = asyncio.Event() workers: list[TurnTask] = [] @@ -162,7 +172,9 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: assert activity_platform.activity_reports == [KEEP_ALIVE, IDLE] async def test_new_worker_admitted_after_previous_completion_gets_own_indicator( - self, execution: Callable[..., ExecutionContext], activity_platform: LifecyclePlatform + self, + execution: Callable[..., ExecutionContext], + activity_platform: LifecyclePlatform, ) -> None: releases = [asyncio.Event(), asyncio.Event()] entered, admitted = asyncio.Event(), asyncio.Event() @@ -195,4 +207,9 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: await workers[1].task await asyncio.sleep(0) await ctx._join_activity_completion() - assert activity_platform.activity_reports == [KEEP_ALIVE, IDLE, KEEP_ALIVE, IDLE] + assert activity_platform.activity_reports == [ + KEEP_ALIVE, + IDLE, + KEEP_ALIVE, + IDLE, + ] From a397d8cb905d5316a87f62da23c8c5af1895e78e Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 18 Oct 2026 14:03:47 +0300 Subject: [PATCH 05/47] test: Use real time for randomized scheduler grace controls --- tests/runtime/turnadmission/test_grace_controls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/runtime/turnadmission/test_grace_controls.py b/tests/runtime/turnadmission/test_grace_controls.py index 42bc64fdd..0e2e4f44b 100644 --- a/tests/runtime/turnadmission/test_grace_controls.py +++ b/tests/runtime/turnadmission/test_grace_controls.py @@ -24,8 +24,8 @@ WITHIN_GRACE = CYCLE_CANCEL_GRACE_SECONDS / 2 +# Real time avoids looptime rounding randomized reconciliation deadlines down. @pytest.mark.timeout(10) -@pytest.mark.looptime class TestGraceControls: """STOP during the platform-stop observer's grace leaves the next turn unstarted.""" From 0342d962d41cf3234daff3781f5d4e902b2be521 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 18 Oct 2026 10:17:52 +0300 Subject: [PATCH 06/47] feat: Deliver compatible activity detail through the working-state reporter --- docs/turn-outcome.md | 11 ++ src/band/platform/activity.py | 95 ++++++++++ src/band/platform/link.py | 4 + src/band/platform/message_lifecycle.py | 73 ++++++-- src/band/runtime/execution.py | 59 +++++- src/band/runtime/sessionconfig.py | 7 +- src/band/runtime/tools/agent.py | 3 + src/band/runtime/working_state.py | 173 +++++++++++++----- tests/platform/activity/conftest.py | 38 ++++ tests/platform/activity/kit.py | 44 +++++ tests/platform/activity/test_coordination.py | 79 ++++++++ tests/platform/activity/test_deadlines.py | 44 +++++ tests/platform/activity/test_fallback.py | 64 +++++++ tests/platform/activity/test_versions.py | 58 ++++++ tests/runtime/activityownership/conftest.py | 21 ++- tests/runtime/activityownership/kit.py | 21 +++ .../runtime/activityownership/test_detail.py | 144 +++++++++++++++ tests/runtime/conftest.py | 14 ++ tests/runtime/workingstate/test_detail.py | 106 +++++++++++ 19 files changed, 984 insertions(+), 74 deletions(-) create mode 100644 src/band/platform/activity.py create mode 100644 tests/platform/activity/conftest.py create mode 100644 tests/platform/activity/kit.py create mode 100644 tests/platform/activity/test_coordination.py create mode 100644 tests/platform/activity/test_deadlines.py create mode 100644 tests/platform/activity/test_fallback.py create mode 100644 tests/platform/activity/test_versions.py create mode 100644 tests/runtime/activityownership/test_detail.py create mode 100644 tests/runtime/workingstate/test_detail.py diff --git a/docs/turn-outcome.md b/docs/turn-outcome.md index 6951fad99..d674c64a4 100644 --- a/docs/turn-outcome.md +++ b/docs/turn-outcome.md @@ -410,3 +410,14 @@ once. It does not rewrite the original delivery's settled acknowledgement. Explicit interruption invalidates its activity; leave and shutdown clear the indicator while adapter residency retains responsibility for released workers. The authoritative idle report finishes before a replacement reports working. + +`SessionConfig.enable_working_state` controls both the indicator and activity +detail, independently of chat narration and tool capabilities. Changes and +keep-alives share one serialized reporter. The duration cap stops both kinds +of refresh without cancelling provider work; actual completion still clears. + +Detail compatibility is negotiated once per process using the first platform's +`agent_version`. Failed or unsupported discovery preserves boolean reporting. +A detail-bearing HTTP 422 permanently disables detail across all links in the +process and retries that report once without detail, within its original budget. +The version lookup uses at most half that budget, reserving time for liveness. diff --git a/src/band/platform/activity.py b/src/band/platform/activity.py new file mode 100644 index 000000000..beb48c74e --- /dev/null +++ b/src/band/platform/activity.py @@ -0,0 +1,95 @@ +"""Process-wide negotiation for optional room activity detail. + +The first endpoint defines support for the process. A detail rejection on any +endpoint permanently downgrades every link; boolean liveness remains available. +""" + +from __future__ import annotations + +import asyncio +import logging +import re +from concurrent.futures import Future +from threading import Lock + +import httpx + +logger = logging.getLogger(__name__) +MINIMUM_ACTIVITY_VERSION = (1, 25, 0) +RELEASE_VERSION = re.compile(r"([0-9]+)\.([0-9]+)\.([0-9]+)") + + +class ActivitySupport: + """Share discovery without retaining primitives owned by an asyncio loop.""" + + def __init__(self) -> None: + self._lock = Lock() + self._discovery: Future[bool] | None = None + self._disabled = False + + @property + def supported(self) -> bool: + with self._lock: + return ( + not self._disabled + and self._discovery is not None + and self._discovery.done() + and self._discovery.result() + ) + + def disable(self) -> None: + with self._lock: + already_disabled = self._disabled + self._disabled = True + if not already_disabled: + logger.info( + "Activity detail rejected; using boolean reporting for this process" + ) + + async def allows(self, rest_url: str, budget: float) -> bool: + with self._lock: + if self._disabled: + return False + owner = self._discovery is None + if owner: + self._discovery = Future() + discovery = self._discovery + assert discovery is not None + if owner: + supported = False + try: + async with asyncio.timeout(budget): + supported = await self._discover(rest_url, budget) + except asyncio.CancelledError: + raise + except (httpx.HTTPError, ValueError, TimeoutError): + pass + finally: + discovery.set_result(supported) + if not supported: + logger.info( + "Activity detail unavailable; using boolean reporting for this process" + ) + else: + try: + async with asyncio.timeout(budget): + await asyncio.shield(asyncio.wrap_future(discovery)) + except TimeoutError: + return False + return self.supported + + @staticmethod + async def _discover(rest_url: str, budget: float) -> bool: + async with httpx.AsyncClient(timeout=budget) as client: + response = await client.get(f"{rest_url.rstrip('/')}/api/version") + response.raise_for_status() + data = response.json() + version = data.get("agent_version") if isinstance(data, dict) else None + match = RELEASE_VERSION.fullmatch(version) if isinstance(version, str) else None + return ( + match is not None + and tuple(map(int, match.groups())) >= MINIMUM_ACTIVITY_VERSION + ) + + +activity_support = ActivitySupport() diff --git a/src/band/platform/link.py b/src/band/platform/link.py index 524c3cd07..977d0451b 100644 --- a/src/band/platform/link.py +++ b/src/band/platform/link.py @@ -12,6 +12,7 @@ from collections.abc import Awaitable, Callable from typing import TYPE_CHECKING +from band_rest.types.agent_activity_detail import AgentActivityDetail from band_sdk_core import AgentTopicKind, SessionState from band.client.rest import AsyncRestClient @@ -584,6 +585,7 @@ async def report_activity( room_id: str, working: bool, *, + activity: AgentActivityDetail | None = None, timeout_seconds: int = 2, on_stopped: Callable[[], None] | None = None, ) -> bool: @@ -591,6 +593,8 @@ async def report_activity( self.rest, room_id, working, + activity=activity, + rest_url=self.rest_url, timeout_seconds=timeout_seconds, on_stopped=on_stopped, ) diff --git a/src/band/platform/message_lifecycle.py b/src/band/platform/message_lifecycle.py index 2c6d1c8df..3f6ab070d 100644 --- a/src/band/platform/message_lifecycle.py +++ b/src/band/platform/message_lifecycle.py @@ -6,20 +6,27 @@ from __future__ import annotations +import asyncio import logging from collections.abc import Callable from dataclasses import dataclass from datetime import UTC, datetime from http import HTTPStatus +from math import ceil from typing import Any, TypeVar +from band_rest.agent_api_activity.types.report_agent_chat_activity_response import ( + ReportAgentChatActivityResponse, +) from band_rest.core.api_error import ApiError from band_rest.core.http_response import AsyncHttpResponse +from band_rest.types.agent_activity_detail import AgentActivityDetail from band_rest.types.chat_message import ChatMessage from band.client.rest import DEFAULT_REQUEST_OPTIONS, AsyncRestClient from band.core.exceptions import RoomExecutionStoppedError from band.core.types import metadata_to_dict +from band.platform.activity import activity_support from band.runtime.messages import PlatformMessage logger = logging.getLogger(__name__) @@ -167,6 +174,8 @@ async def report_activity( room_id: str, working: bool, *, + activity: AgentActivityDetail | None = None, + rest_url: str | None = None, timeout_seconds: int = 2, on_stopped: Callable[[], None] | None = None, ) -> bool: @@ -179,26 +188,27 @@ async def report_activity( the backstop and activity reporting must never break message processing. - ``timeout_seconds`` bounds each POST so a slow/half-open endpoint - can't stall the keep-alive or wedge teardown. Retries are off: a - dropped keep-alive gets re-sent next cadence tick, and a dropped - ``false`` is cleared by the platform TTL either way — retrying would - only add latency. + One deadline bounds discovery, POST and the sole compatibility retry. + A rejected detail permanently downgrades the process to boolean reports; + other failures wait for the next keep-alive or platform TTL. A successful ``working=True`` report echoed as ``False`` identifies a stopped execution. Notify ``on_stopped`` separately from the request's success boolean; transport failures never imply STOP. """ try: - response = await rest.agent_api_activity.report_agent_chat_activity( - chat_id=room_id, - working=working, - request_options={ - "timeout_in_seconds": timeout_seconds, - "max_retries": 0, - }, - ) - stopped = working and response.data.working is False + async with asyncio.timeout(timeout_seconds): + deadline = asyncio.get_running_loop().time() + timeout_seconds + detail = activity if working else None + if detail is not None and ( + rest_url is None + or not await activity_support.allows(rest_url, timeout_seconds / 2) + ): + detail = None + response = await self._post_activity( + rest, room_id, working, detail, deadline + ) + stopped = working and response.data.working is False except Exception as e: # noqa: BLE001 -- best-effort event emission must not crash the turn/link if not self._activity_report_failing: self._activity_report_failing = True @@ -224,6 +234,41 @@ async def report_activity( on_stopped() return True + @staticmethod + async def _post_activity( + rest: AsyncRestClient, + room_id: str, + working: bool, + activity: AgentActivityDetail | None, + deadline: float, + ) -> ReportAgentChatActivityResponse: + kwargs: dict[str, Any] = {} + if activity is not None and activity_support.supported: + kwargs["activity"] = activity + for attempt in range(2): + try: + return await rest.agent_api_activity.report_agent_chat_activity( + chat_id=room_id, + working=working, + request_options={ + "timeout_in_seconds": max( + 1, ceil(deadline - asyncio.get_running_loop().time()) + ), + "max_retries": 0, + }, + **kwargs, + ) + except ApiError as error: + if ( + attempt + or "activity" not in kwargs + or error.status_code != HTTPStatus.UNPROCESSABLE_ENTITY + ): + raise + activity_support.disable() + kwargs.clear() + raise AssertionError("Activity compatibility retry did not return") + async def get_next_message( self, rest: AsyncRestClient, room_id: str ) -> PlatformMessage | None: diff --git a/src/band/runtime/execution.py b/src/band/runtime/execution.py index 701f25ded..89df6db79 100644 --- a/src/band/runtime/execution.py +++ b/src/band/runtime/execution.py @@ -19,6 +19,7 @@ import logging from collections import deque from collections.abc import Awaitable, Callable, Iterator +from dataclasses import dataclass from datetime import UTC, datetime from enum import Enum, StrEnum from functools import partial @@ -30,6 +31,7 @@ runtime_checkable, ) +from band_rest.types.agent_activity_detail import AgentActivityDetail from band_sdk_core import ( AgentFailure, ClaimRegistry, @@ -84,7 +86,7 @@ from band.runtime.resync import ResyncSchedule from band.runtime.sessionconfig import SessionConfig from band.runtime.tools.agent import AgentTools -from band.runtime.working_state import WorkingStateReporter +from band.runtime.working_state import ActivityObserver, WorkingStateReporter if TYPE_CHECKING: from band.platform.link import BandLink @@ -281,6 +283,18 @@ async def resume_room(self) -> None: ExecutionHandler = Callable[["ExecutionContext", PlatformEvent], Awaitable[None]] +@dataclass(frozen=True) +class ScopeActivityObserver: + context: ExecutionContext + scope: TurnScope + + async def update(self, activity: AgentActivityDetail) -> None: + await self.context.observe_activity(self.scope, activity=activity) + + async def set_manual_wait(self, waiting: bool) -> None: + await self.context.observe_activity(self.scope, waiting=waiting) + + class ExecutionContext: """ Default execution: context accumulation model. @@ -431,10 +445,9 @@ def __init__( # The first stop's graceful result; a retry only settles retained work. self._stop_graceful = True - # Optional seam for clearing user-visible activity state ("reasoning…") - # when a cycle is interrupted/stopped. Filled by the activity-signal - # work; a no-op (None) here. - self._on_activity_clear: Callable[[], Awaitable[None]] | None = None + self._on_activity_clear: Callable[[], Awaitable[None]] | None = ( + self._clear_scope_activity + ) @property def thread_id(self) -> str: @@ -1853,9 +1866,11 @@ async def _invoke_handler(self, event: PlatformEvent) -> None: async def _report_working_state(self, working: bool) -> bool: """Report the room's boolean working state (wired into the reporter).""" + detail = self._working_reporter.activity if working else None return await self.link.report_activity( self.room_id, working, + **({"activity": detail} if detail is not None else {}), timeout_seconds=self.config.working_request_timeout_seconds, on_stopped=partial( self._observe_activity_stop, @@ -1919,7 +1934,39 @@ async def _start_activity(self, scope: TurnScope) -> None: if self._activity_owner is not scope: self._activity_stop_scope = None self._activity_owner = scope - await self._working_reporter.start() + await self._working_reporter.start(owner=scope) + + def activity_observer(self, scope: TurnScope | None) -> ActivityObserver | None: + return ScopeActivityObserver(self, scope) if scope is not None else None + + def owns_activity(self, scope: TurnScope) -> bool: + return self._activity_owner is scope and not scope.aborted and not self._closing + + async def observe_activity( + self, + scope: TurnScope, + *, + activity: AgentActivityDetail | None = None, + waiting: bool | None = None, + ) -> None: + if ( + self._activity_owner is None + and scope is self.current_scope + and not scope.settled + ): + await self._acquire_activity(scope) + await self._working_reporter.update( + current=partial(self.owns_activity, scope), + activity=activity, + waiting=waiting, + ) + + async def _clear_scope_activity(self) -> None: + scope = self.current_scope + if scope is not None: + await self._working_reporter.clear_detail( + current=partial(self.owns_activity, scope) + ) def register_turn_task(self, scope: TurnScope, owner: TurnTask) -> None: """Bind completion to the same owner that delivers worker cancellation.""" diff --git a/src/band/runtime/sessionconfig.py b/src/band/runtime/sessionconfig.py index 9c9b73171..9e21f6f58 100644 --- a/src/band/runtime/sessionconfig.py +++ b/src/band/runtime/sessionconfig.py @@ -43,14 +43,13 @@ class SessionConfig(BaseSettings): # Queue traffic cannot postpone a check; active handlers remain serial. idle_resync_seconds: PositiveFiniteSeconds = 60.0 - # --- Working-state (boolean "is the agent reasoning") reporting --- - # Kill-switch: disable all working-state reporting instantly if needed. + # Indicator and activity detail share this switch, including approval waits. enable_working_state: bool = True # Keep-alive cadence: how often working:true is refreshed while a cycle runs. # Must be < TTL/2 so a single missed/slow ping still lands inside the TTL. working_keep_alive_seconds: float = 3.0 - # Per-POST deadline for each activity report (must be < cadence so a slow - # POST can't pile onto the next keep-alive tick). + # Total deadline for discovery, POST and a compatibility fallback. + # Must be < cadence so a slow report cannot pile onto the next tick. working_request_timeout_seconds: int = 2 # Optional upper bound on how long we keep asserting working:true for one # cycle. When exceeded we stop refreshing (platform TTL clears it); we never diff --git a/src/band/runtime/tools/agent.py b/src/band/runtime/tools/agent.py index 88d53634b..823291c83 100644 --- a/src/band/runtime/tools/agent.py +++ b/src/band/runtime/tools/agent.py @@ -74,6 +74,7 @@ validate_tool_arguments, ) from band.runtime.turn import records_turn_effects +from band.runtime.working_state import ActivityObserver if TYPE_CHECKING: from anthropic.types import ToolParam @@ -344,6 +345,7 @@ def __init__( self._agent_id = agent_id self._ctx: ExecutionContext | None = None self._scope: TurnScope | None = None + self._activity_observer: ActivityObserver | None = None self.turn = Turn() @property @@ -384,6 +386,7 @@ def from_context( ) tools._ctx = ctx tools._scope = ctx.current_scope if bind_scope else None + tools._activity_observer = ctx.activity_observer(tools._scope) tools.turn = Turn(posts_missing_reply=ctx.config.report_turn_failures_to_room) return tools diff --git a/src/band/runtime/working_state.py b/src/band/runtime/working_state.py index 4d842c7fe..318a9aac6 100644 --- a/src/band/runtime/working_state.py +++ b/src/band/runtime/working_state.py @@ -1,26 +1,8 @@ -"""Boolean working-state keep-alive reporter. - -Owns the per-room "is the agent working" signal for a single reasoning cycle: -sends ``working: true`` when a cycle starts, refreshes it on a keep-alive -cadence while the cycle is active, and sends an authoritative ``working: false`` -when the cycle ends. The platform applies a short TTL, so if refreshes stop for -any reason (crash, disconnect, loop-block) the indicator clears on its own — the -keep-alive is a *liveness* signal, not a correctness signal. - -The reporter is transport-agnostic: it is given an async ``report(working)`` -callback (wired to ``BandLink.report_activity`` by the runtime) and never raises -into the caller's reasoning loop. - -Contract: ``report`` MUST be time-bounded. The reporter awaits it (including the -authoritative false-send during teardown), so an unbounded callback could wedge -cycle teardown and cancellation propagation. ``BandLink.report_activity`` applies -a per-POST timeout to satisfy this. - -Cadence accuracy: the keep-alive sleeps ``keep_alive_seconds`` and *then* reports, -so the effective interval between successful refreshes is -``keep_alive_seconds + report-latency``. The platform-TTL headroom is preserved by -the ``cadence < TTL/2`` guard on ``SessionConfig`` (with the per-POST timeout kept -below the cadence), so even one missed/slow ping stays within the TTL window. +"""Serialized working-state and names-only activity for one provider turn. + +The boolean callback must be time bounded. It reads immutable detail under the +same lock as changes and heartbeats. Terminal false is shielded; platform TTL +backstops failed reports. A duration cap stops refreshes without cancelling work. """ from __future__ import annotations @@ -28,12 +10,52 @@ import asyncio import logging from collections.abc import Awaitable, Callable +from typing import Protocol + +from band_rest.types.agent_activity_detail import AgentActivityDetail logger = logging.getLogger(__name__) ReportFn = Callable[[bool], Awaitable[object]] +UNKNOWN_ACTIVITY_TOOL_NAME = "unknown" + + +class ActivityObserver(Protocol): + async def update(self, activity: AgentActivityDetail) -> None: ... + + async def set_manual_wait(self, waiting: bool) -> None: ... + + +def _observer(tools: object) -> ActivityObserver | None: + return getattr(tools, "_activity_observer", None) + + +async def report_tool_activity(tools: object, name: str) -> None: + if observer := _observer(tools): + await observer.update( + AgentActivityDetail( + kind="tool_call", name=name or UNKNOWN_ACTIVITY_TOOL_NAME + ) + ) + + +async def end_tool_activity(tools: object) -> None: + if observer := _observer(tools): + await observer.update(AgentActivityDetail(kind="reasoning")) + + +async def report_memory_activity(tools: object) -> None: + if observer := _observer(tools): + await observer.update(AgentActivityDetail(kind="memory")) + + +async def set_permission_wait(tools: object, waiting: bool) -> None: + if observer := _observer(tools): + await observer.set_manual_wait(waiting) + + def _retrieve_task_error(task: asyncio.Task[None]) -> None: if not task.cancelled(): task.exception() @@ -60,6 +82,9 @@ def __init__( self._enabled = enabled self._active = False + self._cycle: object | None = None + self._ordinary = AgentActivityDetail(kind="reasoning") + self._manual_wait = False self._started_at: float | None = None self._task: asyncio.Task[None] | None = None self._false_send: asyncio.Future[None] | None = None @@ -68,19 +93,66 @@ def __init__( # (boolean payload carries no sequence number to disambiguate). self._lock = asyncio.Lock() - async def start(self) -> None: - """Begin a reasoning cycle: send ``working: true`` and arm keep-alive.""" - if not self._enabled or self._active: - # Re-entrant start while active is a no-op: the signal is already - # alive and the keep-alive task is already running. + @property + def activity(self) -> AgentActivityDetail: + """The immutable effective detail, read by the bool callback under its lock.""" + return ( + AgentActivityDetail(kind="permission") + if self._manual_wait + else self._ordinary + ) + + async def start(self, *, owner: object | None = None) -> None: + if not self._enabled: return await self._finish_false_send() - self._active = True - self._started_at = asyncio.get_running_loop().time() - await self._safe_report(True) - self._task = asyncio.create_task( - self._keep_alive(), name="working-state-keepalive" - ) + async with self._lock: + if self._active: + return + self._active = True + cycle = owner if owner is not None else object() + self._cycle = cycle + self._ordinary = AgentActivityDetail(kind="reasoning") + self._manual_wait = False + self._started_at = asyncio.get_running_loop().time() + await self._report_locked(True) + if self._cycle is cycle and self._active: + self._task = asyncio.create_task( + self._keep_alive(cycle), name="working-state-keepalive" + ) + + async def update( + self, + *, + current: Callable[[], bool], + activity: AgentActivityDetail | None = None, + waiting: bool | None = None, + ) -> None: + if not current() or not self._enabled: + return + async with self._lock: + if not current() or not self._active or self._cap_reached(): + return + previous = self.activity + if activity is not None: + self._ordinary = ( + AgentActivityDetail( + kind="tool_call", + name=activity.name or UNKNOWN_ACTIVITY_TOOL_NAME, + ) + if activity.kind == "tool_call" + else AgentActivityDetail(kind=activity.kind) + ) + if waiting is not None: + self._manual_wait = waiting + if self.activity != previous: + await self._report_locked(True) + + async def clear_detail(self, *, current: Callable[[], bool]) -> None: + async with self._lock: + if current(): + self._ordinary = AgentActivityDetail(kind="reasoning") + self._manual_wait = False def defer_false_report(self) -> None: """Start ``working: false`` and return without waiting for it. @@ -125,6 +197,9 @@ async def stop(self) -> None: def _arm_false_report(self) -> asyncio.Task[None] | None: self._active = False + self._cycle = None + self._ordinary = AgentActivityDetail(kind="reasoning") + self._manual_wait = False task, self._task = self._task, None if task is not None: task.cancel() @@ -151,7 +226,7 @@ async def _finish_false_send(self) -> None: if send.done(): self._false_send = None - async def _keep_alive(self) -> None: + async def _keep_alive(self, cycle: object) -> None: """Re-send ``working: true`` on cadence until stopped or capped.""" while True: await asyncio.sleep(self._keep_alive_seconds) @@ -167,7 +242,7 @@ async def _keep_alive(self) -> None: self._max_working_state_seconds, ) return - await self._safe_report(True) + await self._safe_report(True, cycle=cycle) def _cap_reached(self) -> bool: if self._max_working_state_seconds is None or self._started_at is None: @@ -175,14 +250,20 @@ def _cap_reached(self) -> bool: elapsed = asyncio.get_running_loop().time() - self._started_at return elapsed >= self._max_working_state_seconds - async def _safe_report(self, working: bool) -> None: - """Report under the serialization lock; never raise.""" + async def _safe_report(self, working: bool, *, cycle: object | None = None) -> None: async with self._lock: - try: - await self._report(working) - except asyncio.CancelledError: - raise - except Exception: - logger.exception( - "working-state report failed (working=%s); swallowing", working - ) + if working and ( + not self._active or self._cycle is not cycle or self._cap_reached() + ): + return + await self._report_locked(working) + + async def _report_locked(self, working: bool) -> None: + try: + await self._report(working) + except asyncio.CancelledError: + raise + except Exception: + logger.exception( + "working-state report failed (working=%s); swallowing", working + ) diff --git a/tests/platform/activity/conftest.py b/tests/platform/activity/conftest.py new file mode 100644 index 000000000..1deada2d2 --- /dev/null +++ b/tests/platform/activity/conftest.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from collections.abc import AsyncIterator + +import pytest +from pytest_httpx import HTTPXMock + +import band.platform.message_lifecycle as lifecycle +from band.platform.activity import ActivitySupport +from band.platform.link import BandLink +from tests.platform.activity.kit import VERSION_URL, ActivityPeer +from tests.runtime.helpers import rest_client_over + + +@pytest.fixture +def support(monkeypatch: pytest.MonkeyPatch) -> ActivitySupport: + support = ActivitySupport() + monkeypatch.setattr(lifecycle, "activity_support", support) + return support + + +@pytest.fixture +async def peer(httpx_mock: HTTPXMock, support: ActivitySupport) -> ActivityPeer: + peer = ActivityPeer() + httpx_mock.add_callback( + peer.discover, url=VERSION_URL, is_optional=True, is_reusable=True + ) + return peer + + +@pytest.fixture +async def link(peer: ActivityPeer) -> AsyncIterator[BandLink]: + async with rest_client_over(peer.respond) as rest: + link = BandLink( + agent_id="agent", api_key="test-key", rest_url="https://platform.test" + ) + link.rest = rest + yield link diff --git a/tests/platform/activity/kit.py b/tests/platform/activity/kit.py new file mode 100644 index 000000000..d282ae3a5 --- /dev/null +++ b/tests/platform/activity/kit.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import asyncio +import json +from collections import deque +from typing import Any + +import httpx + +VERSION_URL = "https://platform.test/api/version" + + +class ActivityPeer: + def __init__(self) -> None: + self.version: object = {"agent_version": "1.25.0"} + self.version_status = 200 + self.version_content: bytes | None = None + self.version_requests: list[httpx.Request] = [] + self.reports: list[dict[str, Any]] = [] + self.responses: deque[httpx.Response] = deque() + self.post_entered = asyncio.Event() + self.post_release = asyncio.Event() + self.hold_post_number: int | None = None + self.discovering = asyncio.Event() + self.discovery_release: asyncio.Event | None = None + + async def discover(self, request: httpx.Request) -> httpx.Response: + self.version_requests.append(request) + self.discovering.set() + if self.discovery_release is not None: + await self.discovery_release.wait() + if self.version_content is not None: + return httpx.Response(self.version_status, content=self.version_content) + return httpx.Response(self.version_status, json=self.version) + + async def respond(self, request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + self.reports.append(body) + if len(self.reports) == self.hold_post_number: + self.post_entered.set() + await self.post_release.wait() + if self.responses: + return self.responses.popleft() + return httpx.Response(200, json={"data": {"working": body["working"]}}) diff --git a/tests/platform/activity/test_coordination.py b/tests/platform/activity/test_coordination.py new file mode 100644 index 000000000..716f52a2a --- /dev/null +++ b/tests/platform/activity/test_coordination.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import asyncio + +import pytest +from band_rest import AgentActivityDetail + +from band.platform.activity import ActivitySupport +from band.platform.link import BandLink +from tests.platform.activity.kit import VERSION_URL, ActivityPeer + + +class TestActivityCoordination: + async def test_concurrent_links_share_discovery_and_waiter_cancellation_is_independent( + self, link: BandLink, peer: ActivityPeer + ) -> None: + peer.discovery_release = asyncio.Event() + detail = AgentActivityDetail(kind="reasoning") + owner = asyncio.create_task( + link.report_activity("first", True, activity=detail) + ) + await peer.discovering.wait() + other = BandLink( + agent_id="second", api_key="test", rest_url="https://platform.test" + ) + other.rest = link.rest + waiter = asyncio.create_task( + other.report_activity("second", True, activity=detail) + ) + await asyncio.sleep(0) + waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await waiter + assert not owner.done() + peer.discovery_release.set() + assert await owner + assert await other.report_activity("second", True, activity=detail) + assert len(peer.version_requests) == 1 + assert ( + peer.reports == [{"working": True, "activity": {"kind": "reasoning"}}] * 2 + ) + + async def test_discovery_owner_cancellation_unblocks_future_callers_as_boolean( + self, link: BandLink, peer: ActivityPeer + ) -> None: + peer.discovery_release = asyncio.Event() + detail = AgentActivityDetail(kind="reasoning") + owner = asyncio.create_task( + link.report_activity("first", True, activity=detail) + ) + await peer.discovering.wait() + owner.cancel() + with pytest.raises(asyncio.CancelledError): + await owner + assert await link.report_activity("next", True, activity=detail) + assert peer.reports == [{"working": True}] + assert len(peer.version_requests) == 1 + + def test_completed_negotiation_survives_later_event_loops( + self, support: ActivitySupport, peer: ActivityPeer + ) -> None: + assert asyncio.run(support.allows(VERSION_URL.removesuffix("/api/version"), 1)) + assert asyncio.run(support.allows("https://unqueried.test", 1)) + assert len(peer.version_requests) == 1 + + @pytest.mark.parametrize("failure", ["server", "malformed"]) + async def test_discovery_failure_is_cached_without_harming_boolean_reports( + self, link: BandLink, peer: ActivityPeer, failure: str + ) -> None: + if failure == "server": + peer.version_status = 500 + else: + peer.version_content = b"not-json" + for room in ("first", "second"): + assert await link.report_activity( + room, True, activity=AgentActivityDetail(kind="memory") + ) + assert len(peer.version_requests) == 1 + assert peer.reports == [{"working": True}] * 2 diff --git a/tests/platform/activity/test_deadlines.py b/tests/platform/activity/test_deadlines.py new file mode 100644 index 000000000..420d66cfc --- /dev/null +++ b/tests/platform/activity/test_deadlines.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import asyncio + +import httpx +import pytest +from band_rest import AgentActivityDetail + +from band.platform.link import BandLink +from tests.platform.activity.kit import ActivityPeer + + +@pytest.mark.looptime +class TestActivityDeadline: + async def test_stalled_discovery_reserves_half_the_budget_for_boolean_post( + self, link: BandLink, peer: ActivityPeer + ) -> None: + peer.discovery_release = asyncio.Event() + started = asyncio.get_running_loop().time() + assert await link.report_activity( + "room", True, activity=AgentActivityDetail(kind="reasoning") + ) + elapsed = asyncio.get_running_loop().time() - started + assert elapsed <= 1.01 + assert peer.reports == [{"working": True}] + assert len(peer.version_requests) == 1 + + async def test_retry_cannot_extend_total_deadline_and_downgrade_survives_timeout( + self, link: BandLink, peer: ActivityPeer + ) -> None: + peer.responses.append(httpx.Response(422)) + peer.hold_post_number = 2 + detail = AgentActivityDetail(kind="tool_call", name="Bash") + started = asyncio.get_running_loop().time() + assert not await link.report_activity("room", True, activity=detail) + elapsed = asyncio.get_running_loop().time() - started + assert elapsed <= 2.01 + assert peer.reports == [ + {"working": True, "activity": {"kind": "tool_call", "name": "Bash"}}, + {"working": True}, + ] + peer.hold_post_number = None + assert await link.report_activity("next", True, activity=detail) + assert peer.reports[-1] == {"working": True} diff --git a/tests/platform/activity/test_fallback.py b/tests/platform/activity/test_fallback.py new file mode 100644 index 000000000..f48b2efb3 --- /dev/null +++ b/tests/platform/activity/test_fallback.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from unittest.mock import Mock + +import httpx +import pytest +from band_rest import AgentActivityDetail + +from band.platform.link import BandLink +from tests.platform.activity.kit import ActivityPeer + + +class TestActivityFallback: + @pytest.mark.parametrize( + "body", [b'{"errors":{"activity":["unknown"]}}', b"not json", b""] + ) + async def test_422_retries_without_detail_and_latches_across_links( + self, link: BandLink, peer: ActivityPeer, body: bytes + ) -> None: + peer.responses.append(httpx.Response(422, content=body)) + detail = AgentActivityDetail(kind="tool_call", name="Read") + assert await link.report_activity("first", True, activity=detail) + other = BandLink( + agent_id="other-agent", api_key="test-key", rest_url="https://other.test" + ) + other.rest = link.rest + assert await other.report_activity("second", True, activity=detail) + assert peer.reports == [ + {"working": True, "activity": {"kind": "tool_call", "name": "Read"}}, + {"working": True}, + {"working": True}, + ] + assert len(peer.version_requests) == 1 + + @pytest.mark.parametrize("status", [400, 403, 500]) + async def test_other_errors_do_not_retry_or_disable( + self, link: BandLink, peer: ActivityPeer, status: int + ) -> None: + peer.responses.append(httpx.Response(status, json={"error": "failure"})) + detail = AgentActivityDetail(kind="reasoning") + assert not await link.report_activity("room", True, activity=detail) + assert await link.report_activity("room", True, activity=detail) + assert ( + peer.reports == [{"working": True, "activity": {"kind": "reasoning"}}] * 2 + ) + + async def test_fallback_echoed_false_keeps_stop_notification( + self, link: BandLink, peer: ActivityPeer + ) -> None: + peer.responses.extend( + [ + httpx.Response(422), + httpx.Response(200, json={"data": {"working": False}}), + ] + ) + stopped = Mock() + assert await link.report_activity( + "room", + True, + activity=AgentActivityDetail(kind="reasoning"), + on_stopped=stopped, + ) + stopped.assert_called_once_with() + assert peer.reports[-1] == {"working": True} diff --git a/tests/platform/activity/test_versions.py b/tests/platform/activity/test_versions.py new file mode 100644 index 000000000..a3e8e695b --- /dev/null +++ b/tests/platform/activity/test_versions.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import pytest +from band_rest import AgentActivityDetail + +from band.platform.link import BandLink +from tests.platform.activity.kit import ActivityPeer + + +class TestActivityVersion: + @pytest.mark.parametrize( + ("version", "supported"), + [ + ({"agent_version": "1.24.99"}, False), + ({"agent_version": "1.25.0"}, True), + ({"agent_version": "1.26.10"}, True), + ({"agent_version": "2.0.0"}, True), + ({"platform_version": "999.0.0"}, False), + ({"agent_version": "1.25.0-beta"}, False), + ({"agent_version": "broken"}, False), + ({"agent_version": "1.25"}, False), + ({"agent_version": None}, False), + ([], False), + ], + ) + async def test_agent_version_controls_real_request_detail( + self, link: BandLink, peer: ActivityPeer, version: object, supported: bool + ) -> None: + peer.version = version + detail = AgentActivityDetail(kind="tool_call", name="Bash") + assert await link.report_activity("room", True, activity=detail) + assert await link.report_activity("other", True, activity=detail) + assert len(peer.version_requests) == 1 + expected = {"working": True} + if supported: + expected["activity"] = {"kind": "tool_call", "name": "Bash"} + assert peer.reports == [expected, expected] + assert "authorization" not in peer.version_requests[0].headers + + @pytest.mark.parametrize("kind", ["reasoning", "memory", "permission"]) + async def test_non_tool_detail_omits_name_and_false_omits_detail( + self, link: BandLink, peer: ActivityPeer, kind: str + ) -> None: + detail = AgentActivityDetail(kind=kind) + assert await link.report_activity("room", True, activity=detail) + assert await link.report_activity("room", False, activity=detail) + assert peer.reports == [ + {"working": True, "activity": {"kind": kind}}, + {"working": False}, + ] + + async def test_boolean_callers_do_not_discover( + self, link: BandLink, peer: ActivityPeer + ) -> None: + assert await link.report_activity("room", True) + assert await link.report_activity("room", False) + assert not peer.version_requests + assert peer.reports == [{"working": True}, {"working": False}] diff --git a/tests/runtime/activityownership/conftest.py b/tests/runtime/activityownership/conftest.py index 16bcf6bfc..7fa9fec6e 100644 --- a/tests/runtime/activityownership/conftest.py +++ b/tests/runtime/activityownership/conftest.py @@ -4,7 +4,10 @@ from unittest.mock import AsyncMock import pytest +from pytest_httpx import HTTPXMock +import band.platform.message_lifecycle as lifecycle +from band.platform.activity import ActivitySupport from band.platform.event import PlatformEvent from band.platform.link import BandLink from band.runtime.execution import ExecutionContext @@ -14,25 +17,33 @@ from tests.runtime.helpers import ( AGENT_ID, ROOM_ID, - LifecyclePlatform, rest_client_over, ) @pytest.fixture -async def activity_platform() -> AsyncIterator[LifecyclePlatform]: +async def activity_platform( + monkeypatch: pytest.MonkeyPatch, httpx_mock: HTTPXMock +) -> AsyncIterator[ActivityPlatform]: + monkeypatch.setattr(lifecycle, "activity_support", ActivitySupport()) + httpx_mock.add_response( + url="https://app.band.ai/api/version", + json={"agent_version": "1.25.0"}, + is_optional=True, + ) yield ActivityPlatform() @pytest.fixture async def execution( - activity_platform: LifecyclePlatform, + activity_platform: ActivityPlatform, ) -> AsyncIterator[Callable[..., ExecutionContext]]: contexts: list[ExecutionContext] = [] - async with rest_client_over(activity_platform.answer) as rest: + async with rest_client_over(activity_platform.respond) as rest: def create( handler: Callable[[ExecutionContext, PlatformEvent], Awaitable[None]], + hub_room_id: str | None = None, **settings: object, ) -> ExecutionContext: link = BandLink(agent_id=AGENT_ID, api_key="test-key") @@ -45,6 +56,7 @@ def create( link, handler, agent_id=AGENT_ID, + hub_room_id=hub_room_id, on_platform_stop=runtime._on_platform_stop, config=SessionConfig(enable_context_hydration=False, **settings), ) @@ -53,4 +65,5 @@ def create( yield create for ctx in contexts: + activity_platform.activity_release.set() await ctx.stop() diff --git a/tests/runtime/activityownership/kit.py b/tests/runtime/activityownership/kit.py index d43a5b921..bfec560b1 100644 --- a/tests/runtime/activityownership/kit.py +++ b/tests/runtime/activityownership/kit.py @@ -1,13 +1,34 @@ from __future__ import annotations +import asyncio +import json +from typing import Any + import httpx from tests.runtime.helpers import ALICE, LifecyclePlatform, page class ActivityPlatform(LifecyclePlatform): + def __init__(self) -> None: + super().__init__() + self.activity_bodies: list[dict[str, Any]] = [] + self.hold_next_activity = False + self.activity_entered = asyncio.Event() + self.activity_release = asyncio.Event() + + async def respond(self, request: httpx.Request) -> httpx.Response: + response = self.answer(request) + if self.hold_next_activity and request.url.path.endswith("/activity"): + self.hold_next_activity = False + self.activity_entered.set() + await self.activity_release.wait() + return response + def answer(self, request: httpx.Request) -> httpx.Response: tail = request.url.path.rsplit("/", 1)[-1] + if tail == "activity": + self.activity_bodies.append(json.loads(request.content)) if tail == "participants": return page([{**ALICE, "role": "member", "status": "active"}]) if request.method == "POST" and tail == "messages": diff --git a/tests/runtime/activityownership/test_detail.py b/tests/runtime/activityownership/test_detail.py new file mode 100644 index 000000000..6383776de --- /dev/null +++ b/tests/runtime/activityownership/test_detail.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Callable + +import pytest +from pytest_httpx import HTTPXMock + +from band.core.turn_lifecycle import TurnTask, bind_turn_task +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from band.runtime.working_state import ( + end_tool_activity, + report_memory_activity, + report_tool_activity, + set_permission_wait, +) +from tests.conftest import make_message_event +from tests.runtime.activityownership.kit import ActivityPlatform +from tests.runtime.helpers import ROOM_ID + + +class TestOwnedDetail: + async def test_real_tools_observer_reports_canonical_wire_detail( + self, execution: Callable[..., ExecutionContext], activity_platform: ActivityPlatform + ) -> None: + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + await report_tool_activity(tools, "Bash") + await set_permission_wait(tools, True) + await end_tool_activity(tools) + await report_memory_activity(tools) + await set_permission_wait(tools, False) + await end_tool_activity(tools) + + ctx = execution(handler) + await ctx._process_event(make_message_event()) + assert activity_platform.activity_bodies == [ + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": True, "activity": {"kind": "tool_call", "name": "Bash"}}, + {"working": True, "activity": {"kind": "permission"}}, + {"working": True, "activity": {"kind": "memory"}}, + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": False}, + ] + + @pytest.mark.looptime + async def test_settled_owner_remains_authoritative_and_transient_tools_are_quiet( + self, execution: Callable[..., ExecutionContext], activity_platform: ActivityPlatform + ) -> None: + release = asyncio.Event() + captured: list[AgentTools] = [] + workers: list[TurnTask] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + if captured: + await report_tool_activity(tools, "unrelated") + return + captured.append(tools) + workers.append(bind_turn_task(tools, asyncio.create_task(release.wait()))) + await set_permission_wait(tools, True) + + ctx = execution(handler) + await ctx._process_event(make_message_event(msg_id="original")) + await ctx._process_event(make_message_event(msg_id="busy")) + await asyncio.sleep(6.1) + assert ( + activity_platform.activity_bodies[-2:] + == [{"working": True, "activity": {"kind": "permission"}}] * 2 + ) + await report_tool_activity(captured[0], "Bash") + await set_permission_wait(captured[0], False) + assert activity_platform.activity_bodies[-1] == { + "working": True, + "activity": {"kind": "tool_call", "name": "Bash"}, + } + release.set() + await workers[0].task + await asyncio.sleep(0) + await ctx._join_activity_completion() + assert activity_platform.activity_bodies[-1] == {"working": False} + before = list(activity_platform.activity_bodies) + await end_tool_activity(captured[0]) + assert activity_platform.activity_bodies == before + + async def test_queued_observation_is_rechecked_after_stop_and_replacement( + self, execution: Callable[..., ExecutionContext], activity_platform: ActivityPlatform + ) -> None: + captured: list[AgentTools] = [] + release = asyncio.Event() + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + captured.append(AgentTools.from_context(ctx)) + await release.wait() + + ctx = execution(handler) + turn = asyncio.create_task(ctx._process_event(make_message_event(msg_id="old"))) + while not captured: + await asyncio.sleep(0) + activity_platform.hold_next_activity = True + change = asyncio.create_task(report_tool_activity(captured[0], "Bash")) + await activity_platform.activity_entered.wait() + stale = asyncio.create_task(report_memory_activity(captured[0])) + ctx.interrupt() + activity_platform.activity_release.set() + await asyncio.gather(turn, change, stale, return_exceptions=True) + release.set() + await ctx._process_event(make_message_event(msg_id="new")) + assert activity_platform.activity_bodies[-3:] == [ + {"working": False}, + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": False}, + ] + assert not any( + body.get("activity", {}).get("kind") == "memory" + for body in activity_platform.activity_bodies + ) + + @pytest.mark.parametrize("excluded", ["disabled", "hub"]) + async def test_excluded_reporting_ignores_observations_without_discovery( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + httpx_mock: HTTPXMock, + excluded: str, + ) -> None: + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + await report_tool_activity(tools, "Bash") + await set_permission_wait(tools, True) + await report_memory_activity(tools) + await end_tool_activity(tools) + + settings = ( + {"enable_working_state": False} + if excluded == "disabled" + else {"hub_room_id": ROOM_ID} + ) + ctx = execution(handler, **settings) + await ctx._process_event(make_message_event()) + assert not activity_platform.activity_bodies + assert not httpx_mock.get_requests() diff --git a/tests/runtime/conftest.py b/tests/runtime/conftest.py index 46f8a06ed..0a26403e8 100644 --- a/tests/runtime/conftest.py +++ b/tests/runtime/conftest.py @@ -32,7 +32,9 @@ ListAgentChatParticipantsResponse, ) +import band.platform.message_lifecycle as lifecycle from band.core.types import PlatformConnection, PlatformMessage +from band.platform.activity import ActivitySupport from band.runtime.presence import RoomPresence from tests.conftest import make_participant_mock from tests.runtime.helpers import AGENT_ID @@ -259,3 +261,15 @@ def link(mock_rest_client: MagicMock) -> MagicMock: link.get_stale_processing_messages = AsyncMock(return_value=[]) link.report_activity = AsyncMock(return_value=True) return link + + +@pytest.fixture(autouse=True) +def runtime_activity_support(monkeypatch: pytest.MonkeyPatch) -> None: + """Keep deployment discovery independent of runtime scheduling regressions.""" + support = ActivitySupport() + + async def discover(rest_url: str, budget: float) -> bool: + return True + + monkeypatch.setattr(support, "_discover", discover) + monkeypatch.setattr(lifecycle, "activity_support", support) diff --git a/tests/runtime/workingstate/test_detail.py b/tests/runtime/workingstate/test_detail.py new file mode 100644 index 000000000..4d776d1fb --- /dev/null +++ b/tests/runtime/workingstate/test_detail.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest +from band_rest import AgentActivityDetail + +from band.runtime.working_state import WorkingStateReporter + + +@pytest.mark.looptime +class TestActivityDetail: + async def test_changes_repeat_current_detail_and_keep_boolean_callback( + self, + ) -> None: + reports: list[tuple[bool, dict[str, Any] | None]] = [] + + async def report(working: bool) -> None: + reports.append( + ( + working, + reporter.activity.model_dump(exclude_unset=True) + if working + else None, + ) + ) + + reporter = WorkingStateReporter(report) + try: + await reporter.start() + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail( + kind="tool_call", name="Bash", arguments="private" + ), + ) + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail(kind="tool_call", name="Bash"), + ) + await asyncio.sleep(3.1) + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail( + kind="reasoning", name="private", output="private" + ), + ) + finally: + await reporter.stop() + assert reports == [ + (True, {"kind": "reasoning"}), + (True, {"kind": "tool_call", "name": "Bash"}), + (True, {"kind": "tool_call", "name": "Bash"}), + (True, {"kind": "reasoning"}), + (False, None), + ] + + async def test_manual_wait_overrides_latest_progress_without_restoring_stale_state( + self, + ) -> None: + reports: list[str] = [] + + async def report(working: bool) -> None: + if working: + reports.append(reporter.activity.kind) + + reporter = WorkingStateReporter(report) + try: + await reporter.start() + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail(kind="tool_call", name="Read"), + ) + await reporter.update(current=lambda: True, waiting=True) + await reporter.update( + current=lambda: True, activity=AgentActivityDetail(kind="reasoning") + ) + await reporter.update(current=lambda: True, waiting=True) + await reporter.update( + current=lambda: True, activity=AgentActivityDetail(kind="memory") + ) + await reporter.update(current=lambda: True, waiting=False) + finally: + await reporter.stop() + assert reports == ["reasoning", "tool_call", "permission", "memory"] + + async def test_cap_prevents_detail_from_reviving_the_indicator(self) -> None: + reports: list[bool] = [] + + async def report(working: bool) -> None: + reports.append(working) + + reporter = WorkingStateReporter(report, max_working_state_seconds=2) + try: + await reporter.start() + await asyncio.sleep(2.1) + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail(kind="tool_call", name="Bash"), + ) + await asyncio.sleep(3) + assert reports == [True] + finally: + await reporter.stop() + assert reports == [True, False] From d76f57131f1981579a6f9c7e74217112d3d9a5e6 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 18 Oct 2026 11:11:58 +0300 Subject: [PATCH 07/47] style: Format detail ownership fixture parameters --- tests/runtime/activityownership/test_detail.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/runtime/activityownership/test_detail.py b/tests/runtime/activityownership/test_detail.py index 6383776de..83e7bdf77 100644 --- a/tests/runtime/activityownership/test_detail.py +++ b/tests/runtime/activityownership/test_detail.py @@ -23,7 +23,9 @@ class TestOwnedDetail: async def test_real_tools_observer_reports_canonical_wire_detail( - self, execution: Callable[..., ExecutionContext], activity_platform: ActivityPlatform + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, ) -> None: async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: tools = AgentTools.from_context(ctx) @@ -47,7 +49,9 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: @pytest.mark.looptime async def test_settled_owner_remains_authoritative_and_transient_tools_are_quiet( - self, execution: Callable[..., ExecutionContext], activity_platform: ActivityPlatform + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, ) -> None: release = asyncio.Event() captured: list[AgentTools] = [] @@ -86,7 +90,9 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: assert activity_platform.activity_bodies == before async def test_queued_observation_is_rechecked_after_stop_and_replacement( - self, execution: Callable[..., ExecutionContext], activity_platform: ActivityPlatform + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, ) -> None: captured: list[AgentTools] = [] release = asyncio.Event() From 184263862293e389d75afcf734471af58a92f8d2 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 18 Oct 2026 11:41:21 +0300 Subject: [PATCH 08/47] test: Avoid reserved peer fixtures in activity wire tests --- tests/platform/activity/conftest.py | 14 ++++---- tests/platform/activity/test_coordination.py | 37 ++++++++++---------- tests/platform/activity/test_deadlines.py | 20 +++++------ tests/platform/activity/test_fallback.py | 23 ++++++------ tests/platform/activity/test_versions.py | 24 +++++++------ 5 files changed, 64 insertions(+), 54 deletions(-) diff --git a/tests/platform/activity/conftest.py b/tests/platform/activity/conftest.py index 1deada2d2..7734dfc78 100644 --- a/tests/platform/activity/conftest.py +++ b/tests/platform/activity/conftest.py @@ -20,17 +20,19 @@ def support(monkeypatch: pytest.MonkeyPatch) -> ActivitySupport: @pytest.fixture -async def peer(httpx_mock: HTTPXMock, support: ActivitySupport) -> ActivityPeer: - peer = ActivityPeer() +async def activity_peer( + httpx_mock: HTTPXMock, support: ActivitySupport +) -> ActivityPeer: + activity_peer = ActivityPeer() httpx_mock.add_callback( - peer.discover, url=VERSION_URL, is_optional=True, is_reusable=True + activity_peer.discover, url=VERSION_URL, is_optional=True, is_reusable=True ) - return peer + return activity_peer @pytest.fixture -async def link(peer: ActivityPeer) -> AsyncIterator[BandLink]: - async with rest_client_over(peer.respond) as rest: +async def link(activity_peer: ActivityPeer) -> AsyncIterator[BandLink]: + async with rest_client_over(activity_peer.respond) as rest: link = BandLink( agent_id="agent", api_key="test-key", rest_url="https://platform.test" ) diff --git a/tests/platform/activity/test_coordination.py b/tests/platform/activity/test_coordination.py index 716f52a2a..060bfdd74 100644 --- a/tests/platform/activity/test_coordination.py +++ b/tests/platform/activity/test_coordination.py @@ -12,14 +12,14 @@ class TestActivityCoordination: async def test_concurrent_links_share_discovery_and_waiter_cancellation_is_independent( - self, link: BandLink, peer: ActivityPeer + self, link: BandLink, activity_peer: ActivityPeer ) -> None: - peer.discovery_release = asyncio.Event() + activity_peer.discovery_release = asyncio.Event() detail = AgentActivityDetail(kind="reasoning") owner = asyncio.create_task( link.report_activity("first", True, activity=detail) ) - await peer.discovering.wait() + await activity_peer.discovering.wait() other = BandLink( agent_id="second", api_key="test", rest_url="https://platform.test" ) @@ -32,48 +32,49 @@ async def test_concurrent_links_share_discovery_and_waiter_cancellation_is_indep with pytest.raises(asyncio.CancelledError): await waiter assert not owner.done() - peer.discovery_release.set() + activity_peer.discovery_release.set() assert await owner assert await other.report_activity("second", True, activity=detail) - assert len(peer.version_requests) == 1 + assert len(activity_peer.version_requests) == 1 assert ( - peer.reports == [{"working": True, "activity": {"kind": "reasoning"}}] * 2 + activity_peer.reports + == [{"working": True, "activity": {"kind": "reasoning"}}] * 2 ) async def test_discovery_owner_cancellation_unblocks_future_callers_as_boolean( - self, link: BandLink, peer: ActivityPeer + self, link: BandLink, activity_peer: ActivityPeer ) -> None: - peer.discovery_release = asyncio.Event() + activity_peer.discovery_release = asyncio.Event() detail = AgentActivityDetail(kind="reasoning") owner = asyncio.create_task( link.report_activity("first", True, activity=detail) ) - await peer.discovering.wait() + await activity_peer.discovering.wait() owner.cancel() with pytest.raises(asyncio.CancelledError): await owner assert await link.report_activity("next", True, activity=detail) - assert peer.reports == [{"working": True}] - assert len(peer.version_requests) == 1 + assert activity_peer.reports == [{"working": True}] + assert len(activity_peer.version_requests) == 1 def test_completed_negotiation_survives_later_event_loops( - self, support: ActivitySupport, peer: ActivityPeer + self, support: ActivitySupport, activity_peer: ActivityPeer ) -> None: assert asyncio.run(support.allows(VERSION_URL.removesuffix("/api/version"), 1)) assert asyncio.run(support.allows("https://unqueried.test", 1)) - assert len(peer.version_requests) == 1 + assert len(activity_peer.version_requests) == 1 @pytest.mark.parametrize("failure", ["server", "malformed"]) async def test_discovery_failure_is_cached_without_harming_boolean_reports( - self, link: BandLink, peer: ActivityPeer, failure: str + self, link: BandLink, activity_peer: ActivityPeer, failure: str ) -> None: if failure == "server": - peer.version_status = 500 + activity_peer.version_status = 500 else: - peer.version_content = b"not-json" + activity_peer.version_content = b"not-json" for room in ("first", "second"): assert await link.report_activity( room, True, activity=AgentActivityDetail(kind="memory") ) - assert len(peer.version_requests) == 1 - assert peer.reports == [{"working": True}] * 2 + assert len(activity_peer.version_requests) == 1 + assert activity_peer.reports == [{"working": True}] * 2 diff --git a/tests/platform/activity/test_deadlines.py b/tests/platform/activity/test_deadlines.py index 420d66cfc..aec390434 100644 --- a/tests/platform/activity/test_deadlines.py +++ b/tests/platform/activity/test_deadlines.py @@ -13,32 +13,32 @@ @pytest.mark.looptime class TestActivityDeadline: async def test_stalled_discovery_reserves_half_the_budget_for_boolean_post( - self, link: BandLink, peer: ActivityPeer + self, link: BandLink, activity_peer: ActivityPeer ) -> None: - peer.discovery_release = asyncio.Event() + activity_peer.discovery_release = asyncio.Event() started = asyncio.get_running_loop().time() assert await link.report_activity( "room", True, activity=AgentActivityDetail(kind="reasoning") ) elapsed = asyncio.get_running_loop().time() - started assert elapsed <= 1.01 - assert peer.reports == [{"working": True}] - assert len(peer.version_requests) == 1 + assert activity_peer.reports == [{"working": True}] + assert len(activity_peer.version_requests) == 1 async def test_retry_cannot_extend_total_deadline_and_downgrade_survives_timeout( - self, link: BandLink, peer: ActivityPeer + self, link: BandLink, activity_peer: ActivityPeer ) -> None: - peer.responses.append(httpx.Response(422)) - peer.hold_post_number = 2 + activity_peer.responses.append(httpx.Response(422)) + activity_peer.hold_post_number = 2 detail = AgentActivityDetail(kind="tool_call", name="Bash") started = asyncio.get_running_loop().time() assert not await link.report_activity("room", True, activity=detail) elapsed = asyncio.get_running_loop().time() - started assert elapsed <= 2.01 - assert peer.reports == [ + assert activity_peer.reports == [ {"working": True, "activity": {"kind": "tool_call", "name": "Bash"}}, {"working": True}, ] - peer.hold_post_number = None + activity_peer.hold_post_number = None assert await link.report_activity("next", True, activity=detail) - assert peer.reports[-1] == {"working": True} + assert activity_peer.reports[-1] == {"working": True} diff --git a/tests/platform/activity/test_fallback.py b/tests/platform/activity/test_fallback.py index f48b2efb3..c88eac256 100644 --- a/tests/platform/activity/test_fallback.py +++ b/tests/platform/activity/test_fallback.py @@ -15,9 +15,9 @@ class TestActivityFallback: "body", [b'{"errors":{"activity":["unknown"]}}', b"not json", b""] ) async def test_422_retries_without_detail_and_latches_across_links( - self, link: BandLink, peer: ActivityPeer, body: bytes + self, link: BandLink, activity_peer: ActivityPeer, body: bytes ) -> None: - peer.responses.append(httpx.Response(422, content=body)) + activity_peer.responses.append(httpx.Response(422, content=body)) detail = AgentActivityDetail(kind="tool_call", name="Read") assert await link.report_activity("first", True, activity=detail) other = BandLink( @@ -25,29 +25,32 @@ async def test_422_retries_without_detail_and_latches_across_links( ) other.rest = link.rest assert await other.report_activity("second", True, activity=detail) - assert peer.reports == [ + assert activity_peer.reports == [ {"working": True, "activity": {"kind": "tool_call", "name": "Read"}}, {"working": True}, {"working": True}, ] - assert len(peer.version_requests) == 1 + assert len(activity_peer.version_requests) == 1 @pytest.mark.parametrize("status", [400, 403, 500]) async def test_other_errors_do_not_retry_or_disable( - self, link: BandLink, peer: ActivityPeer, status: int + self, link: BandLink, activity_peer: ActivityPeer, status: int ) -> None: - peer.responses.append(httpx.Response(status, json={"error": "failure"})) + activity_peer.responses.append( + httpx.Response(status, json={"error": "failure"}) + ) detail = AgentActivityDetail(kind="reasoning") assert not await link.report_activity("room", True, activity=detail) assert await link.report_activity("room", True, activity=detail) assert ( - peer.reports == [{"working": True, "activity": {"kind": "reasoning"}}] * 2 + activity_peer.reports + == [{"working": True, "activity": {"kind": "reasoning"}}] * 2 ) async def test_fallback_echoed_false_keeps_stop_notification( - self, link: BandLink, peer: ActivityPeer + self, link: BandLink, activity_peer: ActivityPeer ) -> None: - peer.responses.extend( + activity_peer.responses.extend( [ httpx.Response(422), httpx.Response(200, json={"data": {"working": False}}), @@ -61,4 +64,4 @@ async def test_fallback_echoed_false_keeps_stop_notification( on_stopped=stopped, ) stopped.assert_called_once_with() - assert peer.reports[-1] == {"working": True} + assert activity_peer.reports[-1] == {"working": True} diff --git a/tests/platform/activity/test_versions.py b/tests/platform/activity/test_versions.py index a3e8e695b..1eb74f30e 100644 --- a/tests/platform/activity/test_versions.py +++ b/tests/platform/activity/test_versions.py @@ -24,35 +24,39 @@ class TestActivityVersion: ], ) async def test_agent_version_controls_real_request_detail( - self, link: BandLink, peer: ActivityPeer, version: object, supported: bool + self, + link: BandLink, + activity_peer: ActivityPeer, + version: object, + supported: bool, ) -> None: - peer.version = version + activity_peer.version = version detail = AgentActivityDetail(kind="tool_call", name="Bash") assert await link.report_activity("room", True, activity=detail) assert await link.report_activity("other", True, activity=detail) - assert len(peer.version_requests) == 1 + assert len(activity_peer.version_requests) == 1 expected = {"working": True} if supported: expected["activity"] = {"kind": "tool_call", "name": "Bash"} - assert peer.reports == [expected, expected] - assert "authorization" not in peer.version_requests[0].headers + assert activity_peer.reports == [expected, expected] + assert "authorization" not in activity_peer.version_requests[0].headers @pytest.mark.parametrize("kind", ["reasoning", "memory", "permission"]) async def test_non_tool_detail_omits_name_and_false_omits_detail( - self, link: BandLink, peer: ActivityPeer, kind: str + self, link: BandLink, activity_peer: ActivityPeer, kind: str ) -> None: detail = AgentActivityDetail(kind=kind) assert await link.report_activity("room", True, activity=detail) assert await link.report_activity("room", False, activity=detail) - assert peer.reports == [ + assert activity_peer.reports == [ {"working": True, "activity": {"kind": kind}}, {"working": False}, ] async def test_boolean_callers_do_not_discover( - self, link: BandLink, peer: ActivityPeer + self, link: BandLink, activity_peer: ActivityPeer ) -> None: assert await link.report_activity("room", True) assert await link.report_activity("room", False) - assert not peer.version_requests - assert peer.reports == [{"working": True}, {"working": False}] + assert not activity_peer.version_requests + assert activity_peer.reports == [{"working": True}, {"working": False}] From 06d8124481475460782840b8854285cf7c0a2d92 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 18 Oct 2026 14:28:32 +0300 Subject: [PATCH 09/47] test: Preserve backlog ordering across valid recovery retries --- tests/runtime/test_execution.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/runtime/test_execution.py b/tests/runtime/test_execution.py index 8fdb098d5..b72e049f8 100644 --- a/tests/runtime/test_execution.py +++ b/tests/runtime/test_execution.py @@ -6,7 +6,7 @@ import logging from datetime import UTC, datetime, timedelta from typing import Any -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, call import pytest from band_sdk_core import AgentFailure, ClaimRegistry, RetryTracker @@ -1645,9 +1645,14 @@ async def test_refused_claims_never_spend_the_retry_budget( mock_handler.assert_awaited_once() assert not ctx._retry_tracker.is_permanently_failed(msg.id) + @pytest.mark.parametrize("initial_recovery_delay", [0.0, 10.0]) async def test_startup_backlog_claim_failure_does_not_process_newer_ws_event( - self, mock_link_with_next, mock_handler - ): + self, + mock_link_with_next, + mock_handler, + monkeypatch: pytest.MonkeyPatch, + initial_recovery_delay: float, + ) -> None: """Startup sync should not switch to WebSocket after an unclaimable backlog message.""" backlog_msg = PlatformMessage( @@ -1673,6 +1678,11 @@ async def test_startup_backlog_claim_failure_does_not_process_newer_ws_event( ), ) + monkeypatch.setattr( + "band.runtime.resync.random.uniform", + lambda lower, upper: initial_recovery_delay, + ) + await ctx.on_event( make_message_event(room_id="room-123", msg_id="msg-newer-ws") ) @@ -1681,8 +1691,10 @@ async def test_startup_backlog_claim_failure_does_not_process_newer_ws_event( lambda: mock_link_with_next.mark_processing.await_count >= 1 ) - mock_link_with_next.mark_processing.assert_awaited_once_with( - "room-123", "msg-older-claim-fails" + assert ( + mock_link_with_next.mark_processing.await_args_list + == [call("room-123", "msg-older-claim-fails")] + * mock_link_with_next.mark_processing.await_count ) mock_handler.assert_not_called() assert ctx._sync_complete is False From 2d697035fd7fec9f46339ddbede6e1622ec115f8 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 18 Oct 2026 10:35:27 +0300 Subject: [PATCH 10/47] feat: Map provider and Band tool progress to room activity --- src/band/adapters/agno.py | 36 +- src/band/adapters/anthropic.py | 106 +- src/band/adapters/claude_sdk.py | 60 +- src/band/adapters/codex.py | 181 +-- src/band/adapters/copilot_sdk.py | 74 +- src/band/adapters/crewai_flow.py | 22 +- src/band/adapters/gemini.py | 91 +- src/band/adapters/google_adk.py | 17 +- src/band/adapters/langgraph.py | 4 + src/band/adapters/letta.py | 55 +- src/band/adapters/nooa/tools.py | 60 +- src/band/adapters/omp_acp.py | 1 + src/band/adapters/opencode/adapter.py | 27 +- src/band/adapters/opencode/approvals.py | 11 + src/band/adapters/pydantic_ai.py | 3 + src/band/adapters/strands.py | 4 + src/band/integrations/acp/client_adapter.py | 24 +- src/band/integrations/acp/collecting.py | 59 +- src/band/integrations/acp/room_emitter.py | 14 + src/band/integrations/acp/sessions.py | 7 + .../integrations/claude_sdk/dedup_tools.py | 6 + src/band/integrations/crewai/catalog.py | 8 +- src/band/integrations/crewai/tools.py | 5 +- src/band/integrations/mcp/engine.py | 4 +- .../integrations/parlant/bandtools/chat.py | 6 +- src/band/integrations/parlant/customtools.py | 14 +- src/band/integrations/pydantic_ai/tools.py | 4 +- src/band/integrations/slack/adapter.py | 1 + src/band/runtime/tools/agent.py | 8 +- src/band/runtime/turn.py | 41 +- src/band/runtime/working_state.py | 28 +- src/band/testing/activity.py | 18 + src/band/testing/fake_tools.py | 2 + tests/adapters/activity/kit.py | 24 + tests/adapters/activity/test_execution.py | 67 ++ tests/adapters/activity/test_model_event.py | 38 + tests/adapters/activity/test_native.py | 98 ++ tests/adapters/agno/adapter/__init__.py | 1 + tests/adapters/agno/adapter/kit.py | 26 + .../adapter/test_bandentrypointbinding.py | 93 ++ .../adapter/test_bandinstructioninjection.py | 66 ++ .../agno/adapter/test_emitexecution.py | 116 ++ .../agno/adapter/test_emitthoughts.py | 64 ++ ...est_failedrundoesnotcontaminatenextturn.py | 58 + .../agno/adapter/test_featurefilters.py | 49 + .../agno/adapter/test_hubcontactexposure.py | 57 + .../adapter/test_memorycollisionwarning.py | 40 + tests/adapters/agno/adapter/test_oncleanup.py | 24 + tests/adapters/agno/adapter/test_onstarted.py | 45 + .../test_perruntoolexposureendtoend.py | 51 + .../agno/adapter/test_persistandaccumulate.py | 74 ++ tests/adapters/agno/adapter/test_reply.py | 68 ++ .../agno/adapter/test_roomtoolresolution.py | 89 ++ .../agno/adapter/test_runfailurereporting.py | 125 ++ .../agno/adapter/test_sessionisolation.py | 75 ++ .../agno/adapter/test_usedbeforestarted.py | 19 + tests/adapters/agno/helpers.py | 38 +- tests/adapters/agno/rehydration/__init__.py | 1 + .../rehydration/test_multiturncarryover.py | 38 + .../test_rehydrationpipeline.py} | 132 --- .../rehydration/test_unansweredmessage.py | 95 ++ tests/adapters/agno/test_adapter.py | 1020 ----------------- tests/runtime/activityownership/kit.py | 18 +- .../activityownership/test_band_tools.py | 50 + .../activityownership/test_opencode.py | 6 +- 65 files changed, 2244 insertions(+), 1522 deletions(-) create mode 100644 src/band/testing/activity.py create mode 100644 tests/adapters/activity/kit.py create mode 100644 tests/adapters/activity/test_execution.py create mode 100644 tests/adapters/activity/test_model_event.py create mode 100644 tests/adapters/activity/test_native.py create mode 100644 tests/adapters/agno/adapter/__init__.py create mode 100644 tests/adapters/agno/adapter/kit.py create mode 100644 tests/adapters/agno/adapter/test_bandentrypointbinding.py create mode 100644 tests/adapters/agno/adapter/test_bandinstructioninjection.py create mode 100644 tests/adapters/agno/adapter/test_emitexecution.py create mode 100644 tests/adapters/agno/adapter/test_emitthoughts.py create mode 100644 tests/adapters/agno/adapter/test_failedrundoesnotcontaminatenextturn.py create mode 100644 tests/adapters/agno/adapter/test_featurefilters.py create mode 100644 tests/adapters/agno/adapter/test_hubcontactexposure.py create mode 100644 tests/adapters/agno/adapter/test_memorycollisionwarning.py create mode 100644 tests/adapters/agno/adapter/test_oncleanup.py create mode 100644 tests/adapters/agno/adapter/test_onstarted.py create mode 100644 tests/adapters/agno/adapter/test_perruntoolexposureendtoend.py create mode 100644 tests/adapters/agno/adapter/test_persistandaccumulate.py create mode 100644 tests/adapters/agno/adapter/test_reply.py create mode 100644 tests/adapters/agno/adapter/test_roomtoolresolution.py create mode 100644 tests/adapters/agno/adapter/test_runfailurereporting.py create mode 100644 tests/adapters/agno/adapter/test_sessionisolation.py create mode 100644 tests/adapters/agno/adapter/test_usedbeforestarted.py create mode 100644 tests/adapters/agno/rehydration/__init__.py create mode 100644 tests/adapters/agno/rehydration/test_multiturncarryover.py rename tests/adapters/agno/{test_rehydration.py => rehydration/test_rehydrationpipeline.py} (50%) create mode 100644 tests/adapters/agno/rehydration/test_unansweredmessage.py delete mode 100644 tests/adapters/agno/test_adapter.py create mode 100644 tests/runtime/activityownership/test_band_tools.py diff --git a/src/band/adapters/agno.py b/src/band/adapters/agno.py index 910881d55..4466b1ab0 100644 --- a/src/band/adapters/agno.py +++ b/src/band/adapters/agno.py @@ -37,6 +37,7 @@ is_image_passthrough_result, redact_tool_call_args, ) +from band.runtime.working_state import end_tool_activity, report_tool_activity try: from agno.models.message import Message @@ -451,10 +452,7 @@ async def _run_agent( ) -> RunOutput | None: """Run the Agno agent with the room's tools bound for this call. - When ``Emit.TOOL_CALLS`` is enabled the run is streamed so tool_call / - tool_result events are emitted *as each tool runs* (see - :meth:`_run_streamed`), matching the other adapters' live reporting. - Otherwise it runs non-streaming, exactly as before. + Consume provider progress as it happens, independently of chat narration. """ try: agent = self._agent @@ -469,17 +467,14 @@ async def _run_agent( session_id, ) with _bind_room_tools(tools): - if Emit.TOOL_CALLS in self.features.emit: - response = await self._run_streamed( - agent, - messages, - tools, - session_id=session_id, - room_id=room_id, - msg_id=msg_id, - ) - else: - response = await agent.arun(input=messages, session_id=session_id) + response = await self._run_streamed( + agent, + messages, + tools, + session_id=session_id, + room_id=room_id, + msg_id=msg_id, + ) # Agno swallows run exceptions into a normal-looking return value # (status=error); surface it so the shared except path below treats # the turn as failed rather than as a silent empty reply. @@ -532,9 +527,14 @@ async def _run_streamed( # None, which would read as a successful turn with no output. raise AgnoRunError(_error_summary(item.content)) else: - await self._emit_stream_event( - item, tools, room_id=room_id, msg_id=msg_id - ) + if isinstance(item, ToolCallStartedEvent) and item.tool is not None: + await report_tool_activity(tools, item.tool.tool_name or "") + elif isinstance(item, ToolCallCompletedEvent): + await end_tool_activity(tools) + if Emit.TOOL_CALLS in self.features.emit: + await self._emit_stream_event( + item, tools, room_id=room_id, msg_id=msg_id + ) return final def _persist_turn(self, room_id: str, response: RunOutput) -> None: diff --git a/src/band/adapters/anthropic.py b/src/band/adapters/anthropic.py index 41c524026..982ceab26 100644 --- a/src/band/adapters/anthropic.py +++ b/src/band/adapters/anthropic.py @@ -40,6 +40,7 @@ redact_tool_call_args, ) from band.runtime.tools.schema import is_failed_tool_output +from band.runtime.working_state import tool_activity logger = logging.getLogger(__name__) @@ -478,61 +479,62 @@ async def _process_tool_calls( logger.debug("Executing tool: %s with input: %s", tool_name, tool_input) - # Report tool call if enabled (JSON format with tool_call_id for linking) - # Best-effort: event reporting must never crash tool execution - if Emit.TOOL_CALLS in self.features.emit: + async with tool_activity(tools, tool_name): + # Report tool call if enabled (JSON format with tool_call_id for linking) + # Best-effort: event reporting must never crash tool execution + if Emit.TOOL_CALLS in self.features.emit: + try: + await tools.send_event( + content=json.dumps( + { + ToolEventKey.NAME: tool_name, + ToolEventKey.ARGS: redact_tool_call_args( + tool_name, tool_input + ), + ToolEventKey.TOOL_CALL_ID: tool_use_id, + } + ), + message_type="tool_call", + ) + except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn + logger.warning( + "Failed to send tool_call event: %s", + e, + ) + + # Execute tool (check custom tools first, then platform tools) try: - await tools.send_event( - content=json.dumps( - { - ToolEventKey.NAME: tool_name, - ToolEventKey.ARGS: redact_tool_call_args( - tool_name, tool_input - ), - ToolEventKey.TOOL_CALL_ID: tool_use_id, - } - ), - message_type="tool_call", - ) + custom_tool = find_custom_tool(self._custom_tools, tool_name) + if custom_tool: + result = await execute_custom_tool( + custom_tool, tool_input, turn=tools.turn + ) + succeeded = not is_failed_tool_output(result) + else: + outcome = await tools.execute_tool_call_structured( + tool_name, tool_input + ) + result = outcome.value + succeeded = outcome.ok + if succeeded and tool_name == BandTool.NO_REPLY: + declined = True + if not succeeded: + failed = True + if is_image_passthrough_result(tool_name, result): + content = _image_tool_result_content(result) + result_str = image_block_placeholder(len(content)) + else: + content = result_str = ( + json.dumps(result, default=str) + if not isinstance(result, str) + else result + ) + is_error = False except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn - logger.warning( - "Failed to send tool_call event: %s", - e, - ) - - # Execute tool (check custom tools first, then platform tools) - try: - custom_tool = find_custom_tool(self._custom_tools, tool_name) - if custom_tool: - result = await execute_custom_tool( - custom_tool, tool_input, turn=tools.turn - ) - succeeded = not is_failed_tool_output(result) - else: - outcome = await tools.execute_tool_call_structured( - tool_name, tool_input - ) - result = outcome.value - succeeded = outcome.ok - if succeeded and tool_name == BandTool.NO_REPLY: - declined = True - if not succeeded: failed = True - if is_image_passthrough_result(tool_name, result): - content = _image_tool_result_content(result) - result_str = image_block_placeholder(len(content)) - else: - content = result_str = ( - json.dumps(result, default=str) - if not isinstance(result, str) - else result - ) - is_error = False - except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn - failed = True - content = result_str = f"Error: {e}" - is_error = True - logger.error("Tool %s failed: %s", tool_name, e) + content = result_str = f"Error: {e}" + is_error = True + logger.error("Tool %s failed: %s", tool_name, e) # Report tool result if enabled (JSON format with tool_call_id for linking) # Best-effort: event reporting must never crash tool execution diff --git a/src/band/adapters/claude_sdk.py b/src/band/adapters/claude_sdk.py index 432898145..622971f72 100644 --- a/src/band/adapters/claude_sdk.py +++ b/src/band/adapters/claude_sdk.py @@ -23,6 +23,12 @@ from functools import partial from typing import TYPE_CHECKING, Any, ClassVar, Literal, NoReturn, Self, cast +from band.runtime.working_state import ( + end_tool_activity, + report_tool_activity, + set_permission_wait, +) + try: from claude_agent_sdk import ( # type: ignore[import-not-found] AssistantMessage, @@ -1381,6 +1387,7 @@ async def _on_tool_use( # namespaces our tools mcp__band__*; see _semantic_tool_name). tool_name = self._semantic_tool_name(block.name) pending_tool_names[block.id] = tool_name + await report_tool_activity(tools, tool_name) await self._narrate_tool_call(block, tool_name, room_id, tools) async def _narrate_tool_call( @@ -1530,6 +1537,7 @@ async def _on_tool_result( Shared by both envelopes a result can arrive in: user-type messages (the protocol shape) and assistant messages (accepted defensively). """ + await end_tool_activity(tools) result_tool_name = pending_tool_names.pop(block.tool_use_id, None) await self._narrate_tool_result(block, result_tool_name, room_id, tools) @@ -1832,36 +1840,42 @@ async def _resolve_manual_approval( # so the caller isn't left waiting for a timeout nobody will see. tools = self._room_tools.get(room_id) mention = [requester["id"]] if requester else None - if tools: - try: - await tools.send_notice( - APPROVAL_REQUESTED_TEMPLATE.format(summary=summary, token=token), - mentions=mention, - ) - except Exception: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn - logger.warning("Room %s: Failed to send approval notification", room_id) - # A reply that claimed it meanwhile owns the answer; otherwise - # decline without a notice, so the missing-reply guard stays. - if registry.withdraw(entry): - self._clear_pending_approval(room_id, entry) - return PermissionResultDeny( - message="Could not deliver approval prompt, tool use declined" + try: + if tools: + await set_permission_wait(tools, True) + try: + await tools.send_notice( + APPROVAL_REQUESTED_TEMPLATE.format( + summary=summary, token=token + ), + mentions=mention, ) + except Exception: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn + logger.warning( + "Room %s: Failed to send approval notification", room_id + ) + # A reply that claimed it meanwhile owns the answer; otherwise + # decline without a notice, so the missing-reply guard stays. + if registry.withdraw(entry): + return PermissionResultDeny( + message="Could not deliver approval prompt, tool use declined" + ) + + # The request has been posted (or there was nowhere to post it) -- + # either way, on_message must return now so Band's room loop can + # dispatch the reply that will resolve this wait. The turn is then + # judged at its real end (see _on_turn_complete). + if tools: + tools.turn.detach() + self._release_turn(room_id) - # The request has been posted (or there was nowhere to post it) -- - # either way, on_message must return now so Band's room loop can - # dispatch the reply that will resolve this wait. The turn is then - # judged at its real end (see _on_turn_complete). - if tools: - tools.turn.detach() - self._release_turn(room_id) - - try: decision = await registry.wait( entry, pending.future, timeout_s=self._approvals.wait_timeout_s ) finally: self._clear_pending_approval(room_id, entry) + if tools: + await set_permission_wait(tools, bool(registry)) match decision: case Timeout.TIMED_OUT: diff --git a/src/band/adapters/codex.py b/src/band/adapters/codex.py index b9115e676..da7e359d1 100644 --- a/src/band/adapters/codex.py +++ b/src/band/adapters/codex.py @@ -121,6 +121,13 @@ is_image_passthrough_result, redact_tool_call_args, ) +from band.runtime.working_state import ( + end_tool_activity, + report_memory_activity, + report_tool_activity, + set_permission_wait, + tool_activity, +) from band.workspaces import ( RoomWorkspaces, WorkspaceResolver, @@ -1632,6 +1639,7 @@ async def _process_turn_events( # --- Phase 2: Context compaction events --- if event.method == "context/compacted": + await report_memory_activity(tools) if ( self.config.emit_turn_lifecycle_events and Emit.TASK_EVENTS in self.features.emit @@ -1675,6 +1683,12 @@ async def _process_turn_events( if event.method == "item/started": item = params.get("item") + if isinstance(item, dict) and item.get("type") in _TOOL_ITEM_TYPES: + await report_tool_activity( + tools, self._extract_tool_item(item["type"], item).name + ) + elif isinstance(item, dict): + await end_tool_activity(tools) if ( isinstance(item, dict) and item.get("type") == CodexItemType.AGENT_MESSAGE @@ -1716,6 +1730,8 @@ async def _process_turn_events( if event.method == "item/completed": item = params.get("item") + if isinstance(item, dict) and item.get("type") in _TOOL_ITEM_TYPES: + await end_tool_activity(tools) if isinstance(item, dict): if item.get("type") == CodexItemType.AGENT_MESSAGE: message = result.native_messages.setdefault( @@ -2485,98 +2501,103 @@ async def _handle_server_request( arguments = {} call_id = str(params.get("callId") or "") - # Don't emit reporting for codex-local slash commands — they already - # surface their outcome in the room themselves. - should_report = ( - Emit.TOOL_CALLS in self.features.emit - and tool_name not in _SILENT_REPORTING_TOOLS - ) - - if should_report: - await tools.send_event( - content=json.dumps( - { - ToolEventKey.NAME: tool_name, - ToolEventKey.ARGS: redact_tool_call_args( - tool_name, arguments - ), - ToolEventKey.TOOL_CALL_ID: call_id, - } - ), - message_type="tool_call", + async with tool_activity(tools, tool_name): + # Don't emit reporting for codex-local slash commands — they already + # surface their outcome in the room themselves. + should_report = ( + Emit.TOOL_CALLS in self.features.emit + and tool_name not in _SILENT_REPORTING_TOOLS ) - try: - custom_tool = find_custom_tool(self._custom_tools, tool_name) - if custom_tool: - result = await execute_custom_tool( - custom_tool, arguments, turn=tools.turn - ) - success = True - else: - # Structured: a base tool (e.g. band_send_message) can fail without - # raising (bad args, API error) via ok=False; the plain variant would - # report that failure to Codex as success. - outcome = await tools.execute_tool_call_structured( - tool_name, arguments - ) - result = outcome.value - success = outcome.ok - if success and is_image_passthrough_result(tool_name, result): - content_items = _image_content_items(result) - text_result = image_block_placeholder(len(content_items)) - else: - text_result = ( - result - if isinstance(result, str) - else json.dumps(result, default=str) - ) - content_items = [_text_item(text_result)] - await client.respond( - event.id, _tool_response(content_items, success=success) - ) if should_report: await tools.send_event( content=json.dumps( { ToolEventKey.NAME: tool_name, - ToolEventKey.OUTPUT: text_result, + ToolEventKey.ARGS: redact_tool_call_args( + tool_name, arguments + ), ToolEventKey.TOOL_CALL_ID: call_id, } ), - message_type="tool_result", + message_type="tool_call", ) - except ValidationError as exc: - errors = format_validation_error(exc) - error_text = f"Invalid arguments for {tool_name}: {errors}" - logger.error("Validation error for tool %s: %s", tool_name, exc) - await client.respond(event.id, _tool_result(error_text, success=False)) - if should_report: - await tools.send_event( - content=json.dumps( - { - ToolEventKey.NAME: tool_name, - ToolEventKey.OUTPUT: error_text, - ToolEventKey.TOOL_CALL_ID: call_id, - } - ), - message_type="tool_result", + + try: + custom_tool = find_custom_tool(self._custom_tools, tool_name) + if custom_tool: + result = await execute_custom_tool( + custom_tool, arguments, turn=tools.turn + ) + success = True + else: + # Structured: a base tool (e.g. band_send_message) can fail without + # raising (bad args, API error) via ok=False; the plain variant would + # report that failure to Codex as success. + outcome = await tools.execute_tool_call_structured( + tool_name, arguments + ) + result = outcome.value + success = outcome.ok + if success and is_image_passthrough_result(tool_name, result): + content_items = _image_content_items(result) + text_result = image_block_placeholder(len(content_items)) + else: + text_result = ( + result + if isinstance(result, str) + else json.dumps(result, default=str) + ) + content_items = [_text_item(text_result)] + await client.respond( + event.id, _tool_response(content_items, success=success) ) - except Exception as exc: - error_text = f"Error: {exc}" - logger.exception("Tool execution failed for %s", tool_name) - await client.respond(event.id, _tool_result(error_text, success=False)) - if should_report: - await tools.send_event( - content=json.dumps( - { - ToolEventKey.NAME: tool_name, - ToolEventKey.OUTPUT: error_text, - ToolEventKey.TOOL_CALL_ID: call_id, - } - ), - message_type="tool_result", + if should_report: + await tools.send_event( + content=json.dumps( + { + ToolEventKey.NAME: tool_name, + ToolEventKey.OUTPUT: text_result, + ToolEventKey.TOOL_CALL_ID: call_id, + } + ), + message_type="tool_result", + ) + except ValidationError as exc: + errors = format_validation_error(exc) + error_text = f"Invalid arguments for {tool_name}: {errors}" + logger.error("Validation error for tool %s: %s", tool_name, exc) + await client.respond( + event.id, _tool_result(error_text, success=False) ) + if should_report: + await tools.send_event( + content=json.dumps( + { + ToolEventKey.NAME: tool_name, + ToolEventKey.OUTPUT: error_text, + ToolEventKey.TOOL_CALL_ID: call_id, + } + ), + message_type="tool_result", + ) + except Exception as exc: + error_text = f"Error: {exc}" + logger.exception("Tool execution failed for %s", tool_name) + await client.respond( + event.id, _tool_result(error_text, success=False) + ) + if should_report: + await tools.send_event( + content=json.dumps( + { + ToolEventKey.NAME: tool_name, + ToolEventKey.OUTPUT: error_text, + ToolEventKey.TOOL_CALL_ID: call_id, + } + ), + message_type="tool_result", + ) return @@ -3245,6 +3266,7 @@ async def _resolve_manual_approval( ) raise + await set_permission_wait(tools, True) self._release_turn(room_id, tools) decision = await registry.wait( entry, pending.future, timeout_s=self.config.approval_wait_timeout_s @@ -3264,6 +3286,7 @@ async def _resolve_manual_approval( return timeout_decision finally: self._clear_pending_approval(room_id, entry) + await set_permission_wait(tools, bool(registry)) async def _forward_raw_task_event( self, diff --git a/src/band/adapters/copilot_sdk.py b/src/band/adapters/copilot_sdk.py index 5b48037d5..b20df1631 100644 --- a/src/band/adapters/copilot_sdk.py +++ b/src/band/adapters/copilot_sdk.py @@ -57,6 +57,7 @@ is_image_passthrough_result, redact_tool_call_args, ) +from band.runtime.working_state import tool_activity try: from copilot import ( @@ -744,46 +745,47 @@ async def _execute_bridged_tool( error="room inactive", ) - should_report = Emit.TOOL_CALLS in self.features.emit - if should_report: - await self._report_tool_call(room_tools, invocation, arguments) + async with tool_activity(room_tools, tool_name): + should_report = Emit.TOOL_CALLS in self.features.emit + if should_report: + await self._report_tool_call(room_tools, invocation, arguments) - try: - custom_tool = find_custom_tool(self._custom_tools, tool_name) - if custom_tool: - result = await execute_custom_tool( - custom_tool, arguments, turn=room_tools.turn + try: + custom_tool = find_custom_tool(self._custom_tools, tool_name) + if custom_tool: + result = await execute_custom_tool( + custom_tool, arguments, turn=room_tools.turn + ) + else: + # Structured variant: a base tool (e.g. band_send_message) can fail + # without raising (bad args, API error) — that surfaces as ok=False, + # not an exception, and must reach the model as a failed call. + outcome = await room_tools.execute_tool_call_structured( + tool_name, arguments + ) + if not outcome.ok: + return await self._fail_tool_call( + room_tools, + invocation, + outcome.error_message or str(outcome.value), + report=should_report, + ) + result = outcome.value + except ValidationError as exc: + logger.error("Validation error for tool %s: %s", tool_name, exc) + error_text = ( + f"Invalid arguments for {tool_name}: {format_validation_error(exc)}" ) - else: - # Structured variant: a base tool (e.g. band_send_message) can fail - # without raising (bad args, API error) — that surfaces as ok=False, - # not an exception, and must reach the model as a failed call. - outcome = await room_tools.execute_tool_call_structured( - tool_name, arguments + return await self._fail_tool_call( + room_tools, invocation, error_text, report=should_report + ) + except Exception as exc: + logger.exception("Tool execution failed for %s", tool_name) + return await self._fail_tool_call( + room_tools, invocation, f"Error: {exc}", report=should_report ) - if not outcome.ok: - return await self._fail_tool_call( - room_tools, - invocation, - outcome.error_message or str(outcome.value), - report=should_report, - ) - result = outcome.value - except ValidationError as exc: - logger.error("Validation error for tool %s: %s", tool_name, exc) - error_text = ( - f"Invalid arguments for {tool_name}: {format_validation_error(exc)}" - ) - return await self._fail_tool_call( - room_tools, invocation, error_text, report=should_report - ) - except Exception as exc: - logger.exception("Tool execution failed for %s", tool_name) - return await self._fail_tool_call( - room_tools, invocation, f"Error: {exc}", report=should_report - ) - binary_results: list[ToolBinaryResult] | None = None + binary_results: list[ToolBinaryResult] | None = None if is_image_passthrough_result(tool_name, result): # Same failure path as the tool-execution try/except above: a # malformed or future-extended content block must still route diff --git a/src/band/adapters/crewai_flow.py b/src/band/adapters/crewai_flow.py index b753b6414..b151e7bcb 100644 --- a/src/band/adapters/crewai_flow.py +++ b/src/band/adapters/crewai_flow.py @@ -75,6 +75,7 @@ execute_custom_tool, get_custom_tool_name, ) +from band.runtime.working_state import tool_activity logger = logging.getLogger(__name__) @@ -595,16 +596,17 @@ async def call_tool( if tool is None: raise ValueError(f"Unknown custom tool: {name}") input_data = dict(arguments or kwargs) - await self._report_call(name, input_data) - try: - result = await execute_custom_tool( - tool=tool, arguments=input_data, turn=self._tools.turn - ) - except Exception as exc: - await self._report_result(name, str(exc), is_error=True) - raise - await self._report_result(name, result) - return result + async with tool_activity(self._tools, name): + await self._report_call(name, input_data) + try: + result = await execute_custom_tool( + tool=tool, arguments=input_data, turn=self._tools.turn + ) + except Exception as exc: + await self._report_result(name, str(exc), is_error=True) + raise + await self._report_result(name, result) + return result async def _report_call(self, tool_name: str, input_data: dict[str, Any]) -> None: await self._reporter.report_call(self._tools, tool_name, input_data) diff --git a/src/band/adapters/gemini.py b/src/band/adapters/gemini.py index a38b733d2..a725af4f8 100644 --- a/src/band/adapters/gemini.py +++ b/src/band/adapters/gemini.py @@ -18,6 +18,8 @@ ) from typing_extensions import Unpack +from band.runtime.working_state import tool_activity + try: from google import genai # type: ignore[missing-module-attribute] from google.genai import types # type: ignore[missing-import] @@ -541,51 +543,54 @@ async def _process_function_calls( tool_input = dict(function_call.args or {}) tool_call_id = function_call.id or f"gemini_tool_call_{index}" - if Emit.TOOL_CALLS in self.features.emit: - try: - await tools.send_event( - content=json.dumps( - { - ToolEventKey.NAME: tool_name, - ToolEventKey.ARGS: redact_tool_call_args( - tool_name, tool_input - ), - ToolEventKey.TOOL_CALL_ID: tool_call_id, - } - ), - message_type="tool_call", - ) - except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn - logger.warning("Failed to send tool_call event: %s", e) + async with tool_activity(tools, tool_name): + if Emit.TOOL_CALLS in self.features.emit: + try: + await tools.send_event( + content=json.dumps( + { + ToolEventKey.NAME: tool_name, + ToolEventKey.ARGS: redact_tool_call_args( + tool_name, tool_input + ), + ToolEventKey.TOOL_CALL_ID: tool_call_id, + } + ), + message_type="tool_call", + ) + except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn + logger.warning("Failed to send tool_call event: %s", e) - response_parts: list[types.FunctionResponsePart] | None = None - try: - custom_tool = find_custom_tool(self._custom_tools, tool_name) - if custom_tool: - result = await execute_custom_tool( - custom_tool, tool_input, turn=tools.turn - ) - else: - result = await tools.execute_tool_call(tool_name, tool_input) - if is_image_passthrough_result(tool_name, result): - response_parts = _image_function_response_parts(result) - result_str = image_block_placeholder(len(response_parts)) - else: - result_str = ( - json.dumps(result, default=str) - if not isinstance(result, str) - else result + response_parts: list[types.FunctionResponsePart] | None = None + try: + custom_tool = find_custom_tool(self._custom_tools, tool_name) + if custom_tool: + result = await execute_custom_tool( + custom_tool, tool_input, turn=tools.turn + ) + else: + result = await tools.execute_tool_call(tool_name, tool_input) + if is_image_passthrough_result(tool_name, result): + response_parts = _image_function_response_parts(result) + result_str = image_block_placeholder(len(response_parts)) + else: + result_str = ( + json.dumps(result, default=str) + if not isinstance(result, str) + else result + ) + is_error = False + except ValidationError as exc: + errors = format_validation_error(exc) + result_str = f"Invalid arguments for {tool_name}: {errors}" + is_error = True + logger.warning( + "Validation error for tool %s: %s", tool_name, errors ) - is_error = False - except ValidationError as exc: - errors = format_validation_error(exc) - result_str = f"Invalid arguments for {tool_name}: {errors}" - is_error = True - logger.warning("Validation error for tool %s: %s", tool_name, errors) - except Exception as e: - result_str = f"Error: {e}" - is_error = True - logger.exception("Tool %s failed", tool_name) + except Exception as e: + result_str = f"Error: {e}" + is_error = True + logger.exception("Tool %s failed", tool_name) if Emit.TOOL_CALLS in self.features.emit: try: await tools.send_event( diff --git a/src/band/adapters/google_adk.py b/src/band/adapters/google_adk.py index 374c84f65..116fd9ba7 100644 --- a/src/band/adapters/google_adk.py +++ b/src/band/adapters/google_adk.py @@ -47,6 +47,7 @@ is_image_passthrough_result, redact_tool_call_args, ) +from band.runtime.working_state import end_tool_activity, report_tool_activity if TYPE_CHECKING: from google.adk.runners import InMemoryRunner @@ -660,12 +661,10 @@ async def on_message( event.error_code, event.error_message ) - # Report tool calls/results if enabled - if Emit.TOOL_CALLS in self.features.emit: - try: - await self._report_event(event, tools) - except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn - logger.warning("Failed to report event: %s", e) + try: + await self._report_event(event, tools) + except Exception as e: # noqa: BLE001 -- best-effort tool narration + logger.warning("Failed to report event: %s", e) if event.is_final_response(): # Extract text from the final response for history tracking @@ -788,6 +787,9 @@ async def _report_event(self, event: Any, tools: AgentToolsProtocol) -> None: function_calls = event.get_function_calls() if function_calls: for fc in function_calls: + await report_tool_activity(tools, getattr(fc, "name", "") or "") + if Emit.TOOL_CALLS not in self.features.emit: + continue try: tool_name = getattr(fc, "name", "unknown") try: @@ -812,6 +814,9 @@ async def _report_event(self, event: Any, tools: AgentToolsProtocol) -> None: function_responses = event.get_function_responses() if function_responses: for fr in function_responses: + await end_tool_activity(tools) + if Emit.TOOL_CALLS not in self.features.emit: + continue try: tool_name = getattr(fr, "name", "unknown") await tools.send_event( diff --git a/src/band/adapters/langgraph.py b/src/band/adapters/langgraph.py index 8bc60fc09..f4fe72c32 100644 --- a/src/band/adapters/langgraph.py +++ b/src/band/adapters/langgraph.py @@ -35,11 +35,13 @@ image_block_placeholder, redact_tool_call_args, ) +from band.runtime.working_state import end_tool_activity, report_tool_activity if TYPE_CHECKING: from langchain_core.language_models import BaseChatModel from langgraph.checkpoint.base import BaseCheckpointSaver + logger = logging.getLogger(__name__) @@ -437,6 +439,7 @@ async def _handle_stream_event( event_type = event.get("event") if event_type == "on_tool_start": + await report_tool_activity(tools, event.get("name") or "") if Emit.TOOL_CALLS not in self.features.emit: return @@ -459,6 +462,7 @@ async def _handle_stream_event( logger.warning("Failed to send tool_call event: %s", e) elif event_type in {"on_tool_end", "on_tool_error"}: + await end_tool_activity(tools) if Emit.TOOL_CALLS not in self.features.emit: return diff --git a/src/band/adapters/letta.py b/src/band/adapters/letta.py index 4fec51716..7e4342267 100644 --- a/src/band/adapters/letta.py +++ b/src/band/adapters/letta.py @@ -51,6 +51,7 @@ redact_tool_call_args, turn_effect, ) +from band.runtime.working_state import end_tool_activity, report_tool_activity __all__ = [ "LettaAdapter", @@ -477,10 +478,7 @@ async def _send_message( Returns the list of assistant text parts collected during the turn. """ messages = [{"role": "user", "content": content}] - # Per-turn token usage. Reachable only on the per_room path, where the - # non-streamed LettaResponse carries an aggregate .usage; the shared-mode - # Conversations stream exposes no such aggregate, so it stays empty (N-A). - # Emitted on every exit via the finally. + # Shared Conversations has no aggregate usage; per-room step streams do. turn_usage = TurnUsage() try: @@ -489,7 +487,7 @@ async def _send_message( # processing (including deliver_reply, below) runs unbounded so a # slow Band-side delivery is never mislabeled as a Letta timeout. response_messages, turn_usage = await asyncio.wait_for( - self._call_provider(agent_id, messages, room_ctx), + self._call_provider(agent_id, messages, room_ctx, tools), timeout=self.config.turn_timeout_s, ) room_ctx.pending_seed = [] @@ -532,6 +530,7 @@ async def _call_provider( agent_id: str, messages: list[dict[str, str]], room_ctx: RoomContext, + tools: AgentToolsProtocol, ) -> tuple[list[Any], TurnUsage]: """Round-trip to the Letta API -- no response processing or delivery.""" # Use Conversations API in shared mode, direct agent API in per_room mode @@ -540,14 +539,46 @@ async def _call_provider( conversation_id=room_ctx.conversation_id, messages=messages, ) - response_messages = [resp_msg async for resp_msg in conversation_stream] - return response_messages, TurnUsage() + stream = conversation_stream + per_room = False + else: + stream = await self._client.agents.messages.create( + agent_id=agent_id, + messages=messages, + streaming=True, + stream_tokens=False, + ) + per_room = True + response_messages: list[Any] = [] + usage = TurnUsage() + async with stream: + async for message in stream: + await self._observe_provider_message(message, tools) + if ( + per_room + and getattr(message, "message_type", None) == "usage_statistics" + ): + usage = TurnUsage.from_object( + message, + input="prompt_tokens", + output="completion_tokens", + cache_read="cached_input_tokens", + cache_write="cache_write_tokens", + ) + else: + response_messages.append(message) + return response_messages, usage - response = await self._client.agents.messages.create( - agent_id=agent_id, - messages=messages, - ) - return list(response.messages), self._usage_from_response(response) + @staticmethod + async def _observe_provider_message( + message: Any, tools: AgentToolsProtocol + ) -> None: + match getattr(message, "message_type", None): + case "tool_call_message": + for call in message.tool_calls or [message.tool_call]: + await report_tool_activity(tools, call.name) + case "tool_return_message": + await end_tool_activity(tools) async def _process_response_messages( self, diff --git a/src/band/adapters/nooa/tools.py b/src/band/adapters/nooa/tools.py index 40359c572..6e83b968b 100644 --- a/src/band/adapters/nooa/tools.py +++ b/src/band/adapters/nooa/tools.py @@ -46,6 +46,7 @@ redact_tool_call_args, serialize_tool_result, ) +from band.runtime.working_state import tool_activity logger = logging.getLogger(__name__) @@ -158,34 +159,37 @@ async def _call( self, tools: AgentToolsProtocol, name: str, arguments: dict[str, Any] ) -> Any: """Run one tool call and report it, inside the turn that owns it.""" - call_id = uuid4().hex - await self._report( - tools, - MessageType.TOOL_CALL, - { - ToolEventKey.NAME: name, - ToolEventKey.ARGS: redact_tool_call_args(name, arguments), - ToolEventKey.TOOL_CALL_ID: call_id, - }, - ) - try: - value = serialize_tool_result(await self._specs[name].run(tools, arguments)) - except asyncio.CancelledError: - message = f"{name} was cancelled because its turn ended first" - await self._report_result(tools, name, call_id, message, is_error=True) - raise - except Exception as error: - message = f"{name} failed: {error}" - logger.warning("Band tool %s failed: %s", name, error) - await self._report_result(tools, name, call_id, message, is_error=True) - raise BandToolError(message) from error - if (image_result := image_passthrough_result(name, value)) is not None: - images = images_from_result(image_result) - output = image_block_placeholder(len(images)) - await self._report_result(tools, name, call_id, output, is_error=False) - return images - await self._report_result(tools, name, call_id, value, is_error=False) - return value + async with tool_activity(tools, name): + call_id = uuid4().hex + await self._report( + tools, + MessageType.TOOL_CALL, + { + ToolEventKey.NAME: name, + ToolEventKey.ARGS: redact_tool_call_args(name, arguments), + ToolEventKey.TOOL_CALL_ID: call_id, + }, + ) + try: + value = serialize_tool_result( + await self._specs[name].run(tools, arguments) + ) + except asyncio.CancelledError: + message = f"{name} was cancelled because its turn ended first" + await self._report_result(tools, name, call_id, message, is_error=True) + raise + except Exception as error: + message = f"{name} failed: {error}" + logger.warning("Band tool %s failed: %s", name, error) + await self._report_result(tools, name, call_id, message, is_error=True) + raise BandToolError(message) from error + if (image_result := image_passthrough_result(name, value)) is not None: + images = images_from_result(image_result) + output = image_block_placeholder(len(images)) + await self._report_result(tools, name, call_id, output, is_error=False) + return images + await self._report_result(tools, name, call_id, value, is_error=False) + return value async def _report_result( self, diff --git a/src/band/adapters/omp_acp.py b/src/band/adapters/omp_acp.py index 47f5e099d..844c3f126 100644 --- a/src/band/adapters/omp_acp.py +++ b/src/band/adapters/omp_acp.py @@ -278,6 +278,7 @@ async def handler( options=options, room_id=room_id, session_id=session_id, + emitter=emitter, ) if option_id == OMP_APPROVE_OPTION_ID: return AcceptElicitationResponse( diff --git a/src/band/adapters/opencode/adapter.py b/src/band/adapters/opencode/adapter.py index a9f826d1d..d12aa0afc 100644 --- a/src/band/adapters/opencode/adapter.py +++ b/src/band/adapters/opencode/adapter.py @@ -70,6 +70,7 @@ from band.runtime.custom_tools import CustomToolDef, get_custom_tool_name from band.runtime.prompts import render_system_prompt from band.runtime.tools import CHAT_ID_FIELD_NAME, iter_tool_definitions +from band.runtime.working_state import end_tool_activity, report_tool_activity logger = logging.getLogger(__name__) @@ -104,6 +105,7 @@ class TurnState: assistant_message_ids: set[str] = field(default_factory=set) assistant_part_types: dict[str, str] = field(default_factory=dict) reported_tool_calls: set[str] = field(default_factory=set) + activity_tool_status: dict[str, str | None] = field(default_factory=dict) reported_tool_results: set[str] = field(default_factory=set) worker: TurnTask | None = None abort_started: bool = False @@ -895,7 +897,30 @@ async def _report_tool_part( self, room_state: RoomState, part: OpencodePart ) -> None: """Report the tool's call/result.""" - if part.state is None or Emit.TOOL_CALLS not in self.features.emit: + if part.state is None: + return + turn = room_state.turn + if turn is not None and part.id is not None: + call_id = part.call_id or part.id + previous = turn.activity_tool_status.get(call_id) + status = part.state.status + turn.activity_tool_status[call_id] = status + if ( + status in {OpencodeToolStatus.PENDING, OpencodeToolStatus.RUNNING} + and previous is None + ): + await report_tool_activity( + turn.tools, self._canonical_tool_name(part.tool or "") + ) + elif status in { + OpencodeToolStatus.COMPLETED, + OpencodeToolStatus.ERROR, + } and previous not in { + OpencodeToolStatus.COMPLETED, + OpencodeToolStatus.ERROR, + }: + await end_tool_activity(turn.tools) + if Emit.TOOL_CALLS not in self.features.emit: return state = part.state diff --git a/src/band/adapters/opencode/approvals.py b/src/band/adapters/opencode/approvals.py index 2404893f1..dcdf90c5e 100644 --- a/src/band/adapters/opencode/approvals.py +++ b/src/band/adapters/opencode/approvals.py @@ -32,6 +32,7 @@ ) from band.runtime.decisions import DecisionEntry, DecisionRegistry from band.runtime.formatters import format_tokens, strip_leading_mentions +from band.runtime.working_state import set_permission_wait logger = logging.getLogger(__name__) @@ -227,6 +228,10 @@ def _parked_on_human(self) -> bool: """ return self._permissions.has_unclaimed() or self._questions.has_unclaimed() + async def _report_manual_wait(self) -> None: + if tools := self._ports.tools(): + await set_permission_wait(tools, self.awaiting_human()) + async def wait_until_idle(self) -> None: """Block until no manual ask is awaiting a human reply.""" await self._idle.wait() @@ -329,6 +334,7 @@ async def _ask_room(self, entry: DecisionEntry[PendingPermission]) -> None: entry, self._config.approval_wait_timeout_s, self._expire_permission ) self._park_on_human() + await self._report_manual_wait() pattern_text = ", ".join(pending.patterns) if pending.patterns else "n/a" await self._notify_room( APPROVAL_REQUESTED_TEMPLATE.format( @@ -385,6 +391,7 @@ async def on_question_asked(self, request: OpencodeQuestionRequest) -> None: entry, self._config.question_wait_timeout_s, self._expire_question ) self._park_on_human() + await self._report_manual_wait() await self._notify_room( format_question_prompt(pending.questions, request_id), self._ports.turn_mentions(), @@ -662,6 +669,7 @@ async def _send_permission_reply( reply, ) self._forget(self._permissions, entry) + await self._report_manual_wait() return True async def _reply_permission( @@ -686,6 +694,7 @@ async def _send_question_reply( entry.token, ) self._forget(self._questions, entry) + await self._report_manual_wait() return True async def _reply_question( @@ -715,6 +724,7 @@ async def _send_question_reject( entry.token, ) self._forget(self._questions, entry) + await self._report_manual_wait() return True async def _reject_question(self, pending: PendingQuestion) -> bool: @@ -735,6 +745,7 @@ async def _fail_request( # abort_session kills the whole session, so no sibling ask can be # answered either -- drop them all, timers included. self.cancel() + await self._report_manual_wait() self._ports.fail_turn(message) await self._ports.abort_session() diff --git a/src/band/adapters/pydantic_ai.py b/src/band/adapters/pydantic_ai.py index 732a4dcdb..e6592f1e1 100644 --- a/src/band/adapters/pydantic_ai.py +++ b/src/band/adapters/pydantic_ai.py @@ -83,6 +83,7 @@ redact_tool_call_args, ) from band.runtime.tools.schema import is_failed_tool_output +from band.runtime.working_state import end_tool_activity, report_tool_activity logger = logging.getLogger(__name__) @@ -626,6 +627,7 @@ async def on_message( if event_count & (event_count - 1) == 0: _log_native_progress("stream_progress", events=event_count) if isinstance(event, FunctionToolCallEvent): + await report_tool_activity(tools, event.part.tool_name) _log_native_progress( "tool_started", events=event_count, @@ -650,6 +652,7 @@ async def on_message( except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn logger.warning("Failed to send tool_call event: %s", e) elif isinstance(event, FunctionToolResultEvent): + await end_tool_activity(tools) tool_succeeded = isinstance( event.part, ToolReturnPart ) and not is_failed_tool_output(event.part.content) diff --git a/src/band/adapters/strands.py b/src/band/adapters/strands.py index 809b6eba8..0a90a511a 100644 --- a/src/band/adapters/strands.py +++ b/src/band/adapters/strands.py @@ -10,6 +10,8 @@ from band_sdk_core import AgentFailure from pydantic import BaseModel +from band.runtime.working_state import end_tool_activity, report_tool_activity + try: from strands import Agent from strands.agent import AgentResult @@ -383,6 +385,7 @@ def register_hooks(self, registry: HookRegistry, **kwargs: Any) -> None: registry.add_callback(AfterToolCallEvent, self._on_after_tool) async def _on_before_tool(self, event: BeforeToolCallEvent) -> None: + await report_tool_activity(self._tools, event.tool_use["name"]) call: dict[ToolEventKey, object] = { ToolEventKey.NAME: event.tool_use["name"], ToolEventKey.ARGS: redact_tool_call_args( @@ -399,6 +402,7 @@ async def _on_before_tool(self, event: BeforeToolCallEvent) -> None: ) async def _on_after_tool(self, event: AfterToolCallEvent) -> None: + await end_tool_activity(self._tools) name = event.tool_use["name"] output = _result_text(event.result) succeeded = event.result.get("status") == "success" and not band_tool_errored( diff --git a/src/band/integrations/acp/client_adapter.py b/src/band/integrations/acp/client_adapter.py index 24e6dc98f..4cb730b89 100644 --- a/src/band/integrations/acp/client_adapter.py +++ b/src/band/integrations/acp/client_adapter.py @@ -12,7 +12,7 @@ Mapping, Sequence, ) -from contextlib import AsyncExitStack, asynccontextmanager +from contextlib import AsyncExitStack, asynccontextmanager, nullcontext from dataclasses import dataclass from enum import StrEnum from functools import partial @@ -869,6 +869,7 @@ async def _prompt_room( self._install_turn_handlers( runtime, emitter=emitter, + tools=tools, room_id=room_id, session_id=session_id, ) @@ -925,6 +926,8 @@ async def _prompt_room( except Exception as error: await self._fail_turn(room, generation, tools, error) raise + finally: + runtime.set_activity_tools(session_id, None) @staticmethod async def _fail_turn( @@ -1006,9 +1009,11 @@ def _install_turn_handlers( runtime: ACPRuntime, *, emitter: RoomTurnEmitter, + tools: AgentToolsProtocol, room_id: str, session_id: str, ) -> None: + runtime.set_activity_tools(session_id, tools) runtime.set_permission_handler( session_id, self._make_permission_handler(emitter, room_id), @@ -1050,6 +1055,7 @@ async def handler( options=options, room_id=room_id, session_id=session_id, + emitter=emitter, ) logger.info( @@ -1101,20 +1107,22 @@ async def _resolve_permission_option( options: object, room_id: str, session_id: str, + emitter: RoomTurnEmitter | None = None, ) -> str | None: """Return a validated permission choice, or cancel the request.""" if self._resolve_permission is None: return select_allow_option_id(options) offered = self._permission_options(options) - option_id = await self._resolve_permission( - ACPPermissionRequest( - room_id=room_id, - session_id=session_id, - tool_call=call, - options=offered, + async with emitter.permission_wait() if emitter is not None else nullcontext(): + option_id = await self._resolve_permission( + ACPPermissionRequest( + room_id=room_id, + session_id=session_id, + tool_call=call, + options=offered, + ) ) - ) if option_id is None: return None if not isinstance(option_id, str): diff --git a/src/band/integrations/acp/collecting.py b/src/band/integrations/acp/collecting.py index b576f7618..9e2165c78 100644 --- a/src/band/integrations/acp/collecting.py +++ b/src/band/integrations/acp/collecting.py @@ -9,6 +9,7 @@ from acp.interfaces import Client from acp.schema import ConfigOptionUpdate, DeclineElicitationResponse +from band.core.protocols import AgentToolsProtocol from band.integrations.acp.client_profiles import ACPClientProfile, NoopACPClientProfile from band.integrations.acp.permissions import ( ElicitationHandler, @@ -18,8 +19,18 @@ ) from band.integrations.acp.session_config import SessionConfigOption from band.integrations.acp.stream import ACPChunkStream, ChunkSink -from band.integrations.acp.types import CollectedChunk +from band.integrations.acp.types import ( + ACPToolCall, + ChunkType, + CollectedChunk, + ToolStatus, +) from band.integrations.acp.updates import ACPUpdateParser +from band.runtime.working_state import ( + UNKNOWN_ACTIVITY_TOOL_NAME, + end_tool_activity, + report_tool_activity, +) logger = logging.getLogger(__name__) @@ -44,6 +55,8 @@ def __init__( self._stream = ACPChunkStream() self._permission_handlers: dict[str, PermissionHandler] = {} self._elicitation_handlers: dict[str, ElicitationHandler] = {} + self._activity_tools: dict[str, AgentToolsProtocol] = {} + self._activity_calls: dict[str, dict[str, tuple[str, bool]]] = {} # Retain locks while a straggler can still hold one. self._session_locks: dict[str, asyncio.Lock] = {} # Each session's latest advertised config catalog; like the locks, @@ -76,8 +89,50 @@ async def session_update( async with self._session_lock(session_id): chunk = self._chunk_from_update(update) if chunk is not None: + await self._observe_activity(session_id, update, chunk) await self._stream.ingest(session_id, chunk) + def set_activity_tools( + self, session_id: str, tools: AgentToolsProtocol | None + ) -> None: + if tools is None: + self._activity_tools.pop(session_id, None) + self._activity_calls.pop(session_id, None) + else: + self._activity_tools[session_id] = tools + self._activity_calls[session_id] = {} + + async def _observe_activity( + self, session_id: str, update: object, chunk: CollectedChunk + ) -> None: + tools = self._activity_tools.get(session_id) + if tools is None or chunk.chunk_type not in { + ChunkType.TOOL_CALL, + ChunkType.TOOL_RESULT, + }: + return + call_id = str(getattr(update, "tool_call_id", "")) + calls = self._activity_calls[session_id] + previous = calls.get(call_id) + status = getattr(update, "status", None) + if status in {ToolStatus.COMPLETED, ToolStatus.FAILED}: + if previous is None or not previous[1]: + calls[call_id] = ( + previous[0] if previous else UNKNOWN_ACTIVITY_TOOL_NAME, + True, + ) + await end_tool_activity(tools) + return + if previous is not None and previous[1]: + return + name = previous[0] if previous else UNKNOWN_ACTIVITY_TOOL_NAME + call = chunk.tool + if isinstance(call, ACPToolCall) and call.mcp_server is not None: + name = call.name + if previous is None or previous[0] != name: + calls[call_id] = (name, False) + await report_tool_activity(tools, name) + async def request_permission( # type: ignore[override] # ACP Client uses specific types; we widen to object self, options: object, @@ -155,6 +210,7 @@ def config_options(self, session_id: str) -> tuple[SessionConfigOption, ...]: def forget_session(self, session_id: str) -> None: """Drop a closed session's catalog and any state its profile keeps.""" + self.set_activity_tools(session_id, None) self._config_options.pop(session_id, None) # A custom profile written before this hook has none. forget = getattr(self._profile, ACPClientProfile.forget_session.__name__, None) @@ -162,6 +218,7 @@ def forget_session(self, session_id: str) -> None: forget(session_id) def reset_session(self, session_id: str) -> None: + self.set_activity_tools(session_id, None) self._stream.reset_session(session_id) self._permission_handlers.pop(session_id, None) self._elicitation_handlers.pop(session_id, None) diff --git a/src/band/integrations/acp/room_emitter.py b/src/band/integrations/acp/room_emitter.py index 07b8d5525..91c3c7027 100644 --- a/src/band/integrations/acp/room_emitter.py +++ b/src/band/integrations/acp/room_emitter.py @@ -3,6 +3,8 @@ from __future__ import annotations import logging +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager from typing import Self from band.core.content import has_visible_content @@ -16,6 +18,7 @@ ToolStatus, ) from band.runtime.tools import TurnEffect, turn_effect +from band.runtime.working_state import set_permission_wait logger = logging.getLogger(__name__) @@ -77,6 +80,17 @@ def __init__( self._emit = frozenset(emit) if emit is not None else frozenset(Emit) self._pending_text: list[str] = [] self._staged_effects: list[TurnEffect] = [] + self._manual_waits = 0 + + @asynccontextmanager + async def permission_wait(self) -> AsyncIterator[None]: + self._manual_waits += 1 + try: + await set_permission_wait(self._tools, True) + yield + finally: + self._manual_waits -= 1 + await set_permission_wait(self._tools, self._manual_waits > 0) async def emit(self, chunk: CollectedChunk) -> None: self._stage_tool_outcome(chunk) diff --git a/src/band/integrations/acp/sessions.py b/src/band/integrations/acp/sessions.py index 7d92d470b..75e724135 100644 --- a/src/band/integrations/acp/sessions.py +++ b/src/band/integrations/acp/sessions.py @@ -16,6 +16,7 @@ SetSessionConfigOptionResponse, ) +from band.core.protocols import AgentToolsProtocol from band.integrations.acp.collecting import ACPCollectingClient from band.integrations.acp.permissions import ElicitationHandler, PermissionHandler from band.integrations.acp.session_config import ( @@ -235,6 +236,12 @@ def reset_session(self, session_id: str) -> None: if self._client is not None: self._client.reset_session(session_id) + def set_activity_tools( + self, session_id: str, tools: AgentToolsProtocol | None + ) -> None: + if self._client is not None: + self._client.set_activity_tools(session_id, tools) + def set_permission_handler( self, session_id: str, diff --git a/src/band/integrations/claude_sdk/dedup_tools.py b/src/band/integrations/claude_sdk/dedup_tools.py index 70f9716f4..2016cd31b 100644 --- a/src/band/integrations/claude_sdk/dedup_tools.py +++ b/src/band/integrations/claude_sdk/dedup_tools.py @@ -42,6 +42,8 @@ from collections import OrderedDict from typing import TYPE_CHECKING, Any +from band.runtime.working_state import ActivityObserver, _observer + if TYPE_CHECKING: from band.core.protocols import AgentToolsProtocol @@ -199,6 +201,10 @@ async def send_message( # --- transparent passthrough -------------------------------------- + @property + def _activity_observer(self) -> ActivityObserver | None: + return _observer(self._inner) + def __getattr__(self, name: str) -> Any: # ``__getattr__`` only runs for attributes not found via the # normal MRO. ``send_message`` is defined above, so it never diff --git a/src/band/integrations/crewai/catalog.py b/src/band/integrations/crewai/catalog.py index e33df22f1..8105d6ba4 100644 --- a/src/band/integrations/crewai/catalog.py +++ b/src/band/integrations/crewai/catalog.py @@ -29,6 +29,7 @@ serialize_tool_result, validate_tool_arguments, ) +from band.runtime.working_state import model_tool_activity # --- Rendering a tool result as the text CrewAI hands the model --- @@ -288,9 +289,10 @@ async def _send_event(call: Invocation, **kwargs: Any) -> Any: # bypassing args_schema, and _execute_tool turns the ValueError into the # error result the caller expects. args = validate_tool_arguments(BandTool.SEND_EVENT, SEND_EVENT_ARGS_SCHEMA, kwargs) - await call.tools.send_event( - args["content"], args["message_type"], metadata=args.get("metadata") - ) + async with model_tool_activity(call.tools, BandTool.SEND_EVENT): + await call.tools.send_event( + args["content"], args["message_type"], metadata=args.get("metadata") + ) return succeeded("Event sent") diff --git a/src/band/integrations/crewai/tools.py b/src/band/integrations/crewai/tools.py index d5610d62f..e15d00d49 100644 --- a/src/band/integrations/crewai/tools.py +++ b/src/band/integrations/crewai/tools.py @@ -26,6 +26,8 @@ from pydantic import BaseModel +from band.runtime.working_state import tool_activity + if TYPE_CHECKING: from crewai.tools import BaseTool @@ -95,7 +97,8 @@ def _execute_tool( async def _execute() -> str: try: - return await coro_factory(tools) + async with tool_activity(tools, tool_name): + return await coro_factory(tools) except Exception as e: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn error_msg = str(e) if tool_name == BandTool.SEND_MESSAGE and isinstance( diff --git a/src/band/integrations/mcp/engine.py b/src/band/integrations/mcp/engine.py index 4e7fe1320..0744bb1d6 100644 --- a/src/band/integrations/mcp/engine.py +++ b/src/band/integrations/mcp/engine.py @@ -64,6 +64,7 @@ ) from band.runtime.tools.inputs.chat import require_visible_content from band.runtime.tools.inputs.identifiers import ResourceId +from band.runtime.working_state import model_tool_activity logger = logging.getLogger(__name__) @@ -219,7 +220,8 @@ async def dispatch_tool( """ method = resolve_tool_method(tools, definition) try: - return await method(**arguments) + async with model_tool_activity(tools, definition.name): + return await method(**arguments) except (ValueError, BandToolError) as error: raise enrich_send_message_error(definition, tools, error) from error diff --git a/src/band/integrations/parlant/bandtools/chat.py b/src/band/integrations/parlant/bandtools/chat.py index 6886419d2..0c78221b2 100644 --- a/src/band/integrations/parlant/bandtools/chat.py +++ b/src/band/integrations/parlant/bandtools/chat.py @@ -16,6 +16,7 @@ ) from band.integrations.parlant.sessiontools import require_session_tools from band.runtime.tools import ParticipantAddStatus, serialize_tool_result +from band.runtime.working_state import model_tool_activity EVENT_MESSAGE_TYPES: tuple[str, ...] = get_args(EventMessageType) @@ -28,6 +29,8 @@ "the 'data'/'metadata' dict described above." ) +from band.runtime.tools.types import BandTool + @band_tool( "sending message", @@ -59,7 +62,8 @@ async def band_send_event( if message_type not in EVENT_MESSAGE_TYPES: return invalid_choice("message_type", message_type, EVENT_MESSAGE_TYPES) - await tools.send_event(content, message_type, None) + async with model_tool_activity(tools, BandTool.SEND_EVENT): + await tools.send_event(content, message_type, None) return ToolResult(data=f"Event ({message_type}) sent successfully") diff --git a/src/band/integrations/parlant/customtools.py b/src/band/integrations/parlant/customtools.py index 0f5c1f8d9..eb2ea0334 100644 --- a/src/band/integrations/parlant/customtools.py +++ b/src/band/integrations/parlant/customtools.py @@ -17,6 +17,7 @@ ) from band.runtime.custom_tools import CustomToolDef, execute_custom_tool from band.runtime.tools import tool_result_text +from band.runtime.working_state import tool_activity # Parlant delivers every argument as a string, which only lax validation # converts, so even a strict field is validated laxly. @@ -30,12 +31,13 @@ def build_custom_tool(tool_def: CustomToolDef) -> Any: async def run(context: ToolContext, **arguments: Any) -> ToolResult: tools = require_session_tools(context) - result = await execute_custom_tool( - tool_def, - parse_delivered(arguments, described.fields), - turn=tools.turn, - strict=STRICT_VALIDATION, - ) + async with tool_activity(tools, described.name): + result = await execute_custom_tool( + tool_def, + parse_delivered(arguments, described.fields), + turn=tools.turn, + strict=STRICT_VALIDATION, + ) return ToolResult(data=tool_result_text(result)) run.__name__ = described.name diff --git a/src/band/integrations/pydantic_ai/tools.py b/src/band/integrations/pydantic_ai/tools.py index a9d724b3b..caa967a8a 100644 --- a/src/band/integrations/pydantic_ai/tools.py +++ b/src/band/integrations/pydantic_ai/tools.py @@ -37,6 +37,7 @@ turn_effect, validate_tool_arguments, ) +from band.runtime.working_state import model_tool_activity logger = logging.getLogger(__name__) @@ -202,7 +203,8 @@ async def dispatch(ctx: RunContext[AgentToolsProtocol], **arguments: Any) -> Any # something to hand the model as a tool error it could act on. method = _resolve_method(ctx.deps, definition) try: - result = await method(**kwargs) + async with model_tool_activity(ctx.deps, definition.name): + result = await method(**kwargs) except Exception as error: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn return await _dispatch_failed(ctx, definition, error, notify_room=True) try: diff --git a/src/band/integrations/slack/adapter.py b/src/band/integrations/slack/adapter.py index f9b7ba152..191b503a3 100644 --- a/src/band/integrations/slack/adapter.py +++ b/src/band/integrations/slack/adapter.py @@ -156,6 +156,7 @@ def __init__( # adapter judging ``wrap`` sees what the brain did through this tee. self._ctx = wrap._ctx self._scope = wrap._scope + self._activity_observer = wrap._activity_observer self.turn = wrap.turn self._slack = slack self._binding = binding diff --git a/src/band/runtime/tools/agent.py b/src/band/runtime/tools/agent.py index 823291c83..56d340f23 100644 --- a/src/band/runtime/tools/agent.py +++ b/src/band/runtime/tools/agent.py @@ -73,8 +73,8 @@ serialize_tool_result, validate_tool_arguments, ) -from band.runtime.turn import records_turn_effects -from band.runtime.working_state import ActivityObserver +from band.runtime.turn import records_turn_effects, reports_tool_activity +from band.runtime.working_state import ActivityObserver, model_tool_activity if TYPE_CHECKING: from anthropic.types import ToolParam @@ -286,6 +286,7 @@ def cache_parameters(self) -> Any: ... @records_turn_effects +@reports_tool_activity class AgentTools(AgentToolsProtocol): """ Room-bound tools for LLM platform interaction. @@ -2042,7 +2043,8 @@ async def execute_tool_call_structured( try: method = getattr(self, definition.method_name) - result = await method(**arguments) + async with model_tool_activity(self, tool_name): + result = await method(**arguments) return ToolCallOutcome(value=serialize_tool_result(result), ok=True) except (BandToolError, RoomExecutionStoppedError): # Stopped execution must escape ordinary tool-error conversion. diff --git a/src/band/runtime/turn.py b/src/band/runtime/turn.py index c99551e48..5620ca6be 100644 --- a/src/band/runtime/turn.py +++ b/src/band/runtime/turn.py @@ -9,7 +9,12 @@ from band.core.turn import Turn from band.runtime.tools.effects import turn_effect from band.runtime.tools.registry import TOOL_DEFINITIONS -from band.runtime.tools.types import Surface, TurnEffect +from band.runtime.tools.types import BandTool, Surface, TurnEffect +from band.runtime.working_state import ( + end_tool_activity, + report_memory_activity, + report_tool_activity, +) _ToolsClass = TypeVar("_ToolsClass", bound=type) @@ -48,3 +53,37 @@ def records_turn_effects(cls: _ToolsClass) -> _ToolsClass: if definition.surface == Surface.AGENT and method is not None: setattr(cls, definition.method_name, _recording(definition.name, method)) return cls + + +MEMORY_SAVE_TOOLS = frozenset( + {BandTool.STORE_MEMORY, BandTool.SUPERSEDE_MEMORY, BandTool.ARCHIVE_MEMORY} +) + + +def _observing( + tool_name: str, method: Callable[..., Awaitable[Any]] +) -> Callable[..., Awaitable[Any]]: + @functools.wraps(method) + async def observe(self: Any, *args: Any, **kwargs: Any) -> Any: + if tool_name in MEMORY_SAVE_TOOLS: + await report_memory_activity(self) + else: + await report_tool_activity(self, tool_name) + try: + return await method(self, *args, **kwargs) + finally: + await end_tool_activity(self) + + return observe + + +def reports_tool_activity(cls: _ToolsClass) -> _ToolsClass: + for definition in TOOL_DEFINITIONS.values(): + method = cls.__dict__.get(definition.method_name) + if ( + definition.surface == Surface.AGENT + and definition.name != BandTool.SEND_EVENT + and method is not None + ): + setattr(cls, definition.method_name, _observing(definition.name, method)) + return cls diff --git a/src/band/runtime/working_state.py b/src/band/runtime/working_state.py index 318a9aac6..c35ba81d2 100644 --- a/src/band/runtime/working_state.py +++ b/src/band/runtime/working_state.py @@ -9,11 +9,15 @@ import asyncio import logging -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from inspect import getattr_static from typing import Protocol from band_rest.types.agent_activity_detail import AgentActivityDetail +from band.runtime.tools.types import BandTool + logger = logging.getLogger(__name__) ReportFn = Callable[[bool], Awaitable[object]] @@ -29,6 +33,9 @@ async def set_manual_wait(self, waiting: bool) -> None: ... def _observer(tools: object) -> ActivityObserver | None: + # Dynamic attribute proxies do not opt into runtime instrumentation. + if getattr_static(tools, "_activity_observer", None) is None: + return None return getattr(tools, "_activity_observer", None) @@ -56,6 +63,25 @@ async def set_permission_wait(tools: object, waiting: bool) -> None: await observer.set_manual_wait(waiting) +@asynccontextmanager +async def tool_activity(tools: object, name: str) -> AsyncIterator[None]: + await report_tool_activity(tools, name) + try: + yield + finally: + await end_tool_activity(tools) + + +@asynccontextmanager +async def model_tool_activity(tools: object, name: str) -> AsyncIterator[None]: + """Observe the dual-use event method only at its model invocation boundary.""" + if name == BandTool.SEND_EVENT: + async with tool_activity(tools, name): + yield + else: + yield + + def _retrieve_task_error(task: asyncio.Task[None]) -> None: if not task.cancelled(): task.exception() diff --git a/src/band/testing/activity.py b/src/band/testing/activity.py new file mode 100644 index 000000000..ebed4b670 --- /dev/null +++ b/src/band/testing/activity.py @@ -0,0 +1,18 @@ +"""Record the optional producer observation seam without scheduling reports.""" + +from __future__ import annotations + +from typing import Any + +from band_rest.types.agent_activity_detail import AgentActivityDetail + + +class ActivityRecorder: + def __init__(self) -> None: + self.events: list[dict[str, Any]] = [] + + async def update(self, activity: AgentActivityDetail) -> None: + self.events.append(activity.model_dump(exclude_unset=True)) + + async def set_manual_wait(self, waiting: bool) -> None: + self.events.append({"waiting": waiting}) diff --git a/src/band/testing/fake_tools.py b/src/band/testing/fake_tools.py index 35248e458..ee498fd90 100644 --- a/src/band/testing/fake_tools.py +++ b/src/band/testing/fake_tools.py @@ -82,6 +82,7 @@ strip_handle_prefix, ) from band.runtime.turn import record_tool_result, records_turn_effects +from band.runtime.working_state import ActivityObserver # Synthetic identity FakeAgentTools uses for the "joins you to the task on # first status/active_form write" semantics band_update_task documents. @@ -326,6 +327,7 @@ def __init__( self._held_messages: list[HeldMessage] = [] self._observers: list[tuple[Callable[[], bool], asyncio.Future[None]]] = [] self.turn = Turn() + self._activity_observer: ActivityObserver | None = None @property def agent_id(self) -> str | None: diff --git a/tests/adapters/activity/kit.py b/tests/adapters/activity/kit.py new file mode 100644 index 000000000..b2ec79ba4 --- /dev/null +++ b/tests/adapters/activity/kit.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from anthropic.types import Message, ToolUseBlock, Usage +from pydantic import BaseModel + + +class ActivityLookup(BaseModel): + """Look up an opaque test value.""" + + note: str + + +def anthropic_tool_response(name: str, note: str) -> Message: + return Message( + id="message", + role="assistant", + type="message", + model="test-model", + content=[ + ToolUseBlock(id="tool-id", name=name, type="tool_use", input={"note": note}) + ], + usage=Usage(input_tokens=1, output_tokens=1), + stop_reason="tool_use", + ) diff --git a/tests/adapters/activity/test_execution.py b/tests/adapters/activity/test_execution.py new file mode 100644 index 000000000..2a964e72d --- /dev/null +++ b/tests/adapters/activity/test_execution.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import asyncio + +import pytest +from google.genai import types + +from band.adapters.anthropic import AnthropicAdapter +from band.adapters.gemini import GeminiAdapter +from band.runtime.custom_tools import get_custom_tool_name +from band.testing.activity import ActivityRecorder +from band.testing.fake_tools import FakeAgentTools +from tests.adapters.activity.kit import ActivityLookup, anthropic_tool_response + + +class TestExecutedToolActivity: + @pytest.mark.parametrize("provider", ["anthropic", "gemini"]) + @pytest.mark.parametrize("outcome", ["success", "failure", "cancelled"]) + async def test_real_custom_execution_reports_names_only_and_always_ends( + self, provider: str, outcome: str + ) -> None: + entered, release = asyncio.Event(), asyncio.Event() + recorder = ActivityRecorder() + tools = FakeAgentTools() + tools._activity_observer = recorder + name = get_custom_tool_name(ActivityLookup) + + async def lookup(args: ActivityLookup) -> str: + assert args.note == "private-argument" + entered.set() + await release.wait() + if outcome == "failure": + raise ValueError("private-result") + return "private-result" + + tool = (ActivityLookup, lookup) + match provider: + case "anthropic": + adapter = AnthropicAdapter(additional_tools=[tool], emit=()) + operation = adapter._process_tool_calls( + anthropic_tool_response(name, "private-argument"), tools + ) + case "gemini": + adapter = GeminiAdapter(additional_tools=[tool], emit=()) + operation = adapter._process_function_calls( + [ + types.FunctionCall( + id="call", name=name, args={"note": "private-argument"} + ) + ], + tools, + ) + task = asyncio.create_task(operation) + await entered.wait() + assert recorder.events == [{"kind": "tool_call", "name": name}] + if outcome == "cancelled": + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + release.set() + await task + assert recorder.events == [ + {"kind": "tool_call", "name": name}, + {"kind": "reasoning"}, + ] + assert not tools.events_sent diff --git a/tests/adapters/activity/test_model_event.py b/tests/adapters/activity/test_model_event.py new file mode 100644 index 000000000..44a3a3cd9 --- /dev/null +++ b/tests/adapters/activity/test_model_event.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import pytest +from pydantic_ai import RunContext +from pydantic_ai.models.test import TestModel +from pydantic_ai.usage import RunUsage + +from band.integrations.mcp.engine import dispatch_tool +from band.integrations.pydantic_ai.tools import _dispatcher +from band.runtime.tools.registry import TOOL_DEFINITIONS +from band.runtime.tools.types import BandTool +from band.testing.activity import ActivityRecorder +from band.testing.fake_tools import FakeAgentTools + + +class TestModelEventActivity: + @pytest.mark.parametrize("surface", ["mcp", "pydantic"]) + async def test_model_event_reports_while_raw_telemetry_stays_quiet( + self, surface: str + ) -> None: + tools = FakeAgentTools() + recorder = ActivityRecorder() + tools._activity_observer = recorder + await tools.send_event("internal", "thought") + assert not recorder.events + definition = TOOL_DEFINITIONS[BandTool.SEND_EVENT] + arguments = {"content": "model", "message_type": "thought"} + if surface == "mcp": + await dispatch_tool(tools, definition, arguments) + else: + dispatch = _dispatcher(definition, definition.input_model) + await dispatch( + RunContext(deps=tools, model=TestModel(), usage=RunUsage()), **arguments + ) + assert recorder.events == [ + {"kind": "tool_call", "name": BandTool.SEND_EVENT}, + {"kind": "reasoning"}, + ] diff --git a/tests/adapters/activity/test_native.py b/tests/adapters/activity/test_native.py new file mode 100644 index 000000000..10fee1729 --- /dev/null +++ b/tests/adapters/activity/test_native.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import pytest +from claude_agent_sdk import ToolResultBlock, ToolUseBlock +from google.adk.events import Event +from google.genai import types + +from band.adapters.claude_sdk import ClaudeSDKAdapter +from band.adapters.google_adk import GoogleADKAdapter +from band.adapters.langgraph import LangGraphAdapter +from band.testing.activity import ActivityRecorder +from band.testing.fake_tools import FakeAgentTools + + +class TestNativeActivity: + @pytest.mark.parametrize("provider", ["claude", "langgraph", "adk"]) + async def test_start_and_end_are_independent_of_narration( + self, provider: str + ) -> None: + tools = FakeAgentTools() + recorder = ActivityRecorder() + tools._activity_observer = recorder + match provider: + case "claude": + adapter = ClaudeSDKAdapter(emit=()) + pending: dict[str, str] = {} + await adapter._on_tool_use( + ToolUseBlock( + id="call", name="Bash", input={"command": "private-command"} + ), + pending, + "room", + tools, + ) + await adapter._on_tool_result( + ToolResultBlock(tool_use_id="call", content="private-result"), + pending, + "room", + tools, + ) + case "langgraph": + adapter = LangGraphAdapter(graph_factory=lambda tools: None, emit=()) + await adapter._handle_stream_event( + { + "event": "on_tool_start", + "name": "Bash", + "data": {"input": "private-command"}, + }, + "room", + tools, + ) + await adapter._handle_stream_event( + { + "event": "on_tool_error", + "name": "Bash", + "data": {"error": "private-result"}, + }, + "room", + tools, + ) + case "adk": + adapter = GoogleADKAdapter(emit=()) + await adapter._report_event( + Event( + author="test", + content=types.Content( + parts=[ + types.Part( + function_call=types.FunctionCall( + name="Bash", args={"command": "private-command"} + ) + ) + ] + ), + ), + tools, + ) + await adapter._report_event( + Event( + author="test", + content=types.Content( + parts=[ + types.Part( + function_response=types.FunctionResponse( + name="Bash", + response={"output": "private-result"}, + ) + ) + ] + ), + ), + tools, + ) + assert recorder.events == [ + {"kind": "tool_call", "name": "Bash"}, + {"kind": "reasoning"}, + ] + assert not tools.events_sent diff --git a/tests/adapters/agno/adapter/__init__.py b/tests/adapters/agno/adapter/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/adapters/agno/adapter/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/agno/adapter/kit.py b/tests/adapters/agno/adapter/kit.py new file mode 100644 index 000000000..0e87a250a --- /dev/null +++ b/tests/adapters/agno/adapter/kit.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from datetime import UTC, datetime + +from band.core.types import PlatformMessage + + +def _msg( + room_id: str, + content: str, + *, + msg_id: str = "m1", + sender_id: str = "user-1", +) -> PlatformMessage: + """A minimal PlatformMessage for driving on_message in a given room.""" + return PlatformMessage( + id=msg_id, + room_id=room_id, + content=content, + sender_id=sender_id, + sender_type="User", + sender_name="Alice", + message_type="text", + metadata={}, + created_at=datetime.now(UTC), + ) diff --git a/tests/adapters/agno/adapter/test_bandentrypointbinding.py b/tests/adapters/agno/adapter/test_bandentrypointbinding.py new file mode 100644 index 000000000..33479532a --- /dev/null +++ b/tests/adapters/agno/adapter/test_bandentrypointbinding.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import json +from typing import Any + +from agno.tools.function import ToolResult + +from band.adapters.agno import ( + _bind_room_tools, + _make_band_entrypoint, +) +from band.testing import FakeAgentTools + + +class TestBandEntrypointBinding: + async def test_routes_to_execute_tool_call_inside_context(self, tools): + entry = _make_band_entrypoint("band_lookup_peers") + + with _bind_room_tools(tools): + result = await entry(page=1) + + assert tools.tool_calls == [ + {"tool_name": "band_lookup_peers", "arguments": {"page": 1}} + ] + assert json.loads(result) == {"status": "ok"} + + async def test_passes_string_results_through_unchanged(self): + class _StrTools(FakeAgentTools): + async def execute_tool_call(self, tool_name: str, arguments: dict) -> Any: + return "raw-string" + + entry = _make_band_entrypoint("band_lookup_peers") + with _bind_room_tools(_StrTools()): + assert await entry() == "raw-string" + + async def test_errors_outside_any_bound_context(self, tools): + entry = _make_band_entrypoint("band_lookup_peers") + + # Bind then exit; the ContextVar must reset so later calls have no tools. + with _bind_room_tools(tools): + pass + result = await entry(page=1) + + assert "no active Band context" in result + assert tools.tool_calls == [] + + async def test_read_room_file_image_result_passes_through_as_agno_image(self): + class _ImageTools(FakeAgentTools): + async def execute_tool_call(self, tool_name: str, arguments: dict) -> Any: + return { + "content": [ + {"type": "image", "data": "ZmFrZQ==", "mimeType": "image/png"} + ] + } + + entry = _make_band_entrypoint("band_read_room_file") + with _bind_room_tools(_ImageTools()): + result = await entry(file_id="f1") + + assert isinstance(result, ToolResult) + assert result.images is not None + assert len(result.images) == 1 + assert result.images[0].content == b"fake" + assert result.images[0].mime_type == "image/png" + + async def test_read_room_file_non_image_result_stays_json(self): + class _TextTools(FakeAgentTools): + async def execute_tool_call(self, tool_name: str, arguments: dict) -> Any: + return {"name": "notes.txt", "content_type": "text/plain"} + + entry = _make_band_entrypoint("band_read_room_file") + with _bind_room_tools(_TextTools()): + result = await entry(file_id="f1") + + assert result == '{"name": "notes.txt", "content_type": "text/plain"}' + + async def test_read_room_file_malformed_image_data_returns_error_string(self): + """A decode_image_block failure (invalid base64) must degrade to the + adapter's usual error string, not raise uncaught out of the tool + entrypoint Agno invokes directly.""" + + class _MalformedImageTools(FakeAgentTools): + async def execute_tool_call(self, tool_name: str, arguments: dict) -> Any: + return { + "content": [{"type": "image", "data": "A", "mimeType": "image/png"}] + } + + entry = _make_band_entrypoint("band_read_room_file") + with _bind_room_tools(_MalformedImageTools()): + result = await entry(file_id="f1") + + assert isinstance(result, str) + assert result.startswith("Error reading room file:") diff --git a/tests/adapters/agno/adapter/test_bandinstructioninjection.py b/tests/adapters/agno/adapter/test_bandinstructioninjection.py new file mode 100644 index 000000000..93c4c9924 --- /dev/null +++ b/tests/adapters/agno/adapter/test_bandinstructioninjection.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import pytest + +from band.core.types import Capability + + +class TestBandInstructionInjection: + """Drive a real Agno agent so we assert on the system prompt Agno actually + assembled and sent to the model, not the attribute the adapter set.""" + + @pytest.mark.parametrize( + ("capabilities", "present", "absent"), + [ + (set(), [], ["## Memory Tools", "## Contact Management Tools"]), + ( + {Capability.MEMORY}, + ["## Memory Tools"], + ["## Contact Management Tools"], + ), + ( + {Capability.CONTACTS}, + ["## Contact Management Tools"], + ["## Memory Tools"], + ), + ], + ) + async def test_capability_sections_gated_in_model_prompt( + self, run_real_agent, sample_platform_message, capabilities, present, absent + ): + model = await run_real_agent( + sample_platform_message, + capabilities=capabilities, + ) + prompt = model.captured_system_prompt + + assert "## Environment" in prompt # base guidance always injected + assert all(section in prompt for section in present) + assert all(section not in prompt for section in absent) + + async def test_developer_instructions_survive_in_prompt( + self, run_real_agent, sample_platform_message + ): + model = await run_real_agent( + sample_platform_message, + instructions="You are Dev, a niche specialist.", + additional_context="Keep replies under 10 words.", + ) + prompt = model.captured_system_prompt + + assert "You are Dev, a niche specialist." in prompt + assert "Keep replies under 10 words." in prompt + assert "## Environment" in prompt + + async def test_guidance_injected_at_startup_before_any_message( + self, make_started_adapter + ): + # Band guidance is injected in on_started, not lazily on first message. + _adapter, agent = await make_started_adapter() + + # The Band operating contract frames the agent via ``description`` (ahead of + # the developer's own ``instructions``), not via ``additional_context``. + assert isinstance(agent.description, str) + assert "## Environment" in agent.description + # Band-registered identity is injected so the model knows who it is. + assert "You are TestBot, desc." in agent.description diff --git a/tests/adapters/agno/adapter/test_emitexecution.py b/tests/adapters/agno/adapter/test_emitexecution.py new file mode 100644 index 000000000..4874ad68e --- /dev/null +++ b/tests/adapters/agno/adapter/test_emitexecution.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import json + +from agno.run.agent import RunOutput + +from band.core.types import Emit +from tests.adapters.agno.helpers import ( + tool_events, + tool_execution, +) + + +class TestEmitExecution: + async def test_emits_tool_call_and_result_events( + self, make_started_adapter, sample_platform_message, tools + ): + # Streamed run: started + completed events for one tool call. Reporting + # is live (during the run), driven off Agno's native stream events. + events = tool_events( + tool_execution("band_lookup_peers", args={"page": "1"}, result="ok") + ) + adapter, _ = await make_started_adapter(emit=Emit.TOOL_CALLS, events=events) + + await adapter.on_message( + sample_platform_message, + tools, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + types = [e["message_type"] for e in tools.events_sent] + assert types == ["tool_call", "tool_result"] + call_payload = json.loads(tools.events_sent[0]["content"]) + result_payload = json.loads(tools.events_sent[1]["content"]) + assert call_payload == { + "name": "band_lookup_peers", + "args": {"page": "1"}, + "tool_call_id": "tc_1", + } + assert result_payload["output"] == "ok" + assert result_payload["is_error"] is False + + async def test_band_send_message_tool_call_is_reported( + self, make_started_adapter, sample_platform_message, tools + ): + """band_send_message is reported like any other tool call — no suppression.""" + events = tool_events(tool_execution("band_send_message")) + adapter, _ = await make_started_adapter(emit=Emit.TOOL_CALLS, events=events) + + await adapter.on_message( + sample_platform_message, + tools, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + types = [e["message_type"] for e in tools.events_sent] + assert types == ["tool_call", "tool_result"] + + async def test_send_room_file_tool_call_event_redacts_content( + self, make_started_adapter, sample_platform_message, tools + ): + """band_send_room_file's tool_call event must report a bounded + placeholder for content, not the raw file bytes -- generic ARGS + reporting has no idea this one tool's content argument can carry up + to MAX_SEND_CONTENT_BYTES of real file data.""" + events = tool_events( + tool_execution( + "band_send_room_file", + args={"content": "raw file bytes", "filename": "notes.txt"}, + result="ok", + ) + ) + adapter, _ = await make_started_adapter(emit=Emit.TOOL_CALLS, events=events) + + await adapter.on_message( + sample_platform_message, + tools, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + call_payload = json.loads(tools.events_sent[0]["content"]) + assert call_payload["args"]["content"] == "<14 byte file content>" + assert call_payload["args"]["filename"] == "notes.txt" + + async def test_no_events_without_execution_emit( + self, make_started_adapter, sample_platform_message, tools + ): + # emit=() -> non-streaming run, no live reporting. The final + # RunOutput still carries executions, but nothing is emitted. + response = RunOutput(tools=[tool_execution("band_lookup_peers")]) + adapter, agent = await make_started_adapter(response, emit=()) + + await adapter.on_message( + sample_platform_message, + tools, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + assert tools.events_sent == [] + agent.arun.assert_called_once() diff --git a/tests/adapters/agno/adapter/test_emitthoughts.py b/tests/adapters/agno/adapter/test_emitthoughts.py new file mode 100644 index 000000000..2eb5fee8d --- /dev/null +++ b/tests/adapters/agno/adapter/test_emitthoughts.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from agno.run.agent import RunOutput + +from band.core.types import Emit + + +class TestEmitThoughts: + async def test_emits_reasoning_as_thought( + self, make_started_adapter, sample_platform_message, tools + ): + response = RunOutput(reasoning_content="thinking hard") + adapter, _ = await make_started_adapter(response, emit=Emit.THOUGHTS) + + await adapter.on_message( + sample_platform_message, + tools, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + tools.assert_event_sent(message_type="thought") + assert tools.events_sent[0]["content"] == "thinking hard" + + async def test_no_thought_without_thoughts_emit( + self, make_started_adapter, sample_platform_message, tools + ): + response = RunOutput(reasoning_content="thinking hard") + adapter, _ = await make_started_adapter(response, emit=()) + + await adapter.on_message( + sample_platform_message, + tools, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + assert tools.events_sent == [] + + async def test_no_thought_for_blank_reasoning( + self, make_started_adapter, sample_platform_message, tools + ): + adapter, _ = await make_started_adapter( + RunOutput(reasoning_content=" "), + emit=Emit.THOUGHTS, + ) + + await adapter.on_message( + sample_platform_message, + tools, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + assert tools.events_sent == [] diff --git a/tests/adapters/agno/adapter/test_failedrundoesnotcontaminatenextturn.py b/tests/adapters/agno/adapter/test_failedrundoesnotcontaminatenextturn.py new file mode 100644 index 000000000..c04393f08 --- /dev/null +++ b/tests/adapters/agno/adapter/test_failedrundoesnotcontaminatenextturn.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from band.adapters.agno import ( + AgnoAdapter, +) +from tests.adapters.agno.adapter.kit import _msg +from tests.adapters.agno.helpers import ( + run_input, +) + + +class TestFailedRunDoesNotContaminateNextTurn: + async def test_failed_turn_leaves_no_residue_in_next_run_input( + self, make_agno_agent, tools + ): + # Turn 1 raises mid-run; turn 2 succeeds. The injected system/user + # messages from the failed turn must not survive into turn 2's input. + agent = make_agno_agent() + succeeding = agent.arun.side_effect + agent.arun = MagicMock(side_effect=[RuntimeError("boom"), succeeding()]) + adapter = AgnoAdapter(agent=agent, emit=()) + await adapter.on_started("TestBot", "desc") + + first = _msg("room-1", "first question", msg_id="m1") + with pytest.raises(RuntimeError): + await adapter.on_message( + first, + tools, + [], + "P1-participants", + "C1-contacts", + is_session_bootstrap=True, + room_id="room-1", + ) + + second = _msg("room-1", "second question", msg_id="m2") + await adapter.on_message( + second, + tools, + [], + "P2-participants", + "C2-contacts", + is_session_bootstrap=False, + room_id="room-1", + ) + + contents = [m.content for m in run_input(agent)] + # No residue from the failed turn 1. + assert not any("P1-participants" in c for c in contents) + assert not any("C1-contacts" in c for c in contents) + assert first.format_for_llm() not in contents + # Turn 2's own injected context and live message are present. + assert any("P2-participants" in c for c in contents) + assert contents[-1] == second.format_for_llm() diff --git a/tests/adapters/agno/adapter/test_featurefilters.py b/tests/adapters/agno/adapter/test_featurefilters.py new file mode 100644 index 000000000..c05d72dea --- /dev/null +++ b/tests/adapters/agno/adapter/test_featurefilters.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from typing import Any, ClassVar + +from band.adapters.agno import ( + _bind_room_tools, +) +from tests.adapters.agno.helpers import ( + SchemaTools, + openai_tool_schema, +) + + +class TestFeatureFilters: + """include_tools/exclude_tools/include_categories gate which Band tools + are wired (parity with LangGraph).""" + + ALL_SCHEMAS: ClassVar[list[dict[str, Any]]] = [ + openai_tool_schema("band_send_message"), # chat + openai_tool_schema("band_lookup_peers"), # chat + openai_tool_schema("band_store_memory"), # memory + openai_tool_schema("band_add_contact"), # contacts + ] + + async def _resolved_names(self, adapter) -> list[str]: + tools = SchemaTools(self.ALL_SCHEMAS) + with _bind_room_tools(tools): + resolved = await adapter._resolve_room_tools() + return [fn.name for fn in resolved] + + async def test_include_tools_keeps_only_named(self, make_started_adapter): + adapter, _ = await make_started_adapter(include_tools=["band_send_message"]) + + assert await self._resolved_names(adapter) == ["band_send_message"] + + async def test_exclude_tools_drops_named(self, make_started_adapter): + adapter, _ = await make_started_adapter(exclude_tools=["band_send_message"]) + + names = await self._resolved_names(adapter) + assert "band_send_message" not in names + assert "band_lookup_peers" in names + + async def test_include_categories_keeps_only_category(self, make_started_adapter): + adapter, _ = await make_started_adapter(include_categories=["chat"]) + + assert sorted(await self._resolved_names(adapter)) == [ + "band_lookup_peers", + "band_send_message", + ] diff --git a/tests/adapters/agno/adapter/test_hubcontactexposure.py b/tests/adapters/agno/adapter/test_hubcontactexposure.py new file mode 100644 index 000000000..22b82e2f4 --- /dev/null +++ b/tests/adapters/agno/adapter/test_hubcontactexposure.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from band.adapters.agno import ( + _bind_room_tools, +) +from tests.adapters.agno.helpers import ( + ContactAwareTools, + SchemaTools, +) + + +class TestHubContactExposure: + """The adapter decides contact exposure (mirrors LangGraph): the CONTACTS + capability OR a hub room force-includes contact tool schemas, resolved per + run so visibility is strictly per-room.""" + + async def test_normal_room_does_not_request_contacts(self, make_started_adapter): + adapter, _ = await make_started_adapter() + tools = SchemaTools([], room_id="room-A") + + with _bind_room_tools(tools): + await adapter._resolve_room_tools() + + assert tools.schema_calls == [ + {"include_memory": False, "include_contacts": False} + ] + + async def test_hub_room_forces_contacts(self, make_started_adapter): + adapter, _ = await make_started_adapter() + tools = SchemaTools([], hub_room_id="hub", room_id="hub") + + with _bind_room_tools(tools): + await adapter._resolve_room_tools() + + assert tools.schema_calls == [ + {"include_memory": False, "include_contacts": True} + ] + + async def test_contacts_do_not_leak_into_normal_room_after_hub( + self, make_started_adapter + ): + # Core regression: after a hub room exposes contact tools, a subsequent + # normal room's resolution must NOT include them. The old additive wiring + # accumulated the union on the shared agent; per-run resolution does not. + adapter, _ = await make_started_adapter() + + hub = ContactAwareTools(hub_room_id="hub", room_id="hub") + with _bind_room_tools(hub): + hub_names = [fn.name for fn in await adapter._resolve_room_tools()] + assert "band_add_contact" in hub_names + + normal = ContactAwareTools(room_id="room-A") + with _bind_room_tools(normal): + normal_names = [fn.name for fn in await adapter._resolve_room_tools()] + + assert normal_names == ["band_send_message"] + assert "band_add_contact" not in normal_names diff --git a/tests/adapters/agno/adapter/test_memorycollisionwarning.py b/tests/adapters/agno/adapter/test_memorycollisionwarning.py new file mode 100644 index 000000000..0ca67f9a0 --- /dev/null +++ b/tests/adapters/agno/adapter/test_memorycollisionwarning.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import warnings + +import pytest + +from band.adapters.agno import ( + AgnoAdapter, +) +from band.core.types import Capability + + +class TestMemoryCollisionWarning: + """Collision is detected against the runtime agent at startup, not __init__.""" + + async def test_warns_on_update_memory_on_run_with_memory_capability( + self, make_agno_agent + ): + agent = make_agno_agent(update_memory_on_run=True) + adapter = AgnoAdapter(agent=agent, capabilities=Capability.MEMORY) + + with pytest.warns(UserWarning, match="update_memory_on_run"): + await adapter.on_started("TestBot", "desc") + + async def test_warns_on_agentic_memory_with_memory_capability( + self, make_agno_agent + ): + agent = make_agno_agent(enable_agentic_memory=True) + adapter = AgnoAdapter(agent=agent, capabilities=Capability.MEMORY) + + with pytest.warns(UserWarning, match="enable_agentic_memory"): + await adapter.on_started("TestBot", "desc") + + async def test_no_warning_without_memory_capability(self, make_agno_agent): + agent = make_agno_agent(update_memory_on_run=True, enable_agentic_memory=True) + adapter = AgnoAdapter(agent=agent) # no MEMORY capability -> no collision + + with warnings.catch_warnings(): + warnings.simplefilter("error") + await adapter.on_started("TestBot", "desc") diff --git a/tests/adapters/agno/adapter/test_oncleanup.py b/tests/adapters/agno/adapter/test_oncleanup.py new file mode 100644 index 000000000..9c609f0f7 --- /dev/null +++ b/tests/adapters/agno/adapter/test_oncleanup.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from agno.models.message import Message + +from band.adapters.agno import ( + AgnoAdapter, +) + + +class TestOnCleanup: + async def test_drops_room_transcript(self, make_agno_agent): + agent = make_agno_agent() + adapter = AgnoAdapter(agent=agent) + adapter._message_history["room-1"] = [Message(role="user", content="hi")] + + await adapter.on_cleanup("room-1") + + assert "room-1" not in adapter._message_history + + async def test_unknown_room_is_noop(self, make_agno_agent): + agent = make_agno_agent() + adapter = AgnoAdapter(agent=agent) + + await adapter.on_cleanup("never-seen") # must not raise diff --git a/tests/adapters/agno/adapter/test_onstarted.py b/tests/adapters/agno/adapter/test_onstarted.py new file mode 100644 index 000000000..1e8cd7cc4 --- /dev/null +++ b/tests/adapters/agno/adapter/test_onstarted.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from agno.run.agent import RunOutput + +from band.adapters.agno import ( + AgnoAdapter, +) +from tests.adapters.agno.adapter.kit import _msg + + +class TestOnStarted: + async def test_runs_against_the_given_agent(self, make_agno_agent): + agent = make_agno_agent() + adapter = AgnoAdapter(agent=agent) + + await adapter.on_started("TestBot", "desc") + + # The adapter uses the caller's instance directly, no copy. + assert adapter.agent is agent + + async def test_syncs_converter_identity(self, make_started_adapter): + adapter, _ = await make_started_adapter() + + assert adapter.history_converter._agent_name == "TestBot" + + async def test_runs_the_given_agent_on_a_message(self, make_agno_agent, tools): + # End-to-end: the given agent must be the one actually run on a message. + # The adapter delivers nothing on its own; plain agent text is not sent + # (only a ``band_send_message`` tool call reaches the room). + agent = make_agno_agent(response=RunOutput(content="hi there")) + adapter = AgnoAdapter(agent=agent, emit=()) + await adapter.on_started("TestBot", "desc") + + await adapter.on_message( + _msg("room-1", "hello"), + tools, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + agent.arun.assert_called_once() + tools.assert_no_messages_sent() diff --git a/tests/adapters/agno/adapter/test_perruntoolexposureendtoend.py b/tests/adapters/agno/adapter/test_perruntoolexposureendtoend.py new file mode 100644 index 000000000..28aad1e8b --- /dev/null +++ b/tests/adapters/agno/adapter/test_perruntoolexposureendtoend.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from agno.agent import Agent as AgnoAgent + +from band.adapters.agno import ( + AgnoAdapter, +) +from tests.adapters.agno.adapter.kit import _msg +from tests.adapters.agno.helpers import ( + CapturingModel, + ContactAwareTools, +) + + +class TestPerRunToolExposureEndToEnd: + """Drive a real Agno agent so we assert on the tools Agno actually offered + the model per run -- proving the factory is installed and invoked per turn, + and that contact tools do not leak across rooms through the shared agent.""" + + async def test_model_receives_only_active_room_tools(self): + model = CapturingModel() + agno = AgnoAgent(model=model, instructions="You are Dev.") + adapter = AgnoAdapter(agent=agno, emit=()) + await adapter.on_started("Bot", "desc") + + # Hub room: contact tools are offered to the model. + hub = ContactAwareTools(hub_room_id="hub", room_id="hub") + await adapter.on_message( + _msg("hub", "hi"), + hub, + [], + None, + None, + is_session_bootstrap=True, + room_id="hub", + ) + assert model.captured_tool_names is not None + assert "band_add_contact" in model.captured_tool_names + + # Normal room afterwards on the same shared agent: no contact leak. + normal = ContactAwareTools(room_id="room-A") + await adapter.on_message( + _msg("room-A", "hi"), + normal, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-A", + ) + assert model.captured_tool_names == ["band_send_message"] diff --git a/tests/adapters/agno/adapter/test_persistandaccumulate.py b/tests/adapters/agno/adapter/test_persistandaccumulate.py new file mode 100644 index 000000000..2624391d7 --- /dev/null +++ b/tests/adapters/agno/adapter/test_persistandaccumulate.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +from agno.models.message import Message +from agno.run.agent import RunOutput + +from band.adapters.agno import ( + AgnoAdapter, +) + + +class TestPersistAndAccumulate: + def test_persist_keeps_only_conversation_roles(self, make_agno_agent): + agent = make_agno_agent() + adapter = AgnoAdapter(agent=agent) + response = RunOutput( + messages=[ + Message(role="system", content="instructions"), + Message(role="user", content="hi"), + Message(role="assistant", content="hello"), + Message(role="developer", content="state"), + Message(role="tool", content="result"), + ] + ) + + adapter._persist_turn("room-1", response) + + kept = [m.role for m in adapter._message_history["room-1"]] + assert kept == ["user", "assistant", "tool"] + + def test_bootstrap_seeds_committed_transcript_from_history( + self, make_agno_agent, sample_platform_message + ): + # Bootstrap seeds the committed transcript from rehydrated history. The + # returned run input is that seed plus this turn's live message, but + # building it must NOT push the live message into the committed store. + agent = make_agno_agent() + adapter = AgnoAdapter(agent=agent) + seed = [Message(role="user", content="earlier")] + + run_input_msgs = adapter._build_run_input( + sample_platform_message, + seed, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + assert [m.content for m in run_input_msgs] == [ + "earlier", + sample_platform_message.format_for_llm(), + ] + # Committed transcript holds only the rehydrated seed. + assert [m.content for m in adapter._message_history["room-1"]] == ["earlier"] + + def test_build_run_input_does_not_mutate_committed_transcript( + self, make_agno_agent, sample_platform_message + ): + # A non-bootstrap turn reads the committed transcript but never writes to + # it; the store is only ever advanced by _persist_turn after a run. + agent = make_agno_agent() + adapter = AgnoAdapter(agent=agent) + adapter._message_history["room-1"] = [Message(role="user", content="committed")] + + adapter._build_run_input( + sample_platform_message, + [], + "participants", + "contacts", + is_session_bootstrap=False, + room_id="room-1", + ) + + assert [m.content for m in adapter._message_history["room-1"]] == ["committed"] diff --git a/tests/adapters/agno/adapter/test_reply.py b/tests/adapters/agno/adapter/test_reply.py new file mode 100644 index 000000000..b22bc350c --- /dev/null +++ b/tests/adapters/agno/adapter/test_reply.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from agno.run.agent import RunOutput + +from tests.adapters.agno.helpers import ( + tool_execution, +) + + +class TestReply: + """The adapter delivers nothing on its own. Like the other adapters, the + agent must call ``band_send_message`` to reach the room; plain agent text is + never auto-sent and the adapter never guesses a recipient.""" + + async def test_no_send_when_agent_returns_only_text( + self, make_started_adapter, sample_platform_message, tools + ): + adapter, _ = await make_started_adapter(RunOutput(content="hello")) + + await adapter.on_message( + sample_platform_message, + tools, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + tools.assert_no_messages_sent() + + async def test_no_send_when_agent_called_band_send_message( + self, make_started_adapter, sample_platform_message, tools + ): + # The tool call itself reaches the room; the adapter adds nothing on top. + response = RunOutput( + content="hello", tools=[tool_execution("band_send_message")] + ) + adapter, _ = await make_started_adapter(response) + + await adapter.on_message( + sample_platform_message, + tools, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + tools.assert_no_messages_sent() + + async def test_no_send_for_empty_content( + self, make_started_adapter, sample_platform_message, tools + ): + adapter, _ = await make_started_adapter(RunOutput(content=" ")) + + await adapter.on_message( + sample_platform_message, + tools, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + tools.assert_no_messages_sent() diff --git a/tests/adapters/agno/adapter/test_roomtoolresolution.py b/tests/adapters/agno/adapter/test_roomtoolresolution.py new file mode 100644 index 000000000..bde88b9ba --- /dev/null +++ b/tests/adapters/agno/adapter/test_roomtoolresolution.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +from band.adapters.agno import ( + AgnoAdapter, + _bind_room_tools, +) +from band.core.types import Capability +from tests.adapters.agno.helpers import ( + SchemaTools, + openai_tool_schema, +) + + +class TestRoomToolResolution: + """Band tools are exposed per-run via the ``_resolve_room_tools`` factory + Agno calls each turn, not wired onto the agent. These pin what that factory + returns and the schema requests it makes for the active room.""" + + async def test_resolves_band_tools_for_active_room(self, make_started_adapter): + tools = SchemaTools( + [ + openai_tool_schema("band_send_message"), + openai_tool_schema("band_lookup_peers"), + ] + ) + adapter, _ = await make_started_adapter() + + with _bind_room_tools(tools): + resolved = await adapter._resolve_room_tools() + + assert [fn.name for fn in resolved] == [ + "band_send_message", + "band_lookup_peers", + ] + + async def test_no_band_tools_outside_a_bound_room(self, make_started_adapter): + # Defensive: with no active room bound, the factory exposes no Band tools + # (and does not even request schemas) rather than guessing visibility. + tools = SchemaTools([openai_tool_schema("band_send_message")]) + adapter, _ = await make_started_adapter() + + resolved = await adapter._resolve_room_tools() # no _bind_room_tools + + assert resolved == [] + assert tools.schema_calls == [] + + async def test_capability_flags_drive_schema_request(self, make_started_adapter): + tools = SchemaTools([]) + adapter, _ = await make_started_adapter( + capabilities={Capability.MEMORY, Capability.CONTACTS} + ) + + with _bind_room_tools(tools): + await adapter._resolve_room_tools() + + assert tools.schema_calls == [ + {"include_memory": True, "include_contacts": True} + ] + + async def test_schema_build_is_cached_across_runs(self, make_started_adapter): + # Same contact flag across runs -> schemas are built once and reused, + # not rebuilt every turn. + tools = SchemaTools([openai_tool_schema("band_send_message")]) + adapter, _ = await make_started_adapter() + + with _bind_room_tools(tools): + await adapter._resolve_room_tools() + await adapter._resolve_room_tools() + await adapter._resolve_room_tools() + + assert tools.schema_calls == [ + {"include_memory": False, "include_contacts": False} + ] + + async def test_user_tools_are_reincluded(self, make_agno_agent): + # Replacing agent.tools with our factory must not drop the user's own + # tools; they are re-included alongside the room's Band tools. + user_tool = object() + agent = make_agno_agent() + agent.tools = [user_tool] + adapter = AgnoAdapter(agent=agent) + await adapter.on_started("TestBot", "desc") + + tools = SchemaTools([openai_tool_schema("band_send_message")]) + with _bind_room_tools(tools): + resolved = await adapter._resolve_room_tools() + + assert resolved[0] is user_tool + assert [getattr(t, "name", None) for t in resolved[1:]] == ["band_send_message"] diff --git a/tests/adapters/agno/adapter/test_runfailurereporting.py b/tests/adapters/agno/adapter/test_runfailurereporting.py new file mode 100644 index 000000000..52c35a834 --- /dev/null +++ b/tests/adapters/agno/adapter/test_runfailurereporting.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +from typing import Any + +import pytest +from agno.run import RunStatus +from agno.run.agent import RunErrorEvent, RunOutput + +from band.adapters.agno import ( + AgnoRunError, +) +from band.core.protocols import GENERIC_PROVIDER_FAILURE_MESSAGE +from band.core.types import Emit +from band.testing import FakeAgentTools, reported_failures +from tests.adapters.agno.adapter.kit import _msg + + +class TestRunFailureReporting: + async def test_emits_generic_error_event_and_reraises( + self, make_started_adapter, tools + ): + adapter, agent = await make_started_adapter() + agent.arun.side_effect = RuntimeError("db dsn leaked: secret-token") + + with pytest.raises(RuntimeError): + await adapter.on_message( + _msg("room-A", "hi"), + tools, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-A", + ) + + failures = reported_failures(tools) + assert len(failures) == 1 + assert failures[0]["message"] == GENERIC_PROVIDER_FAILURE_MESSAGE + # The exception text (which can carry secrets) must not leak to the room. + assert "secret-token" not in failures[0]["message"] + assert failures[0]["provider"] == "agno" + # A plain RuntimeError isn't a swallowed Agno run status -- no code. + assert failures[0]["code"] is None + + async def test_error_status_run_is_raised_and_reported( + self, make_started_adapter, tools + ): + # Agno swallows run exceptions (model/API failures included) and + # returns a normal-looking RunOutput with status=error. The adapter + # must surface it as a failure — otherwise the runtime marks the + # message processed and the turn dies silently with no reply. + adapter, _agent = await make_started_adapter( + response=RunOutput( + content="api dsn leaked: secret-token", status=RunStatus.error + ) + ) + + with pytest.raises(AgnoRunError): + await adapter.on_message( + _msg("room-A", "hi"), + tools, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-A", + ) + + failures = reported_failures(tools) + assert len(failures) == 1 + assert "secret-token" not in failures[0]["message"] + assert failures[0]["code"] == RunStatus.error.value + # A failed turn must not be committed to the room transcript. + assert not adapter._message_history.get("room-A") + + async def test_streaming_error_event_is_raised_and_reported( + self, make_started_adapter, tools + ): + # In the streaming path a swallowed error surfaces only as a + # RunErrorEvent — the stream never yields a final RunOutput. The + # adapter must raise on it instead of returning None (a "successful" + # empty turn). + adapter, _agent = await make_started_adapter( + emit=Emit.TOOL_CALLS, + events=[RunErrorEvent(content="api dsn leaked: secret-token")], + ) + + with pytest.raises(AgnoRunError): + await adapter.on_message( + _msg("room-A", "hi"), + tools, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-A", + ) + + failures = reported_failures(tools) + assert len(failures) == 1 + assert "secret-token" not in failures[0]["message"] + assert failures[0]["code"] == RunStatus.error.value + assert not adapter._message_history.get("room-A") + + async def test_error_event_failure_does_not_mask_original( + self, make_started_adapter + ): + adapter, agent = await make_started_adapter() + agent.arun.side_effect = RuntimeError("boom") + + class _FailingEventTools(FakeAgentTools): + async def send_event(self, *args: Any, **kwargs: Any) -> dict[str, Any]: + raise RuntimeError("event transport down") + + # The failed error-report must not replace the original exception. + with pytest.raises(RuntimeError, match="boom"): + await adapter.on_message( + _msg("room-A", "hi"), + _FailingEventTools(), + [], + None, + None, + is_session_bootstrap=True, + room_id="room-A", + ) diff --git a/tests/adapters/agno/adapter/test_sessionisolation.py b/tests/adapters/agno/adapter/test_sessionisolation.py new file mode 100644 index 000000000..59f5a850a --- /dev/null +++ b/tests/adapters/agno/adapter/test_sessionisolation.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +from band.adapters.agno import ( + AgnoAdapter, +) +from band.testing import FakeAgentTools +from tests.adapters.agno.adapter.kit import _msg + + +class TestSessionIsolation: + async def test_arun_uses_room_id_as_session_id(self, make_started_adapter): + adapter, agent = await make_started_adapter() + + await adapter.on_message( + _msg("room-A", "hi"), + FakeAgentTools(), + [], + None, + None, + is_session_bootstrap=True, + room_id="room-A", + ) + + assert agent.arun.call_args.kwargs["session_id"] == "room-A" + + async def test_custom_session_id_factory_is_used(self, make_agno_agent): + agent = make_agno_agent() + adapter = AgnoAdapter( + agent=agent, session_id_factory=lambda room: f"sess::{room}", emit=() + ) + await adapter.on_started("TestBot", "desc") + + await adapter.on_message( + _msg("room-A", "hi"), + FakeAgentTools(), + [], + None, + None, + is_session_bootstrap=True, + room_id="room-A", + ) + + assert agent.arun.call_args.kwargs["session_id"] == "sess::room-A" + + async def test_two_rooms_get_isolated_sessions_and_inputs( + self, make_started_adapter + ): + adapter, agent = await make_started_adapter() + + await adapter.on_message( + _msg("room-A", "alpha-secret"), + FakeAgentTools(), + [], + None, + None, + is_session_bootstrap=True, + room_id="room-A", + ) + await adapter.on_message( + _msg("room-B", "beta-secret"), + FakeAgentTools(), + [], + None, + None, + is_session_bootstrap=True, + room_id="room-B", + ) + + calls = agent.arun.call_args_list + assert calls[0].kwargs["session_id"] == "room-A" + assert calls[1].kwargs["session_id"] == "room-B" + + room_b_input = " ".join(m.content or "" for m in calls[1].kwargs["input"]) + assert "beta-secret" in room_b_input + assert "alpha-secret" not in room_b_input diff --git a/tests/adapters/agno/adapter/test_usedbeforestarted.py b/tests/adapters/agno/adapter/test_usedbeforestarted.py new file mode 100644 index 000000000..7fe19d3a4 --- /dev/null +++ b/tests/adapters/agno/adapter/test_usedbeforestarted.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +import pytest + +from band.adapters.agno import ( + AgnoAdapter, +) +from band.testing import FakeAgentTools + + +class TestUsedBeforeStarted: + async def test_run_agent_before_on_started_raises(self, make_agno_agent): + agent = make_agno_agent() + adapter = AgnoAdapter(agent=agent) + + with pytest.raises(RuntimeError, match="before on_started"): + await adapter._run_agent( + [], FakeAgentTools(), room_id="room-1", msg_id="m1" + ) diff --git a/tests/adapters/agno/helpers.py b/tests/adapters/agno/helpers.py index 074600451..9025e6685 100644 --- a/tests/adapters/agno/helpers.py +++ b/tests/adapters/agno/helpers.py @@ -12,7 +12,7 @@ from collections.abc import Awaitable, Callable from typing import Any -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock from agno.models.base import Model from agno.models.message import Message @@ -61,30 +61,17 @@ def fake_agno_agent( agent.tools = [] resp = response if response is not None else RunOutput() - if events is None: - # Non-streaming path (Emit.TOOL_CALLS off): `await agent.arun(...)`. - async def _arun_once(*args: Any, **kwargs: Any) -> RunOutput: + def arun(*args: Any, **kwargs: Any) -> Any: + async def stream() -> Any: if model_turn is not None: await model_turn() - return resp - - agent.arun = AsyncMock(side_effect=_arun_once) - else: - # Streaming path (Emit.TOOL_CALLS on): the adapter iterates - # `agent.arun(stream=True, ...)`, which yields the run events then the - # final RunOutput. A bare MagicMock returns an async iterator without - # awaiting, matching how the adapter consumes the stream. - def _arun(*args: Any, **kwargs: Any) -> Any: - async def _stream() -> Any: - if model_turn is not None: - await model_turn() - for event in events: - yield event - yield resp - - return _stream() - - agent.arun = MagicMock(side_effect=_arun) + for event in events or []: + yield event + yield resp + + return stream() + + agent.arun = MagicMock(side_effect=arun) return agent @@ -156,6 +143,9 @@ async def aresponse(self, messages: list[Message], **kwargs: Any) -> ModelRespon await send.entrypoint(content=self._content, mentions=["@Alice"]) return ModelResponse(content=self._content) + async def aresponse_stream(self, messages: list[Message], **kwargs: Any) -> Any: + yield await self.aresponse(messages, **kwargs) + @property def captured_system_prompt(self) -> str: """The concatenated system message(s) Agno sent to the model.""" @@ -256,7 +246,7 @@ def make_agent_input( def run_input(copy: MagicMock) -> list[Message]: """The exact list[Message] the faked Agno agent received via arun(input=...).""" - return copy.arun.await_args.kwargs["input"] + return copy.arun.call_args.kwargs["input"] def platform_msg( diff --git a/tests/adapters/agno/rehydration/__init__.py b/tests/adapters/agno/rehydration/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/adapters/agno/rehydration/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/agno/rehydration/test_multiturncarryover.py b/tests/adapters/agno/rehydration/test_multiturncarryover.py new file mode 100644 index 000000000..5abfd70af --- /dev/null +++ b/tests/adapters/agno/rehydration/test_multiturncarryover.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from agno.models.message import Message +from agno.run.agent import RunOutput + +from tests.adapters.agno.helpers import make_agent_input, run_input + + +class TestMultiTurnCarryover: + async def test_persisted_transcript_feeds_the_next_turn( + self, make_started_adapter, sample_platform_message, tools + ): + # Turn 1's run produces a transcript; _persist_turn keeps it and the next + # turn must build on top of it (carryover through the real on_message path). + turn = RunOutput( + content="a1", + messages=[ + Message(role="user", content="[Alice]: q1"), + Message(role="assistant", content="a1"), + ], + ) + adapter, agent = await make_started_adapter(turn, replies_through=tools) + + await adapter.on_event( + make_agent_input( + sample_platform_message, [], is_session_bootstrap=True, tools=tools + ) + ) + await adapter.on_event( + make_agent_input( + sample_platform_message, [], is_session_bootstrap=False, tools=tools + ) + ) + + msgs = run_input(agent) # the second (follow-up) turn's input + assert [m.content for m in msgs[:2]] == ["[Alice]: q1", "a1"] + assert msgs[-1].content == sample_platform_message.format_for_llm() + assert len(msgs) == 3 diff --git a/tests/adapters/agno/test_rehydration.py b/tests/adapters/agno/rehydration/test_rehydrationpipeline.py similarity index 50% rename from tests/adapters/agno/test_rehydration.py rename to tests/adapters/agno/rehydration/test_rehydrationpipeline.py index bc7735021..8a2e04ba8 100644 --- a/tests/adapters/agno/test_rehydration.py +++ b/tests/adapters/agno/rehydration/test_rehydrationpipeline.py @@ -1,16 +1,5 @@ -"""Agno history/context rehydration tests. - -These drive the adapter through the real ``on_event`` path so the real -``AgnoHistoryConverter`` runs, then inspect the exact ``list[Message]`` Agno -received via the faked ``agent.arun(input=...)``. Assertions are on real Agno -``Message`` objects (roles, ``tool_calls``, ``tool_call_id``, ``from_history``), -never on hardcoded prose. History is built with the real runtime formatter -``format_history_for_llm`` rather than hand-rolled converter-ready dicts. -""" - from __future__ import annotations -from agno.models.message import Message from agno.run.agent import RunOutput from band.runtime.formatters import format_history_for_llm @@ -153,124 +142,3 @@ async def test_participants_and_contacts_injected_before_current_message( "[System]: Carol is now a contact", sample_platform_message.format_for_llm(), ] - - -class TestUnansweredMessage: - async def test_current_message_excluded_from_history_then_answered( - self, make_started_adapter, sample_platform_message, tools - ): - current = sample_platform_message - # The platform context includes the current message; the formatter must - # exclude it so it is answered, not replayed as context. - raw = format_history_for_llm( - [ - platform_msg("h1", "previous", sender_name="Alice"), - {**platform_msg(current.id, current.content), "id": current.id}, - ], - exclude_id=current.id, - ) - assert len(raw) == 1 - assert all(current.content not in h["content"] for h in raw) - - adapter, agent = await make_started_adapter( - RunOutput(content="here is your answer"), replies_through=tools - ) - - await adapter.on_event( - make_agent_input(current, raw, is_session_bootstrap=True, tools=tools) - ) - - msgs = run_input(agent) - formatted = current.format_for_llm() - assert sum(1 for m in msgs if m.content == formatted) == 1 - assert msgs[-1].content == formatted - - async def test_answers_unanswered_message_on_restart_bootstrap( - self, make_started_adapter, sample_platform_message, tools - ): - # Agent restarts: first event is bootstrap, with a completed exchange in - # history and a brand-new unanswered question as the current message. - raw = format_history_for_llm( - [ - platform_msg("h1", "Earlier question", sender_name="Alice"), - platform_msg( - "h2", "Earlier answer", sender_type="Agent", sender_name="TestBot" - ), - ], - exclude_id=sample_platform_message.id, - ) - adapter, agent = await make_started_adapter( - RunOutput(content="fresh answer"), replies_through=tools - ) - - await adapter.on_event( - make_agent_input( - sample_platform_message, raw, is_session_bootstrap=True, tools=tools - ) - ) - - agent.arun.assert_awaited_once() - assert run_input(agent)[-1].content == sample_platform_message.format_for_llm() - - async def test_trailing_unanswered_user_turns_are_preserved( - self, make_started_adapter, sample_platform_message, tools - ): - # Several user turns with no assistant reply between them: agno keeps them - # all as user messages (it does not require complete exchanges). - raw = format_history_for_llm( - [ - platform_msg("h1", "first", sender_name="Alice"), - platform_msg("h2", "second", sender_name="Bob"), - platform_msg("h3", "third", sender_name="Alice"), - ], - exclude_id=sample_platform_message.id, - ) - adapter, agent = await make_started_adapter( - RunOutput(content="answering all"), replies_through=tools - ) - - await adapter.on_event( - make_agent_input( - sample_platform_message, raw, is_session_bootstrap=True, tools=tools - ) - ) - - msgs = run_input(agent) - assert [m.role for m in msgs] == ["user", "user", "user", "user"] - assert [m.content for m in msgs[:3]] == [ - "[Alice]: first", - "[Bob]: second", - "[Alice]: third", - ] - - -class TestMultiTurnCarryover: - async def test_persisted_transcript_feeds_the_next_turn( - self, make_started_adapter, sample_platform_message, tools - ): - # Turn 1's run produces a transcript; _persist_turn keeps it and the next - # turn must build on top of it (carryover through the real on_message path). - turn = RunOutput( - content="a1", - messages=[ - Message(role="user", content="[Alice]: q1"), - Message(role="assistant", content="a1"), - ], - ) - adapter, agent = await make_started_adapter(turn, replies_through=tools) - - await adapter.on_event( - make_agent_input( - sample_platform_message, [], is_session_bootstrap=True, tools=tools - ) - ) - await adapter.on_event( - make_agent_input( - sample_platform_message, [], is_session_bootstrap=False, tools=tools - ) - ) - - msgs = run_input(agent) # the second (follow-up) turn's input - assert [m.content for m in msgs[:2]] == ["[Alice]: q1", "a1"] - assert msgs[-1].content == sample_platform_message.format_for_llm() - assert len(msgs) == 3 diff --git a/tests/adapters/agno/rehydration/test_unansweredmessage.py b/tests/adapters/agno/rehydration/test_unansweredmessage.py new file mode 100644 index 000000000..1072ca746 --- /dev/null +++ b/tests/adapters/agno/rehydration/test_unansweredmessage.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from agno.run.agent import RunOutput + +from band.runtime.formatters import format_history_for_llm +from tests.adapters.agno.helpers import make_agent_input, platform_msg, run_input + + +class TestUnansweredMessage: + async def test_current_message_excluded_from_history_then_answered( + self, make_started_adapter, sample_platform_message, tools + ): + current = sample_platform_message + # The platform context includes the current message; the formatter must + # exclude it so it is answered, not replayed as context. + raw = format_history_for_llm( + [ + platform_msg("h1", "previous", sender_name="Alice"), + {**platform_msg(current.id, current.content), "id": current.id}, + ], + exclude_id=current.id, + ) + assert len(raw) == 1 + assert all(current.content not in h["content"] for h in raw) + + adapter, agent = await make_started_adapter( + RunOutput(content="here is your answer"), replies_through=tools + ) + + await adapter.on_event( + make_agent_input(current, raw, is_session_bootstrap=True, tools=tools) + ) + + msgs = run_input(agent) + formatted = current.format_for_llm() + assert sum(1 for m in msgs if m.content == formatted) == 1 + assert msgs[-1].content == formatted + + async def test_answers_unanswered_message_on_restart_bootstrap( + self, make_started_adapter, sample_platform_message, tools + ): + # Agent restarts: first event is bootstrap, with a completed exchange in + # history and a brand-new unanswered question as the current message. + raw = format_history_for_llm( + [ + platform_msg("h1", "Earlier question", sender_name="Alice"), + platform_msg( + "h2", "Earlier answer", sender_type="Agent", sender_name="TestBot" + ), + ], + exclude_id=sample_platform_message.id, + ) + adapter, agent = await make_started_adapter( + RunOutput(content="fresh answer"), replies_through=tools + ) + + await adapter.on_event( + make_agent_input( + sample_platform_message, raw, is_session_bootstrap=True, tools=tools + ) + ) + + agent.arun.assert_called_once() + assert run_input(agent)[-1].content == sample_platform_message.format_for_llm() + + async def test_trailing_unanswered_user_turns_are_preserved( + self, make_started_adapter, sample_platform_message, tools + ): + # Several user turns with no assistant reply between them: agno keeps them + # all as user messages (it does not require complete exchanges). + raw = format_history_for_llm( + [ + platform_msg("h1", "first", sender_name="Alice"), + platform_msg("h2", "second", sender_name="Bob"), + platform_msg("h3", "third", sender_name="Alice"), + ], + exclude_id=sample_platform_message.id, + ) + adapter, agent = await make_started_adapter( + RunOutput(content="answering all"), replies_through=tools + ) + + await adapter.on_event( + make_agent_input( + sample_platform_message, raw, is_session_bootstrap=True, tools=tools + ) + ) + + msgs = run_input(agent) + assert [m.role for m in msgs] == ["user", "user", "user", "user"] + assert [m.content for m in msgs[:3]] == [ + "[Alice]: first", + "[Bob]: second", + "[Alice]: third", + ] diff --git a/tests/adapters/agno/test_adapter.py b/tests/adapters/agno/test_adapter.py deleted file mode 100644 index 09b5b6c1e..000000000 --- a/tests/adapters/agno/test_adapter.py +++ /dev/null @@ -1,1020 +0,0 @@ -"""Agno adapter behavior tests. - -Conformance already covers init defaults, ``on_started`` name/description, and -generic converter wiring; these tests pin Agno-only behavior: running against -the given agent, memory-collision warning, per-run Band-tool resolution (the -callable-tools factory + ContextVar binding), strict per-room tool visibility, -fallback-send, emit reporting, transcript persistence, and cleanup. Rehydration -of platform history lives in ``test_rehydration.py``. -""" - -from __future__ import annotations - -import json -import warnings -from datetime import UTC, datetime -from typing import Any, ClassVar -from unittest.mock import AsyncMock - -import pytest -from agno.agent import Agent as AgnoAgent -from agno.models.message import Message -from agno.run import RunStatus -from agno.run.agent import RunErrorEvent, RunOutput -from agno.tools.function import ToolResult - -from band.adapters.agno import ( - AgnoAdapter, - AgnoRunError, - _bind_room_tools, - _make_band_entrypoint, -) -from band.core.protocols import GENERIC_PROVIDER_FAILURE_MESSAGE -from band.core.types import Capability, Emit, PlatformMessage -from band.testing import FakeAgentTools, reported_failures -from tests.adapters.agno.helpers import ( - CapturingModel, - ContactAwareTools, - SchemaTools, - openai_tool_schema, - run_input, - tool_events, - tool_execution, -) - - -def _msg( - room_id: str, - content: str, - *, - msg_id: str = "m1", - sender_id: str = "user-1", -) -> PlatformMessage: - """A minimal PlatformMessage for driving on_message in a given room.""" - return PlatformMessage( - id=msg_id, - room_id=room_id, - content=content, - sender_id=sender_id, - sender_type="User", - sender_name="Alice", - message_type="text", - metadata={}, - created_at=datetime.now(UTC), - ) - - -class TestOnStarted: - async def test_runs_against_the_given_agent(self, make_agno_agent): - agent = make_agno_agent() - adapter = AgnoAdapter(agent=agent) - - await adapter.on_started("TestBot", "desc") - - # The adapter uses the caller's instance directly, no copy. - assert adapter.agent is agent - - async def test_syncs_converter_identity(self, make_started_adapter): - adapter, _ = await make_started_adapter() - - assert adapter.history_converter._agent_name == "TestBot" - - async def test_runs_the_given_agent_on_a_message(self, make_agno_agent, tools): - # End-to-end: the given agent must be the one actually run on a message. - # The adapter delivers nothing on its own; plain agent text is not sent - # (only a ``band_send_message`` tool call reaches the room). - agent = make_agno_agent(response=RunOutput(content="hi there")) - adapter = AgnoAdapter(agent=agent, emit=()) - await adapter.on_started("TestBot", "desc") - - await adapter.on_message( - _msg("room-1", "hello"), - tools, - [], - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - agent.arun.assert_awaited_once() - tools.assert_no_messages_sent() - - -class TestMemoryCollisionWarning: - """Collision is detected against the runtime agent at startup, not __init__.""" - - async def test_warns_on_update_memory_on_run_with_memory_capability( - self, make_agno_agent - ): - agent = make_agno_agent(update_memory_on_run=True) - adapter = AgnoAdapter(agent=agent, capabilities=Capability.MEMORY) - - with pytest.warns(UserWarning, match="update_memory_on_run"): - await adapter.on_started("TestBot", "desc") - - async def test_warns_on_agentic_memory_with_memory_capability( - self, make_agno_agent - ): - agent = make_agno_agent(enable_agentic_memory=True) - adapter = AgnoAdapter(agent=agent, capabilities=Capability.MEMORY) - - with pytest.warns(UserWarning, match="enable_agentic_memory"): - await adapter.on_started("TestBot", "desc") - - async def test_no_warning_without_memory_capability(self, make_agno_agent): - agent = make_agno_agent(update_memory_on_run=True, enable_agentic_memory=True) - adapter = AgnoAdapter(agent=agent) # no MEMORY capability -> no collision - - with warnings.catch_warnings(): - warnings.simplefilter("error") - await adapter.on_started("TestBot", "desc") - - -class TestRoomToolResolution: - """Band tools are exposed per-run via the ``_resolve_room_tools`` factory - Agno calls each turn, not wired onto the agent. These pin what that factory - returns and the schema requests it makes for the active room.""" - - async def test_resolves_band_tools_for_active_room(self, make_started_adapter): - tools = SchemaTools( - [ - openai_tool_schema("band_send_message"), - openai_tool_schema("band_lookup_peers"), - ] - ) - adapter, _ = await make_started_adapter() - - with _bind_room_tools(tools): - resolved = await adapter._resolve_room_tools() - - assert [fn.name for fn in resolved] == [ - "band_send_message", - "band_lookup_peers", - ] - - async def test_no_band_tools_outside_a_bound_room(self, make_started_adapter): - # Defensive: with no active room bound, the factory exposes no Band tools - # (and does not even request schemas) rather than guessing visibility. - tools = SchemaTools([openai_tool_schema("band_send_message")]) - adapter, _ = await make_started_adapter() - - resolved = await adapter._resolve_room_tools() # no _bind_room_tools - - assert resolved == [] - assert tools.schema_calls == [] - - async def test_capability_flags_drive_schema_request(self, make_started_adapter): - tools = SchemaTools([]) - adapter, _ = await make_started_adapter( - capabilities={Capability.MEMORY, Capability.CONTACTS} - ) - - with _bind_room_tools(tools): - await adapter._resolve_room_tools() - - assert tools.schema_calls == [ - {"include_memory": True, "include_contacts": True} - ] - - async def test_schema_build_is_cached_across_runs(self, make_started_adapter): - # Same contact flag across runs -> schemas are built once and reused, - # not rebuilt every turn. - tools = SchemaTools([openai_tool_schema("band_send_message")]) - adapter, _ = await make_started_adapter() - - with _bind_room_tools(tools): - await adapter._resolve_room_tools() - await adapter._resolve_room_tools() - await adapter._resolve_room_tools() - - assert tools.schema_calls == [ - {"include_memory": False, "include_contacts": False} - ] - - async def test_user_tools_are_reincluded(self, make_agno_agent): - # Replacing agent.tools with our factory must not drop the user's own - # tools; they are re-included alongside the room's Band tools. - user_tool = object() - agent = make_agno_agent() - agent.tools = [user_tool] - adapter = AgnoAdapter(agent=agent) - await adapter.on_started("TestBot", "desc") - - tools = SchemaTools([openai_tool_schema("band_send_message")]) - with _bind_room_tools(tools): - resolved = await adapter._resolve_room_tools() - - assert resolved[0] is user_tool - assert [getattr(t, "name", None) for t in resolved[1:]] == ["band_send_message"] - - -class TestBandInstructionInjection: - """Drive a real Agno agent so we assert on the system prompt Agno actually - assembled and sent to the model, not the attribute the adapter set.""" - - @pytest.mark.parametrize( - ("capabilities", "present", "absent"), - [ - (set(), [], ["## Memory Tools", "## Contact Management Tools"]), - ( - {Capability.MEMORY}, - ["## Memory Tools"], - ["## Contact Management Tools"], - ), - ( - {Capability.CONTACTS}, - ["## Contact Management Tools"], - ["## Memory Tools"], - ), - ], - ) - async def test_capability_sections_gated_in_model_prompt( - self, run_real_agent, sample_platform_message, capabilities, present, absent - ): - model = await run_real_agent( - sample_platform_message, - capabilities=capabilities, - ) - prompt = model.captured_system_prompt - - assert "## Environment" in prompt # base guidance always injected - assert all(section in prompt for section in present) - assert all(section not in prompt for section in absent) - - async def test_developer_instructions_survive_in_prompt( - self, run_real_agent, sample_platform_message - ): - model = await run_real_agent( - sample_platform_message, - instructions="You are Dev, a niche specialist.", - additional_context="Keep replies under 10 words.", - ) - prompt = model.captured_system_prompt - - assert "You are Dev, a niche specialist." in prompt - assert "Keep replies under 10 words." in prompt - assert "## Environment" in prompt - - async def test_guidance_injected_at_startup_before_any_message( - self, make_started_adapter - ): - # Band guidance is injected in on_started, not lazily on first message. - _adapter, agent = await make_started_adapter() - - # The Band operating contract frames the agent via ``description`` (ahead of - # the developer's own ``instructions``), not via ``additional_context``. - assert isinstance(agent.description, str) - assert "## Environment" in agent.description - # Band-registered identity is injected so the model knows who it is. - assert "You are TestBot, desc." in agent.description - - -class TestBandEntrypointBinding: - async def test_routes_to_execute_tool_call_inside_context(self, tools): - entry = _make_band_entrypoint("band_lookup_peers") - - with _bind_room_tools(tools): - result = await entry(page=1) - - assert tools.tool_calls == [ - {"tool_name": "band_lookup_peers", "arguments": {"page": 1}} - ] - assert json.loads(result) == {"status": "ok"} - - async def test_passes_string_results_through_unchanged(self): - class _StrTools(FakeAgentTools): - async def execute_tool_call(self, tool_name: str, arguments: dict) -> Any: - return "raw-string" - - entry = _make_band_entrypoint("band_lookup_peers") - with _bind_room_tools(_StrTools()): - assert await entry() == "raw-string" - - async def test_errors_outside_any_bound_context(self, tools): - entry = _make_band_entrypoint("band_lookup_peers") - - # Bind then exit; the ContextVar must reset so later calls have no tools. - with _bind_room_tools(tools): - pass - result = await entry(page=1) - - assert "no active Band context" in result - assert tools.tool_calls == [] - - async def test_read_room_file_image_result_passes_through_as_agno_image(self): - class _ImageTools(FakeAgentTools): - async def execute_tool_call(self, tool_name: str, arguments: dict) -> Any: - return { - "content": [ - {"type": "image", "data": "ZmFrZQ==", "mimeType": "image/png"} - ] - } - - entry = _make_band_entrypoint("band_read_room_file") - with _bind_room_tools(_ImageTools()): - result = await entry(file_id="f1") - - assert isinstance(result, ToolResult) - assert result.images is not None - assert len(result.images) == 1 - assert result.images[0].content == b"fake" - assert result.images[0].mime_type == "image/png" - - async def test_read_room_file_non_image_result_stays_json(self): - class _TextTools(FakeAgentTools): - async def execute_tool_call(self, tool_name: str, arguments: dict) -> Any: - return {"name": "notes.txt", "content_type": "text/plain"} - - entry = _make_band_entrypoint("band_read_room_file") - with _bind_room_tools(_TextTools()): - result = await entry(file_id="f1") - - assert result == '{"name": "notes.txt", "content_type": "text/plain"}' - - async def test_read_room_file_malformed_image_data_returns_error_string(self): - """A decode_image_block failure (invalid base64) must degrade to the - adapter's usual error string, not raise uncaught out of the tool - entrypoint Agno invokes directly.""" - - class _MalformedImageTools(FakeAgentTools): - async def execute_tool_call(self, tool_name: str, arguments: dict) -> Any: - return { - "content": [{"type": "image", "data": "A", "mimeType": "image/png"}] - } - - entry = _make_band_entrypoint("band_read_room_file") - with _bind_room_tools(_MalformedImageTools()): - result = await entry(file_id="f1") - - assert isinstance(result, str) - assert result.startswith("Error reading room file:") - - -class TestReply: - """The adapter delivers nothing on its own. Like the other adapters, the - agent must call ``band_send_message`` to reach the room; plain agent text is - never auto-sent and the adapter never guesses a recipient.""" - - async def test_no_send_when_agent_returns_only_text( - self, make_started_adapter, sample_platform_message, tools - ): - adapter, _ = await make_started_adapter(RunOutput(content="hello")) - - await adapter.on_message( - sample_platform_message, - tools, - [], - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - tools.assert_no_messages_sent() - - async def test_no_send_when_agent_called_band_send_message( - self, make_started_adapter, sample_platform_message, tools - ): - # The tool call itself reaches the room; the adapter adds nothing on top. - response = RunOutput( - content="hello", tools=[tool_execution("band_send_message")] - ) - adapter, _ = await make_started_adapter(response) - - await adapter.on_message( - sample_platform_message, - tools, - [], - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - tools.assert_no_messages_sent() - - async def test_no_send_for_empty_content( - self, make_started_adapter, sample_platform_message, tools - ): - adapter, _ = await make_started_adapter(RunOutput(content=" ")) - - await adapter.on_message( - sample_platform_message, - tools, - [], - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - tools.assert_no_messages_sent() - - -class TestEmitExecution: - async def test_emits_tool_call_and_result_events( - self, make_started_adapter, sample_platform_message, tools - ): - # Streamed run: started + completed events for one tool call. Reporting - # is live (during the run), driven off Agno's native stream events. - events = tool_events( - tool_execution("band_lookup_peers", args={"page": "1"}, result="ok") - ) - adapter, _ = await make_started_adapter(emit=Emit.TOOL_CALLS, events=events) - - await adapter.on_message( - sample_platform_message, - tools, - [], - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - types = [e["message_type"] for e in tools.events_sent] - assert types == ["tool_call", "tool_result"] - call_payload = json.loads(tools.events_sent[0]["content"]) - result_payload = json.loads(tools.events_sent[1]["content"]) - assert call_payload == { - "name": "band_lookup_peers", - "args": {"page": "1"}, - "tool_call_id": "tc_1", - } - assert result_payload["output"] == "ok" - assert result_payload["is_error"] is False - - async def test_band_send_message_tool_call_is_reported( - self, make_started_adapter, sample_platform_message, tools - ): - """band_send_message is reported like any other tool call — no suppression.""" - events = tool_events(tool_execution("band_send_message")) - adapter, _ = await make_started_adapter(emit=Emit.TOOL_CALLS, events=events) - - await adapter.on_message( - sample_platform_message, - tools, - [], - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - types = [e["message_type"] for e in tools.events_sent] - assert types == ["tool_call", "tool_result"] - - async def test_send_room_file_tool_call_event_redacts_content( - self, make_started_adapter, sample_platform_message, tools - ): - """band_send_room_file's tool_call event must report a bounded - placeholder for content, not the raw file bytes -- generic ARGS - reporting has no idea this one tool's content argument can carry up - to MAX_SEND_CONTENT_BYTES of real file data.""" - events = tool_events( - tool_execution( - "band_send_room_file", - args={"content": "raw file bytes", "filename": "notes.txt"}, - result="ok", - ) - ) - adapter, _ = await make_started_adapter(emit=Emit.TOOL_CALLS, events=events) - - await adapter.on_message( - sample_platform_message, - tools, - [], - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - call_payload = json.loads(tools.events_sent[0]["content"]) - assert call_payload["args"]["content"] == "<14 byte file content>" - assert call_payload["args"]["filename"] == "notes.txt" - - async def test_no_events_without_execution_emit( - self, make_started_adapter, sample_platform_message, tools - ): - # emit=() -> non-streaming run, no live reporting. The final - # RunOutput still carries executions, but nothing is emitted. - response = RunOutput(tools=[tool_execution("band_lookup_peers")]) - adapter, agent = await make_started_adapter(response, emit=()) - - await adapter.on_message( - sample_platform_message, - tools, - [], - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - assert tools.events_sent == [] - # Confirms the non-streaming path: a single awaited arun, not a stream. - agent.arun.assert_awaited_once() - - -class TestEmitThoughts: - async def test_emits_reasoning_as_thought( - self, make_started_adapter, sample_platform_message, tools - ): - response = RunOutput(reasoning_content="thinking hard") - adapter, _ = await make_started_adapter(response, emit=Emit.THOUGHTS) - - await adapter.on_message( - sample_platform_message, - tools, - [], - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - tools.assert_event_sent(message_type="thought") - assert tools.events_sent[0]["content"] == "thinking hard" - - async def test_no_thought_without_thoughts_emit( - self, make_started_adapter, sample_platform_message, tools - ): - response = RunOutput(reasoning_content="thinking hard") - adapter, _ = await make_started_adapter(response, emit=()) - - await adapter.on_message( - sample_platform_message, - tools, - [], - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - assert tools.events_sent == [] - - async def test_no_thought_for_blank_reasoning( - self, make_started_adapter, sample_platform_message, tools - ): - adapter, _ = await make_started_adapter( - RunOutput(reasoning_content=" "), - emit=Emit.THOUGHTS, - ) - - await adapter.on_message( - sample_platform_message, - tools, - [], - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - assert tools.events_sent == [] - - -class TestPersistAndAccumulate: - def test_persist_keeps_only_conversation_roles(self, make_agno_agent): - agent = make_agno_agent() - adapter = AgnoAdapter(agent=agent) - response = RunOutput( - messages=[ - Message(role="system", content="instructions"), - Message(role="user", content="hi"), - Message(role="assistant", content="hello"), - Message(role="developer", content="state"), - Message(role="tool", content="result"), - ] - ) - - adapter._persist_turn("room-1", response) - - kept = [m.role for m in adapter._message_history["room-1"]] - assert kept == ["user", "assistant", "tool"] - - def test_bootstrap_seeds_committed_transcript_from_history( - self, make_agno_agent, sample_platform_message - ): - # Bootstrap seeds the committed transcript from rehydrated history. The - # returned run input is that seed plus this turn's live message, but - # building it must NOT push the live message into the committed store. - agent = make_agno_agent() - adapter = AgnoAdapter(agent=agent) - seed = [Message(role="user", content="earlier")] - - run_input_msgs = adapter._build_run_input( - sample_platform_message, - seed, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - assert [m.content for m in run_input_msgs] == [ - "earlier", - sample_platform_message.format_for_llm(), - ] - # Committed transcript holds only the rehydrated seed. - assert [m.content for m in adapter._message_history["room-1"]] == ["earlier"] - - def test_build_run_input_does_not_mutate_committed_transcript( - self, make_agno_agent, sample_platform_message - ): - # A non-bootstrap turn reads the committed transcript but never writes to - # it; the store is only ever advanced by _persist_turn after a run. - agent = make_agno_agent() - adapter = AgnoAdapter(agent=agent) - adapter._message_history["room-1"] = [Message(role="user", content="committed")] - - adapter._build_run_input( - sample_platform_message, - [], - "participants", - "contacts", - is_session_bootstrap=False, - room_id="room-1", - ) - - assert [m.content for m in adapter._message_history["room-1"]] == ["committed"] - - -class TestFailedRunDoesNotContaminateNextTurn: - async def test_failed_turn_leaves_no_residue_in_next_run_input( - self, make_agno_agent, tools - ): - # Turn 1 raises mid-run; turn 2 succeeds. The injected system/user - # messages from the failed turn must not survive into turn 2's input. - agent = make_agno_agent() - agent.arun = AsyncMock( - side_effect=[RuntimeError("boom"), RunOutput(content="ok")] - ) - adapter = AgnoAdapter(agent=agent, emit=()) - await adapter.on_started("TestBot", "desc") - - first = _msg("room-1", "first question", msg_id="m1") - with pytest.raises(RuntimeError): - await adapter.on_message( - first, - tools, - [], - "P1-participants", - "C1-contacts", - is_session_bootstrap=True, - room_id="room-1", - ) - - second = _msg("room-1", "second question", msg_id="m2") - await adapter.on_message( - second, - tools, - [], - "P2-participants", - "C2-contacts", - is_session_bootstrap=False, - room_id="room-1", - ) - - contents = [m.content for m in run_input(agent)] - # No residue from the failed turn 1. - assert not any("P1-participants" in c for c in contents) - assert not any("C1-contacts" in c for c in contents) - assert first.format_for_llm() not in contents - # Turn 2's own injected context and live message are present. - assert any("P2-participants" in c for c in contents) - assert contents[-1] == second.format_for_llm() - - -class TestOnCleanup: - async def test_drops_room_transcript(self, make_agno_agent): - agent = make_agno_agent() - adapter = AgnoAdapter(agent=agent) - adapter._message_history["room-1"] = [Message(role="user", content="hi")] - - await adapter.on_cleanup("room-1") - - assert "room-1" not in adapter._message_history - - async def test_unknown_room_is_noop(self, make_agno_agent): - agent = make_agno_agent() - adapter = AgnoAdapter(agent=agent) - - await adapter.on_cleanup("never-seen") # must not raise - - -class TestUsedBeforeStarted: - async def test_run_agent_before_on_started_raises(self, make_agno_agent): - agent = make_agno_agent() - adapter = AgnoAdapter(agent=agent) - - with pytest.raises(RuntimeError, match="before on_started"): - await adapter._run_agent( - [], FakeAgentTools(), room_id="room-1", msg_id="m1" - ) - - -class TestSessionIsolation: - async def test_arun_uses_room_id_as_session_id(self, make_started_adapter): - adapter, agent = await make_started_adapter() - - await adapter.on_message( - _msg("room-A", "hi"), - FakeAgentTools(), - [], - None, - None, - is_session_bootstrap=True, - room_id="room-A", - ) - - assert agent.arun.await_args.kwargs["session_id"] == "room-A" - - async def test_custom_session_id_factory_is_used(self, make_agno_agent): - agent = make_agno_agent() - adapter = AgnoAdapter( - agent=agent, session_id_factory=lambda room: f"sess::{room}", emit=() - ) - await adapter.on_started("TestBot", "desc") - - await adapter.on_message( - _msg("room-A", "hi"), - FakeAgentTools(), - [], - None, - None, - is_session_bootstrap=True, - room_id="room-A", - ) - - assert agent.arun.await_args.kwargs["session_id"] == "sess::room-A" - - async def test_two_rooms_get_isolated_sessions_and_inputs( - self, make_started_adapter - ): - adapter, agent = await make_started_adapter() - - await adapter.on_message( - _msg("room-A", "alpha-secret"), - FakeAgentTools(), - [], - None, - None, - is_session_bootstrap=True, - room_id="room-A", - ) - await adapter.on_message( - _msg("room-B", "beta-secret"), - FakeAgentTools(), - [], - None, - None, - is_session_bootstrap=True, - room_id="room-B", - ) - - calls = agent.arun.await_args_list - assert calls[0].kwargs["session_id"] == "room-A" - assert calls[1].kwargs["session_id"] == "room-B" - - room_b_input = " ".join(m.content or "" for m in calls[1].kwargs["input"]) - assert "beta-secret" in room_b_input - assert "alpha-secret" not in room_b_input - - -class TestHubContactExposure: - """The adapter decides contact exposure (mirrors LangGraph): the CONTACTS - capability OR a hub room force-includes contact tool schemas, resolved per - run so visibility is strictly per-room.""" - - async def test_normal_room_does_not_request_contacts(self, make_started_adapter): - adapter, _ = await make_started_adapter() - tools = SchemaTools([], room_id="room-A") - - with _bind_room_tools(tools): - await adapter._resolve_room_tools() - - assert tools.schema_calls == [ - {"include_memory": False, "include_contacts": False} - ] - - async def test_hub_room_forces_contacts(self, make_started_adapter): - adapter, _ = await make_started_adapter() - tools = SchemaTools([], hub_room_id="hub", room_id="hub") - - with _bind_room_tools(tools): - await adapter._resolve_room_tools() - - assert tools.schema_calls == [ - {"include_memory": False, "include_contacts": True} - ] - - async def test_contacts_do_not_leak_into_normal_room_after_hub( - self, make_started_adapter - ): - # Core regression: after a hub room exposes contact tools, a subsequent - # normal room's resolution must NOT include them. The old additive wiring - # accumulated the union on the shared agent; per-run resolution does not. - adapter, _ = await make_started_adapter() - - hub = ContactAwareTools(hub_room_id="hub", room_id="hub") - with _bind_room_tools(hub): - hub_names = [fn.name for fn in await adapter._resolve_room_tools()] - assert "band_add_contact" in hub_names - - normal = ContactAwareTools(room_id="room-A") - with _bind_room_tools(normal): - normal_names = [fn.name for fn in await adapter._resolve_room_tools()] - - assert normal_names == ["band_send_message"] - assert "band_add_contact" not in normal_names - - -class TestFeatureFilters: - """include_tools/exclude_tools/include_categories gate which Band tools - are wired (parity with LangGraph).""" - - ALL_SCHEMAS: ClassVar[list[dict[str, Any]]] = [ - openai_tool_schema("band_send_message"), # chat - openai_tool_schema("band_lookup_peers"), # chat - openai_tool_schema("band_store_memory"), # memory - openai_tool_schema("band_add_contact"), # contacts - ] - - async def _resolved_names(self, adapter) -> list[str]: - tools = SchemaTools(self.ALL_SCHEMAS) - with _bind_room_tools(tools): - resolved = await adapter._resolve_room_tools() - return [fn.name for fn in resolved] - - async def test_include_tools_keeps_only_named(self, make_started_adapter): - adapter, _ = await make_started_adapter(include_tools=["band_send_message"]) - - assert await self._resolved_names(adapter) == ["band_send_message"] - - async def test_exclude_tools_drops_named(self, make_started_adapter): - adapter, _ = await make_started_adapter(exclude_tools=["band_send_message"]) - - names = await self._resolved_names(adapter) - assert "band_send_message" not in names - assert "band_lookup_peers" in names - - async def test_include_categories_keeps_only_category(self, make_started_adapter): - adapter, _ = await make_started_adapter(include_categories=["chat"]) - - assert sorted(await self._resolved_names(adapter)) == [ - "band_lookup_peers", - "band_send_message", - ] - - -class TestRunFailureReporting: - async def test_emits_generic_error_event_and_reraises( - self, make_started_adapter, tools - ): - adapter, agent = await make_started_adapter() - agent.arun.side_effect = RuntimeError("db dsn leaked: secret-token") - - with pytest.raises(RuntimeError): - await adapter.on_message( - _msg("room-A", "hi"), - tools, - [], - None, - None, - is_session_bootstrap=True, - room_id="room-A", - ) - - failures = reported_failures(tools) - assert len(failures) == 1 - assert failures[0]["message"] == GENERIC_PROVIDER_FAILURE_MESSAGE - # The exception text (which can carry secrets) must not leak to the room. - assert "secret-token" not in failures[0]["message"] - assert failures[0]["provider"] == "agno" - # A plain RuntimeError isn't a swallowed Agno run status -- no code. - assert failures[0]["code"] is None - - async def test_error_status_run_is_raised_and_reported( - self, make_started_adapter, tools - ): - # Agno swallows run exceptions (model/API failures included) and - # returns a normal-looking RunOutput with status=error. The adapter - # must surface it as a failure — otherwise the runtime marks the - # message processed and the turn dies silently with no reply. - adapter, _agent = await make_started_adapter( - response=RunOutput( - content="api dsn leaked: secret-token", status=RunStatus.error - ) - ) - - with pytest.raises(AgnoRunError): - await adapter.on_message( - _msg("room-A", "hi"), - tools, - [], - None, - None, - is_session_bootstrap=True, - room_id="room-A", - ) - - failures = reported_failures(tools) - assert len(failures) == 1 - assert "secret-token" not in failures[0]["message"] - assert failures[0]["code"] == RunStatus.error.value - # A failed turn must not be committed to the room transcript. - assert not adapter._message_history.get("room-A") - - async def test_streaming_error_event_is_raised_and_reported( - self, make_started_adapter, tools - ): - # In the streaming path a swallowed error surfaces only as a - # RunErrorEvent — the stream never yields a final RunOutput. The - # adapter must raise on it instead of returning None (a "successful" - # empty turn). - adapter, _agent = await make_started_adapter( - emit=Emit.TOOL_CALLS, - events=[RunErrorEvent(content="api dsn leaked: secret-token")], - ) - - with pytest.raises(AgnoRunError): - await adapter.on_message( - _msg("room-A", "hi"), - tools, - [], - None, - None, - is_session_bootstrap=True, - room_id="room-A", - ) - - failures = reported_failures(tools) - assert len(failures) == 1 - assert "secret-token" not in failures[0]["message"] - assert failures[0]["code"] == RunStatus.error.value - assert not adapter._message_history.get("room-A") - - async def test_error_event_failure_does_not_mask_original( - self, make_started_adapter - ): - adapter, agent = await make_started_adapter() - agent.arun.side_effect = RuntimeError("boom") - - class _FailingEventTools(FakeAgentTools): - async def send_event(self, *args: Any, **kwargs: Any) -> dict[str, Any]: - raise RuntimeError("event transport down") - - # The failed error-report must not replace the original exception. - with pytest.raises(RuntimeError, match="boom"): - await adapter.on_message( - _msg("room-A", "hi"), - _FailingEventTools(), - [], - None, - None, - is_session_bootstrap=True, - room_id="room-A", - ) - - -class TestPerRunToolExposureEndToEnd: - """Drive a real Agno agent so we assert on the tools Agno actually offered - the model per run -- proving the factory is installed and invoked per turn, - and that contact tools do not leak across rooms through the shared agent.""" - - async def test_model_receives_only_active_room_tools(self): - model = CapturingModel() - agno = AgnoAgent(model=model, instructions="You are Dev.") - adapter = AgnoAdapter(agent=agno, emit=()) - await adapter.on_started("Bot", "desc") - - # Hub room: contact tools are offered to the model. - hub = ContactAwareTools(hub_room_id="hub", room_id="hub") - await adapter.on_message( - _msg("hub", "hi"), - hub, - [], - None, - None, - is_session_bootstrap=True, - room_id="hub", - ) - assert model.captured_tool_names is not None - assert "band_add_contact" in model.captured_tool_names - - # Normal room afterwards on the same shared agent: no contact leak. - normal = ContactAwareTools(room_id="room-A") - await adapter.on_message( - _msg("room-A", "hi"), - normal, - [], - None, - None, - is_session_bootstrap=True, - room_id="room-A", - ) - assert model.captured_tool_names == ["band_send_message"] diff --git a/tests/runtime/activityownership/kit.py b/tests/runtime/activityownership/kit.py index bfec560b1..7750c1ea5 100644 --- a/tests/runtime/activityownership/kit.py +++ b/tests/runtime/activityownership/kit.py @@ -2,14 +2,15 @@ import asyncio import json +from itertools import groupby from typing import Any import httpx -from tests.runtime.helpers import ALICE, LifecyclePlatform, page +from tests.runtime.helpers import PostingPlatform -class ActivityPlatform(LifecyclePlatform): +class ActivityPlatform(PostingPlatform): def __init__(self) -> None: super().__init__() self.activity_bodies: list[dict[str, Any]] = [] @@ -17,6 +18,13 @@ def __init__(self) -> None: self.activity_entered = asyncio.Event() self.activity_release = asyncio.Event() + @property + def working_transitions(self) -> list[bool]: + return [ + working + for working, _ in groupby(body["working"] for body in self.activity_bodies) + ] + async def respond(self, request: httpx.Request) -> httpx.Response: response = self.answer(request) if self.hold_next_activity and request.url.path.endswith("/activity"): @@ -29,10 +37,4 @@ def answer(self, request: httpx.Request) -> httpx.Response: tail = request.url.path.rsplit("/", 1)[-1] if tail == "activity": self.activity_bodies.append(json.loads(request.content)) - if tail == "participants": - return page([{**ALICE, "role": "member", "status": "active"}]) - if request.method == "POST" and tail == "messages": - return httpx.Response( - 201, json={"data": {"id": "m-1", "success": True, "recipients": []}} - ) return super().answer(request) diff --git a/tests/runtime/activityownership/test_band_tools.py b/tests/runtime/activityownership/test_band_tools.py new file mode 100644 index 000000000..439532e0d --- /dev/null +++ b/tests/runtime/activityownership/test_band_tools.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from collections.abc import Callable + +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from band.runtime.tools.types import BandTool +from band.runtime.working_state import end_tool_activity, report_tool_activity +from tests.conftest import make_message_event +from tests.runtime.activityownership.kit import ActivityPlatform + + +class TestBandToolActivity: + async def test_direct_registered_method_and_model_event_are_observed_but_telemetry_is_quiet( + self, execution: Callable[..., ExecutionContext], activity_platform: ActivityPlatform + ) -> None: + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + await tools.get_participants() + await report_tool_activity(tools, "Bash") + await tools.send_event("private telemetry", "thought") + assert activity_platform.activity_bodies[-1] == { + "working": True, + "activity": {"kind": "tool_call", "name": "Bash"}, + } + await end_tool_activity(tools) + await tools.execute_tool_call( + BandTool.SEND_EVENT, + {"content": "model telemetry", "message_type": "thought"}, + ) + + ctx = execution(handler) + await ctx._process_event(make_message_event()) + assert activity_platform.activity_bodies == [ + {"working": True, "activity": {"kind": "reasoning"}}, + { + "working": True, + "activity": {"kind": "tool_call", "name": BandTool.GET_PARTICIPANTS}, + }, + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": True, "activity": {"kind": "tool_call", "name": "Bash"}}, + {"working": True, "activity": {"kind": "reasoning"}}, + { + "working": True, + "activity": {"kind": "tool_call", "name": BandTool.SEND_EVENT}, + }, + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": False}, + ] diff --git a/tests/runtime/activityownership/test_opencode.py b/tests/runtime/activityownership/test_opencode.py index 7d8e09879..8a176a58e 100644 --- a/tests/runtime/activityownership/test_opencode.py +++ b/tests/runtime/activityownership/test_opencode.py @@ -19,7 +19,7 @@ ) from tests.conftest import make_message_event from tests.mcpbackends import backends_created_by -from tests.runtime.helpers import IDLE, KEEP_ALIVE, ROOM_ID, LifecyclePlatform +from tests.runtime.helpers import ROOM_ID, LifecyclePlatform class TestOpenCodeLifetime: @@ -59,7 +59,7 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: ctx = execution(handler) try: await ctx._process_event(make_message_event(msg_id="original")) - assert activity_platform.activity_reports == [KEEP_ALIVE] + assert activity_platform.working_transitions == [True] assert activity_platform.marked("processed") == ["original"] assert ctx.interrupt(kind=mode) notify = asyncio.create_task(adapter.on_interrupt(ROOM_ID, mode)) @@ -70,7 +70,7 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: await ctx._join_activity_completion() assert client.aborted_sessions == ["sess-1"] assert cleaned.is_set() - assert activity_platform.activity_reports == [KEEP_ALIVE, IDLE] + assert activity_platform.working_transitions == [True, False] assert activity_platform.marked("processed") == ["original"] finally: release.set() From 64de36274c7c622e2528cade3ac9023ef47ab13d Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 18 Oct 2026 11:04:40 +0300 Subject: [PATCH 11/47] test: Validate activity producers through real execution boundaries --- .github/workflows/ci.yml | 20 +- src/band/adapters/claude_sdk.py | 28 +- src/band/adapters/codex.py | 189 +- src/band/adapters/copilot_sdk.py | 44 +- src/band/adapters/cursor_acp.py | 53 +- src/band/integrations/acp/client_adapter.py | 14 +- src/band/runtime/decisions.py | 12 +- src/band/runtime/working_state.py | 2 +- src/band/testing/activity.py | 7 + tests/adapters/activity/__init__.py | 1 + tests/adapters/activity/kit.py | 26 + tests/adapters/activity/test_acp.py | 57 + tests/adapters/activity/test_letta_stream.py | 74 + .../adapters/activity/test_opencode_waits.py | 70 + .../activity/test_permission_delivery.py | 61 + tests/adapters/agno/adapter/test_activity.py | 50 + .../adapters/claude_sdk/activity/__init__.py | 1 + .../claude_sdk/activity/test_permission.py | 45 + .../approvals/test_manual_approval.py | 4 +- tests/adapters/codex/kit.py | 29 +- .../copilot_sdk/test_session_configuration.py | 3 +- tests/adapters/letta/__init__.py | 1 + tests/adapters/letta/kit.py | 35 + .../adapters/letta/test_agentcreateoptions.py | 85 + .../adapters/letta/test_autorelaydisabled.py | 95 + tests/adapters/letta/test_coldbootseeding.py | 219 ++ .../adapters/letta/test_configenvsourcing.py | 54 + .../letta/test_deleteagentsoncleanup.py | 63 + .../adapters/letta/test_executionreporting.py | 204 ++ .../letta/test_externaltoolrecording.py | 225 ++ tests/adapters/letta/test_extractsummary.py | 30 + tests/adapters/letta/test_formattimeago.py | 38 + .../letta/test_instructionblockupdate.py | 72 + tests/adapters/letta/test_lettaadapterinit.py | 28 + .../letta/test_lettaadapteroncleanup.py | 76 + .../test_lettaadapteronmessageperroom.py | 440 ++++ .../letta/test_lettaadaptersharedmode.py | 300 +++ .../letta/test_memoryconsolidation.py | 38 + tests/adapters/letta/test_rejoincontext.py | 62 + .../adapters/letta/test_sendtoolresolution.py | 87 + tests/adapters/letta/test_summarystorage.py | 53 + tests/adapters/letta/test_taskevents.py | 128 + tests/adapters/lettakit.py | 29 +- tests/adapters/strands/__init__.py | 1 + tests/adapters/strands/activity/__init__.py | 1 + tests/adapters/strands/activity/test_hooks.py | 49 + tests/adapters/test_letta_adapter.py | 2123 ----------------- tests/e2e/baseline/smoke/activity/__init__.py | 1 + tests/e2e/baseline/smoke/activity/conftest.py | 25 + tests/e2e/baseline/smoke/activity/kit.py | 140 ++ .../smoke/activity/test_native_reporting.py | 60 + .../smoke/activity/test_sdk_reporting.py | 63 + tests/e2e/baseline/toolkit/activity.py | 87 + tests/e2e/baseline/toolkit/capture.py | 48 +- tests/e2e/baseline/toolkit/tools.py | 4 +- .../crewaicoverage/__init__.py | 1 + .../crewaicoverage/kit.py | 17 + .../crewaicoverage/test_collection.py | 28 + .../crewaicoverage/test_dependencies.py | 15 + .../crewaicoverage/test_detection.py | 30 + .../crewaicoverage/test_paths.py | 19 + .../crewaicoverage/test_strictness.py | 26 + .../parlantcoverage/__init__.py | 1 + .../parlantcoverage/kit.py | 16 + .../parlantcoverage/test_collection.py | 28 + .../parlantcoverage/test_dependencies.py | 15 + .../parlantcoverage/test_detection.py | 27 + .../parlantcoverage/test_paths.py | 19 + .../test_crewai_job_coverage.py | 97 - .../test_parlant_job_coverage.py | 72 - .../test_strands_nooa_job_coverage.py | 5 + tests/integrations/crewai/__init__.py | 1 + .../integrations/crewai/activity/__init__.py | 1 + tests/integrations/crewai/activity/kit.py | 9 + .../crewai/activity/test_execution.py | 47 + .../integrations/parlant/activity/__init__.py | 1 + tests/integrations/parlant/activity/kit.py | 9 + .../parlant/activity/test_execution.py | 94 + tests/runtime/activityownership/kit.py | 9 +- .../activityownership/test_claude_native.py | 69 + .../test_detached_detail_feature.py | 162 ++ tests/toolkit/activity/__init__.py | 1 + tests/toolkit/activity/test_subscription.py | 59 + 83 files changed, 4170 insertions(+), 2462 deletions(-) create mode 100644 tests/adapters/activity/__init__.py create mode 100644 tests/adapters/activity/test_acp.py create mode 100644 tests/adapters/activity/test_letta_stream.py create mode 100644 tests/adapters/activity/test_opencode_waits.py create mode 100644 tests/adapters/activity/test_permission_delivery.py create mode 100644 tests/adapters/agno/adapter/test_activity.py create mode 100644 tests/adapters/claude_sdk/activity/__init__.py create mode 100644 tests/adapters/claude_sdk/activity/test_permission.py create mode 100644 tests/adapters/letta/__init__.py create mode 100644 tests/adapters/letta/kit.py create mode 100644 tests/adapters/letta/test_agentcreateoptions.py create mode 100644 tests/adapters/letta/test_autorelaydisabled.py create mode 100644 tests/adapters/letta/test_coldbootseeding.py create mode 100644 tests/adapters/letta/test_configenvsourcing.py create mode 100644 tests/adapters/letta/test_deleteagentsoncleanup.py create mode 100644 tests/adapters/letta/test_executionreporting.py create mode 100644 tests/adapters/letta/test_externaltoolrecording.py create mode 100644 tests/adapters/letta/test_extractsummary.py create mode 100644 tests/adapters/letta/test_formattimeago.py create mode 100644 tests/adapters/letta/test_instructionblockupdate.py create mode 100644 tests/adapters/letta/test_lettaadapterinit.py create mode 100644 tests/adapters/letta/test_lettaadapteroncleanup.py create mode 100644 tests/adapters/letta/test_lettaadapteronmessageperroom.py create mode 100644 tests/adapters/letta/test_lettaadaptersharedmode.py create mode 100644 tests/adapters/letta/test_memoryconsolidation.py create mode 100644 tests/adapters/letta/test_rejoincontext.py create mode 100644 tests/adapters/letta/test_sendtoolresolution.py create mode 100644 tests/adapters/letta/test_summarystorage.py create mode 100644 tests/adapters/letta/test_taskevents.py create mode 100644 tests/adapters/strands/__init__.py create mode 100644 tests/adapters/strands/activity/__init__.py create mode 100644 tests/adapters/strands/activity/test_hooks.py delete mode 100644 tests/adapters/test_letta_adapter.py create mode 100644 tests/e2e/baseline/smoke/activity/__init__.py create mode 100644 tests/e2e/baseline/smoke/activity/conftest.py create mode 100644 tests/e2e/baseline/smoke/activity/kit.py create mode 100644 tests/e2e/baseline/smoke/activity/test_native_reporting.py create mode 100644 tests/e2e/baseline/smoke/activity/test_sdk_reporting.py create mode 100644 tests/e2e/baseline/toolkit/activity.py create mode 100644 tests/framework_conformance/crewaicoverage/__init__.py create mode 100644 tests/framework_conformance/crewaicoverage/kit.py create mode 100644 tests/framework_conformance/crewaicoverage/test_collection.py create mode 100644 tests/framework_conformance/crewaicoverage/test_dependencies.py create mode 100644 tests/framework_conformance/crewaicoverage/test_detection.py create mode 100644 tests/framework_conformance/crewaicoverage/test_paths.py create mode 100644 tests/framework_conformance/crewaicoverage/test_strictness.py create mode 100644 tests/framework_conformance/parlantcoverage/__init__.py create mode 100644 tests/framework_conformance/parlantcoverage/kit.py create mode 100644 tests/framework_conformance/parlantcoverage/test_collection.py create mode 100644 tests/framework_conformance/parlantcoverage/test_dependencies.py create mode 100644 tests/framework_conformance/parlantcoverage/test_detection.py create mode 100644 tests/framework_conformance/parlantcoverage/test_paths.py delete mode 100644 tests/framework_conformance/test_crewai_job_coverage.py delete mode 100644 tests/framework_conformance/test_parlant_job_coverage.py create mode 100644 tests/integrations/crewai/__init__.py create mode 100644 tests/integrations/crewai/activity/__init__.py create mode 100644 tests/integrations/crewai/activity/kit.py create mode 100644 tests/integrations/crewai/activity/test_execution.py create mode 100644 tests/integrations/parlant/activity/__init__.py create mode 100644 tests/integrations/parlant/activity/kit.py create mode 100644 tests/integrations/parlant/activity/test_execution.py create mode 100644 tests/runtime/activityownership/test_claude_native.py create mode 100644 tests/runtime/activityownership/test_detached_detail_feature.py create mode 100644 tests/toolkit/activity/__init__.py create mode 100644 tests/toolkit/activity/test_subscription.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0830d6ec..dbff76839 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -148,6 +148,11 @@ jobs: BAND_ALLOW_MISSING_FRAMEWORKS: "1" run: uv run --all-packages pytest tests/e2e/baseline/ --collect-only -q --no-cov + - name: Run CrewAI activity tests + env: + BAND_ALLOW_MISSING_FRAMEWORKS: "1" + run: uv run --all-packages pytest tests/integrations/crewai/activity/ + - name: Run crewai tests env: # dev-crewai venv lacks langgraph/anthropic/parlant/pydantic-ai/etc.; @@ -156,7 +161,7 @@ jobs: # Only the tests that *need* this venv. Every other crewai test fakes the # package through sys.modules, so the `test` job above already runs it and # a second pass here would add no signal — just a path to forget. - # tests/framework_conformance/test_crewai_job_coverage.py keeps this honest. + # tests/framework_conformance/crewaicoverage/ keeps this honest. run: | uv run --all-packages pytest \ tests/adapters/test_crewai_flow_phase3.py \ @@ -218,6 +223,12 @@ jobs: BAND_TEST_EXTRA: "dev-strands-nooa" run: uv run --all-packages pytest tests/e2e/baseline/ --collect-only -q --no-cov + - name: Run Strands activity tests + env: + BAND_ALLOW_MISSING_FRAMEWORKS: "1" + BAND_TEST_EXTRA: "dev-strands-nooa" + run: uv run --all-packages pytest tests/adapters/strands/activity/ + - name: Run Strands and NOOA tests env: # dev-strands-nooa lacks modern framework packages; @@ -290,6 +301,11 @@ jobs: BAND_ALLOW_MISSING_FRAMEWORKS: "1" run: uv run --all-packages pytest tests/e2e/baseline/ --collect-only -q --no-cov + - name: Run Parlant activity tests + env: + BAND_ALLOW_MISSING_FRAMEWORKS: "1" + run: uv run --all-packages pytest tests/integrations/parlant/activity/ + - name: Run parlant tests env: # dev-parlant venv lacks langgraph/anthropic-adapter/pydantic-ai/etc.; @@ -298,7 +314,7 @@ jobs: # Only the tests that *need* this venv. Every other parlant test fakes the # package through sys.modules, so the `test` job above already runs it and # a second pass here would add no signal — just a path to forget. - # tests/framework_conformance/test_parlant_job_coverage.py keeps this honest. + # tests/framework_conformance/parlantcoverage/ keeps this honest. run: | uv run --all-packages pytest \ tests/integrations/parlant/test_catalog.py \ diff --git a/src/band/adapters/claude_sdk.py b/src/band/adapters/claude_sdk.py index 622971f72..56e1c2567 100644 --- a/src/band/adapters/claude_sdk.py +++ b/src/band/adapters/claude_sdk.py @@ -1870,25 +1870,27 @@ async def _resolve_manual_approval( self._release_turn(room_id) decision = await registry.wait( - entry, pending.future, timeout_s=self._approvals.wait_timeout_s + entry, + pending.future, + timeout_s=self._approvals.wait_timeout_s, + retain_entry=True, ) + match decision: + case Timeout.TIMED_OUT: + return await self._apply_approval_timeout( + tools, room_id, token, mention, tool_use_id + ) + case ApprovalReply(): + return await self._apply_approval_reply( + tools, room_id, token, decision, tool_use_id + ) + # Forced by eviction or room teardown: no notice names this call. + return PermissionResultDeny(message="User declined tool use") finally: self._clear_pending_approval(room_id, entry) if tools: await set_permission_wait(tools, bool(registry)) - match decision: - case Timeout.TIMED_OUT: - return await self._apply_approval_timeout( - tools, room_id, token, mention, tool_use_id - ) - case ApprovalReply(): - return await self._apply_approval_reply( - tools, room_id, token, decision, tool_use_id - ) - # Forced by eviction or room teardown: no notice names this call. - return PermissionResultDeny(message="User declined tool use") - async def _apply_approval_reply( self, tools: AgentToolsProtocol | None, diff --git a/src/band/adapters/codex.py b/src/band/adapters/codex.py index da7e359d1..dd01e13b6 100644 --- a/src/band/adapters/codex.py +++ b/src/band/adapters/codex.py @@ -8,7 +8,7 @@ import time as _time from collections import OrderedDict from collections.abc import AsyncIterator, Callable -from contextlib import asynccontextmanager, nullcontext +from contextlib import AsyncExitStack, asynccontextmanager, nullcontext from contextvars import ContextVar from dataclasses import dataclass, fields from dataclasses import field as dataclass_field @@ -2641,89 +2641,98 @@ async def _handle_approval_request( room_session = self._session_approved.get(room_id) session_hit = bool(session_key and room_session and session_key in room_session) - if session_hit: - decision: ApprovalDecision = "acceptForSession" - decided_by = "session_policy" - elif self._room_closing(room_id): - decision = "decline" - decided_by = "room_cleanup" - elif self.config.approval_mode == "manual": - try: - decision = await self._resolve_manual_approval( - tools=tools, - msg=msg, - room_id=room_id, - event=event, - summary=summary, - params=params, - ) - decided_by = msg.sender_name or msg.sender_type or "user" - except Exception: - # Ensure we still answer the server request even if the - # human-facing notification flow fails. - logger.exception("Manual approval flow failed; defaulting to decline") + async with AsyncExitStack() as approvals: + if session_hit: + decision: ApprovalDecision = "acceptForSession" + decided_by = "session_policy" + elif self._room_closing(room_id): decision = "decline" - decided_by = "system_fallback" - else: - decision = ( - "accept" if self.config.approval_mode == "auto_accept" else "decline" - ) - decided_by = f"policy:{self.config.approval_mode}" + decided_by = "room_cleanup" + elif self.config.approval_mode == "manual": + try: + decision = await approvals.enter_async_context( + self._manual_approval( + tools=tools, + msg=msg, + room_id=room_id, + event=event, + summary=summary, + params=params, + ) + ) + decided_by = msg.sender_name or msg.sender_type or "user" + except Exception: + # Ensure we still answer the server request even if the + # human-facing notification flow fails. + logger.exception( + "Manual approval flow failed; defaulting to decline" + ) + decision = "decline" + decided_by = "system_fallback" + else: + decision = ( + "accept" + if self.config.approval_mode == "auto_accept" + else "decline" + ) + decided_by = f"policy:{self.config.approval_mode}" - await client.respond(event.id, _approval_response(decision)) + await client.respond(event.id, _approval_response(decision)) - audit_entry = self._record_approval_audit( - room_id=room_id, - request_id=str(event.id), - method=event.method, - decision=decision, - decided_by=decided_by, - summary=summary, - session_level=session_hit, - ) - await self._emit_approval_audit_event( - tools=tools, - room_id=room_id, - entry=audit_entry, - ) + audit_entry = self._record_approval_audit( + room_id=room_id, + request_id=str(event.id), + method=event.method, + decision=decision, + decided_by=decided_by, + summary=summary, + session_level=session_hit, + ) + await self._emit_approval_audit_event( + tools=tools, + room_id=room_id, + entry=audit_entry, + ) - if ( - self.config.approval_mode != "manual" - and self.config.approval_text_notifications - ): - mention = [ - { - "id": msg.sender_id, - "name": msg.sender_name or msg.sender_type, - } - ] - try: - await tools.send_notice( - f"Approval requested ({summary}). Policy decision: {decision}.", - mentions=mention, - ) - except Exception: - logger.exception("Failed to send approval policy notification") + if ( + self.config.approval_mode != "manual" + and self.config.approval_text_notifications + ): + mention = [ + { + "id": msg.sender_id, + "name": msg.sender_name or msg.sender_type, + } + ] + try: + await tools.send_notice( + f"Approval requested ({summary}). Policy decision: {decision}.", + mentions=mention, + ) + except Exception: + logger.exception("Failed to send approval policy notification") - if Emit.THOUGHTS in self.features.emit: - # Best-effort telemetry — never fail the turn on thought emission - # failures. - await send_event_safe( - tools, - content=(f"Codex approval request handled automatically ({decision})."), - message_type="thought", - metadata={ - "codex_approval_method": event.method, - "codex_approval_type": self._approval_type(event.method), - "codex_approval_options": [ - "accept", - "acceptForSession", - "decline", - ], - }, - log_label="approval thought event", - log_level=logging.DEBUG, - ) + if Emit.THOUGHTS in self.features.emit: + # Best-effort telemetry — never fail the turn on thought emission + # failures. + await send_event_safe( + tools, + content=( + f"Codex approval request handled automatically ({decision})." + ), + message_type="thought", + metadata={ + "codex_approval_method": event.method, + "codex_approval_type": self._approval_type(event.method), + "codex_approval_options": [ + "accept", + "acceptForSession", + "decline", + ], + }, + log_label="approval thought event", + log_level=logging.DEBUG, + ) @staticmethod def _turn_usage(usage: CodexTokenUsage | None) -> TurnUsage: @@ -3155,7 +3164,8 @@ def _extract_thought_text(item_type: str, item: dict[str, Any]) -> str | None: case _: return None - async def _resolve_manual_approval( + @asynccontextmanager + async def _manual_approval( self, *, tools: AgentToolsProtocol, @@ -3164,7 +3174,7 @@ async def _resolve_manual_approval( event: RpcEvent, summary: str, params: dict[str, Any], - ) -> ApprovalDecision: + ) -> AsyncIterator[ApprovalDecision]: mention = self._sender_mention(msg) if event.id is None: raise RuntimeError("approval request must have an id") @@ -3189,7 +3199,8 @@ async def _resolve_manual_approval( pending.native_key, room_id, ) - return "decline" + yield "decline" + return entry = registration.entry token = entry.token for removed in registration.removed: @@ -3202,6 +3213,7 @@ async def _resolve_manual_approval( self.config.max_pending_approvals_per_room, ) try: + await set_permission_wait(tools, True) approval_msg = APPROVAL_REQUESTED_TEMPLATE.format( summary=summary, token=token ) @@ -3266,13 +3278,16 @@ async def _resolve_manual_approval( ) raise - await set_permission_wait(tools, True) self._release_turn(room_id, tools) decision = await registry.wait( - entry, pending.future, timeout_s=self.config.approval_wait_timeout_s + entry, + pending.future, + timeout_s=self.config.approval_wait_timeout_s, + retain_entry=True, ) if decision is not Timeout.TIMED_OUT: - return decision + yield decision + return timeout_decision = self.config.approval_timeout_decision try: await tools.send_notice( @@ -3283,7 +3298,7 @@ async def _resolve_manual_approval( ) except Exception: logger.exception("Failed to send approval timeout notification") - return timeout_decision + yield timeout_decision finally: self._clear_pending_approval(room_id, entry) await set_permission_wait(tools, bool(registry)) @@ -3965,7 +3980,7 @@ async def _handle_approval_command( if (decision_value := _APPROVAL_COMMAND_DECISIONS.get(command)) is None: return False - if not pending: + if not pending.has_unclaimed(): await deliver_notice( tools, NO_APPROVALS_TO_RESOLVE_MESSAGE, diff --git a/src/band/adapters/copilot_sdk.py b/src/band/adapters/copilot_sdk.py index b20df1631..636cb3cbe 100644 --- a/src/band/adapters/copilot_sdk.py +++ b/src/band/adapters/copilot_sdk.py @@ -8,6 +8,7 @@ from __future__ import annotations +import inspect import json import logging import re @@ -57,7 +58,7 @@ is_image_passthrough_result, redact_tool_call_args, ) -from band.runtime.working_state import tool_activity +from band.runtime.working_state import set_permission_wait, tool_activity try: from copilot import ( @@ -99,6 +100,8 @@ from band.core.types import FeatureKwargs, PlatformMessage from band.runtime.custom_tools import CustomToolDef +ASK_USER_TOOL_NAME = "ask_user" + logger = logging.getLogger(__name__) _PROVIDER = "copilot_sdk" @@ -622,14 +625,41 @@ def _session_kwargs( if self._ask_user is not None: # ask_user is session-isolated (no shell/file/host access), so # allowing it does not weaken the approve_all stance above. - available_tools.append("ask_user") - kwargs["on_user_input_request"] = ( - self._ask_user - if callable(self._ask_user) - else self._make_room_ask_user_handler(room_id) - ) + available_tools.append(ASK_USER_TOOL_NAME) + kwargs["on_user_input_request"] = self._activity_user_input_handler(room_id) return kwargs + def _activity_user_input_handler(self, room_id: str) -> UserInputHandler: + handler = ( + self._ask_user + if callable(self._ask_user) + else self._make_room_ask_user_handler(room_id) + ) + manual = callable(self._ask_user) + pending = 0 + + async def handle( + request: UserInputRequest, context: dict[str, str] + ) -> UserInputResponse: + nonlocal pending + tools = self._room_tools.get(room_id) + async with tool_activity(tools, ASK_USER_TOOL_NAME): + result = handler(request, context) + if not inspect.isawaitable(result): + return result + if manual: + pending += 1 + try: + if manual: + await set_permission_wait(tools, True) + return await result + finally: + if manual: + pending -= 1 + await set_permission_wait(tools, pending > 0) + + return handle + def _make_room_ask_user_handler(self, room_id: str) -> UserInputHandler: """Bind the room id for the session's ask_user → room bridge.""" diff --git a/src/band/adapters/cursor_acp.py b/src/band/adapters/cursor_acp.py index bcf3e249a..b32cf2396 100644 --- a/src/band/adapters/cursor_acp.py +++ b/src/band/adapters/cursor_acp.py @@ -5,7 +5,7 @@ import asyncio import logging from collections.abc import AsyncIterator -from contextlib import asynccontextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext from dataclasses import dataclass, field from functools import partial from typing import ClassVar, Literal, Self @@ -58,6 +58,7 @@ permission_option_ids, select_allow_option_id, ) +from band.integrations.acp.room_emitter import RoomTurnEmitter from band.integrations.acp.session_config import SessionConfigResolver from band.runtime.custom_tools import CustomToolDef from band.runtime.decisions import ( @@ -67,6 +68,7 @@ ) from band.runtime.formatters import strip_leading_mentions from band.runtime.tools.registry import BAND_MCP_SERVER_NAME +from band.runtime.working_state import set_permission_wait from band.workspaces import WorkspaceResolver logger = logging.getLogger(__name__) @@ -460,6 +462,12 @@ async def cleanup_all(self) -> None: self._cancel_all_decisions() await super().cleanup_all() + def _permission_activity( + self, emitter: RoomTurnEmitter | None + ) -> AbstractAsyncContextManager[None]: + # Cursor's decision registry owns manual permission, question and plan waits. + return nullcontext() + async def _resolve_cursor_permission( self, request: ACPPermissionRequest ) -> str | None: @@ -626,28 +634,33 @@ async def _wait_for_decision( self._resolve_evicted_decision(registration.evicted) entry = registration.entry try: - await turn.tools.send_notice( - prompt.replace("{token}", entry.token), - mentions=_requester_mentions(turn), + await set_permission_wait(turn.tools, True) + try: + await turn.tools.send_notice( + prompt.replace("{token}", entry.token), + mentions=_requester_mentions(turn), + ) + except Exception: # noqa: BLE001 -- best-effort room notify; any failure (network, REST, unresolved mention) should not block the decision wait below + logger.warning("Could not deliver Cursor %s decision prompt", kind) + # A reply that claimed it meanwhile owns the answer; wait for it. + if decisions.withdraw(entry): + return None + finally: + # Release on_message here, not only in _run_turn's finally -- + # the room needs its queue back the instant a decision is + # outstanding, whether or not the prompt itself landed. + _detach(turn) + + result = await decisions.wait( + entry, future, timeout_s=self.config.decision_timeout_s ) - except Exception: # noqa: BLE001 -- best-effort room notify; any failure (network, REST, unresolved mention) should not block the decision wait below - logger.warning("Could not deliver Cursor %s decision prompt", kind) - # A reply that claimed it meanwhile owns the answer; wait for it. - if decisions.withdraw(entry): + if result is Timeout.TIMED_OUT: + await self._notify_decision_timeout(turn, kind, entry.token) return None + return result finally: - # Release on_message here, not only in _run_turn's finally -- - # the room needs its queue back the instant a decision is - # outstanding, whether or not the prompt itself landed. - _detach(turn) - - result = await decisions.wait( - entry, future, timeout_s=self.config.decision_timeout_s - ) - if result is Timeout.TIMED_OUT: - await self._notify_decision_timeout(turn, kind, entry.token) - return None - return result + decisions.forget(entry) + await set_permission_wait(turn.tools, bool(decisions)) @staticmethod async def _notify_decision_timeout( diff --git a/src/band/integrations/acp/client_adapter.py b/src/band/integrations/acp/client_adapter.py index 4cb730b89..3a4fb6658 100644 --- a/src/band/integrations/acp/client_adapter.py +++ b/src/band/integrations/acp/client_adapter.py @@ -12,7 +12,12 @@ Mapping, Sequence, ) -from contextlib import AsyncExitStack, asynccontextmanager, nullcontext +from contextlib import ( + AbstractAsyncContextManager, + AsyncExitStack, + asynccontextmanager, + nullcontext, +) from dataclasses import dataclass from enum import StrEnum from functools import partial @@ -1100,6 +1105,11 @@ async def _narrate_cancelled_permission( else: await narrate(narration) + def _permission_activity( + self, emitter: RoomTurnEmitter | None + ) -> AbstractAsyncContextManager[None]: + return emitter.permission_wait() if emitter is not None else nullcontext() + async def _resolve_permission_option( self, *, @@ -1114,7 +1124,7 @@ async def _resolve_permission_option( return select_allow_option_id(options) offered = self._permission_options(options) - async with emitter.permission_wait() if emitter is not None else nullcontext(): + async with self._permission_activity(emitter): option_id = await self._resolve_permission( ACPPermissionRequest( room_id=room_id, diff --git a/src/band/runtime/decisions.py b/src/band/runtime/decisions.py index fee965171..795a3ba63 100644 --- a/src/band/runtime/decisions.py +++ b/src/band/runtime/decisions.py @@ -161,11 +161,16 @@ def cancel_all(self) -> list[DecisionEntry[T]]: return self._drop_cancelled(self._core.cancel_all()) async def wait( - self, entry: DecisionEntry[T], future: asyncio.Future[R], *, timeout_s: float + self, + entry: DecisionEntry[T], + future: asyncio.Future[R], + *, + timeout_s: float, + retain_entry: bool = False, ) -> R | Literal[Timeout.TIMED_OUT]: """The answer to ``entry``, or ``TIMED_OUT`` if the deadline claims it first or it was replaced; a reply that claimed it first is always - waited for.""" + waited for. With ``retain_entry``, the caller retires it after delivery.""" try: # asyncio.timeout(), not wait_for: on Python 3.11 wait_for returns # an answer that lands in the same tick as a cancel of this task @@ -184,7 +189,8 @@ async def wait( return await future return Timeout.TIMED_OUT finally: - self.forget(entry) + if not retain_entry: + self.forget(entry) def _holds(self, entry: DecisionEntry[T]) -> bool: # Tickets are per registry, so an entry from a registry a room has diff --git a/src/band/runtime/working_state.py b/src/band/runtime/working_state.py index c35ba81d2..23661f17c 100644 --- a/src/band/runtime/working_state.py +++ b/src/band/runtime/working_state.py @@ -65,8 +65,8 @@ async def set_permission_wait(tools: object, waiting: bool) -> None: @asynccontextmanager async def tool_activity(tools: object, name: str) -> AsyncIterator[None]: - await report_tool_activity(tools, name) try: + await report_tool_activity(tools, name) yield finally: await end_tool_activity(tools) diff --git a/src/band/testing/activity.py b/src/band/testing/activity.py index ebed4b670..d8022b986 100644 --- a/src/band/testing/activity.py +++ b/src/band/testing/activity.py @@ -11,6 +11,13 @@ class ActivityRecorder: def __init__(self) -> None: self.events: list[dict[str, Any]] = [] + @property + def waiting(self) -> bool: + return next( + (event["waiting"] for event in reversed(self.events) if "waiting" in event), + False, + ) + async def update(self, activity: AgentActivityDetail) -> None: self.events.append(activity.model_dump(exclude_unset=True)) diff --git a/tests/adapters/activity/__init__.py b/tests/adapters/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/adapters/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/activity/kit.py b/tests/adapters/activity/kit.py index b2ec79ba4..d686447fc 100644 --- a/tests/adapters/activity/kit.py +++ b/tests/adapters/activity/kit.py @@ -1,8 +1,14 @@ from __future__ import annotations +import asyncio +from collections.abc import AsyncIterator + +import httpx from anthropic.types import Message, ToolUseBlock, Usage from pydantic import BaseModel +from tests.adapters.lettakit import make_tool_call_message, make_tool_return_message + class ActivityLookup(BaseModel): """Look up an opaque test value.""" @@ -22,3 +28,23 @@ def anthropic_tool_response(name: str, note: str) -> Message: usage=Usage(input_tokens=1, output_tokens=1), stop_reason="tool_use", ) + + +class LettaActivityStream(httpx.AsyncByteStream): + def __init__(self) -> None: + self.entered = asyncio.Event() + self.release = asyncio.Event() + self.closed = False + + async def __aiter__(self) -> AsyncIterator[bytes]: + call = make_tool_call_message("shell", '{"command":"private-argument"}') + yield f"data: {call.model_dump_json()}\n\n".encode() + self.entered.set() + await self.release.wait() + result = make_tool_return_message("shell", "private-result") + yield f"data: {result.model_dump_json()}\n\n".encode() + yield b'data: {"message_type":"usage_statistics","prompt_tokens":17,"completion_tokens":5}\n\n' + yield b"data: [DONE]\n\n" + + async def aclose(self) -> None: + self.closed = True diff --git a/tests/adapters/activity/test_acp.py b/tests/adapters/activity/test_acp.py new file mode 100644 index 000000000..5e4de2690 --- /dev/null +++ b/tests/adapters/activity/test_acp.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from acp.schema import ToolCallProgress, ToolCallStart + +from band.integrations.acp.collecting import ACPCollectingClient +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder + + +class TestACPActivity: + async def test_activity_precedes_buffered_narration_and_ignores_private_titles( + self, + ) -> None: + client = ACPCollectingClient() + tools = FakeAgentTools() + recorder = ActivityRecorder() + tools._activity_observer = recorder + client.set_activity_tools("session", tools) + await client.session_update( + "session", + ToolCallStart( + sessionUpdate="tool_call", + toolCallId="call", + title="private command and arguments", + status="pending", + ), + ) + assert recorder.events == [{"kind": "tool_call", "name": "unknown"}] + await client.session_update( + "session", + ToolCallProgress( + sessionUpdate="tool_call_update", + toolCallId="call", + title="private-result", + status="in_progress", + rawOutput={"text": "private-result"}, + ), + ) + assert recorder.events == [{"kind": "tool_call", "name": "unknown"}] + await client.session_update( + "session", + ToolCallProgress( + sessionUpdate="tool_call_update", toolCallId="call", status="completed" + ), + ) + assert recorder.events == [ + {"kind": "tool_call", "name": "unknown"}, + {"kind": "reasoning"}, + ] + client.set_activity_tools("session", None) + await client.session_update( + "session", + ToolCallStart( + sessionUpdate="tool_call", toolCallId="late", title="private-late" + ), + ) + assert recorder.events[-1] == {"kind": "reasoning"} diff --git a/tests/adapters/activity/test_letta_stream.py b/tests/adapters/activity/test_letta_stream.py new file mode 100644 index 000000000..e5891ac0d --- /dev/null +++ b/tests/adapters/activity/test_letta_stream.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import asyncio +import json +from typing import Any + +import httpx +import pytest +from letta_client import AsyncLetta + +from band.adapters.letta import LettaAdapter, LettaAdapterConfig, RoomContext +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder +from tests.adapters.activity.kit import LettaActivityStream + + +class TestLettaStreamActivity: + @pytest.mark.parametrize("cancelled", [False, True]) + async def test_real_sdk_observes_stream_before_response_buffering_and_closes( + self, cancelled: bool + ) -> None: + stream = LettaActivityStream() + requests: list[dict[str, Any]] = [] + + async def respond(request: httpx.Request) -> httpx.Response: + requests.append(json.loads(request.content)) + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=stream + ) + + recorder = ActivityRecorder() + tools = FakeAgentTools() + tools._activity_observer = recorder + async with ( + httpx.AsyncClient(transport=httpx.MockTransport(respond)) as transport, + AsyncLetta( + api_key="test", base_url="https://letta.test", http_client=transport + ) as client, + ): + adapter = LettaAdapter(LettaAdapterConfig(agent_id="agent"), emit=()) + adapter._client = client + task = asyncio.create_task( + adapter._call_provider( + "agent", + [{"role": "user", "content": "hello"}], + RoomContext(agent_id="agent"), + tools, + ) + ) + try: + async with asyncio.timeout(5): + await stream.entered.wait() + assert recorder.events == [{"kind": "tool_call", "name": "shell"}] + assert not task.done() + if cancelled: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + stream.release.set() + messages, usage = await task + assert [message.message_type for message in messages] == [ + "tool_call_message", + "tool_return_message", + ] + assert (usage.input_tokens, usage.output_tokens) == (17, 5) + assert recorder.events[-1] == {"kind": "reasoning"} + finally: + stream.release.set() + assert stream.closed + + assert requests[0]["streaming"] is True + assert requests[0]["stream_tokens"] is False + assert not tools.events_sent diff --git a/tests/adapters/activity/test_opencode_waits.py b/tests/adapters/activity/test_opencode_waits.py new file mode 100644 index 000000000..4e6d834bb --- /dev/null +++ b/tests/adapters/activity/test_opencode_waits.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import asyncio +from unittest.mock import patch + +from band.adapters.opencode import OpencodeAdapterConfig +from band.adapters.opencode.approvals import ApprovalPorts, RoomApprovals +from band.runtime.working_state import end_tool_activity, report_tool_activity +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder +from tests.adapters.opencode.helpers import AskFactory, FakeOpencodeClient + + +class TestOpenCodePermissionActivity: + async def test_last_wait_includes_claimed_reply_transport_and_latest_progress( + self, + ) -> None: + tools = FakeAgentTools() + recorder = ActivityRecorder() + tools._activity_observer = recorder + client = FakeOpencodeClient() + entered, release = asyncio.Event(), asyncio.Event() + + async def abort() -> None: + await client.abort_session("session") + + approvals = RoomApprovals( + OpencodeAdapterConfig(), + ApprovalPorts( + room_id="room", + client=lambda: client, + tools=lambda: tools, + turn_mentions=list, + release_turn_wait=lambda: None, + fail_turn=lambda message: None, + abort_session=abort, + is_own_band_tool=lambda permission: False, + ), + ) + asks = AskFactory() + await approvals.on_permission_asked(asks.permission("first")) + await approvals.on_permission_asked(asks.permission("second")) + original = client.reply_permission + + async def held_reply( + permission_id: str, *, reply: str, message: str | None = None + ) -> None: + if permission_id == "first": + entered.set() + await release.wait() + await original(permission_id, reply=reply, message=message) + + with patch.object(client, "reply_permission", held_reply): + first = asyncio.create_task( + approvals.try_handle_reply("approve first", "user") + ) + try: + await entered.wait() + assert await approvals.try_handle_reply("approve second", "user") + await report_tool_activity(tools, "ordinary-tool") + await end_tool_activity(tools) + assert approvals.awaiting_human() + assert recorder.waiting + finally: + release.set() + await first + approvals.cancel() + assert not approvals.awaiting_human() + assert not recorder.waiting + assert recorder.events[-1] == {"waiting": False} diff --git a/tests/adapters/activity/test_permission_delivery.py b/tests/adapters/activity/test_permission_delivery.py new file mode 100644 index 000000000..b87588c1a --- /dev/null +++ b/tests/adapters/activity/test_permission_delivery.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import asyncio +from typing import Any +from unittest.mock import AsyncMock + +from band.adapters.codex import CodexAdapter, CodexAdapterConfig, CodexCommand +from band.integrations.codex import CodexApprovalMethod +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder +from tests.adapters.codex.kit import make_platform_message, wire_codex_room +from tests.adapters.codexturns import event_request +from tests.adapters.opencode.helpers import wait_for + + +class TestPermissionDelivery: + async def test_codex_retains_claimed_permission_until_provider_response_finishes( + self, + ) -> None: + adapter = CodexAdapter(CodexAdapterConfig(approval_mode="manual"), emit=()) + await adapter.on_started("Test", "Test") + tools = FakeAgentTools() + recorder = ActivityRecorder() + tools._activity_observer = recorder + entered, release = asyncio.Event(), asyncio.Event() + + async def respond(request_id: int, result: dict[str, Any]) -> None: + entered.set() + await release.wait() + + client = AsyncMock() + client.respond.side_effect = respond + wire_codex_room(adapter, client) + params = {"command": "private-command", "approvalId": "approval"} + pending = asyncio.create_task( + adapter._handle_approval_request( + client=client, + tools=tools, + msg=make_platform_message(), + room_id="room-1", + event=event_request(1, CodexApprovalMethod.COMMAND_EXECUTION, params), + params=params, + ) + ) + try: + await wait_for(lambda: bool(tools.messages_sent)) + await adapter._handle_approval_command( + tools=FakeAgentTools(), + msg=make_platform_message(), + room_id="room-1", + command=CodexCommand.APPROVE, + args="", + ) + await entered.wait() + assert recorder.waiting + assert adapter._pending_approvals["room-1"].has_claimed() + finally: + release.set() + await pending + assert not recorder.waiting + assert not adapter._pending_approvals["room-1"] diff --git a/tests/adapters/agno/adapter/test_activity.py b/tests/adapters/agno/adapter/test_activity.py new file mode 100644 index 000000000..1092a4426 --- /dev/null +++ b/tests/adapters/agno/adapter/test_activity.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import pytest +from agno.models.message import Message +from agno.run.agent import RunOutput + +from band.core.types import Emit +from band.testing.activity import ActivityRecorder +from tests.adapters.agno.helpers import tool_events, tool_execution + + +class TestAgnoStreamActivity: + @pytest.mark.parametrize("narrated", [False, True]) + async def test_streamed_tool_activity_preserves_final_response_with_or_without_narration( + self, make_started_adapter, sample_platform_message, tools, narrated: bool + ) -> None: + recorder = ActivityRecorder() + tools._activity_observer = recorder + response = RunOutput( + content="final reply", + messages=[Message(role="assistant", content="final reply")], + ) + adapter, agent = await make_started_adapter( + response, + emit=(Emit.TOOL_CALLS,) if narrated else (), + events=tool_events( + tool_execution( + "shell", + args={"command": "private-argument"}, + result="private-result", + ) + ), + ) + await adapter.on_message( + sample_platform_message, + tools, + [], + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + assert recorder.events == [ + {"kind": "tool_call", "name": "shell"}, + {"kind": "reasoning"}, + ] + assert not tools.messages_sent + assert adapter._message_history["room-1"][-1].content == "final reply" + assert bool(tools.events_sent) is narrated + assert agent.arun.call_args.kwargs["stream"] is True diff --git a/tests/adapters/claude_sdk/activity/__init__.py b/tests/adapters/claude_sdk/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/adapters/claude_sdk/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/claude_sdk/activity/test_permission.py b/tests/adapters/claude_sdk/activity/test_permission.py new file mode 100644 index 000000000..9fbcae179 --- /dev/null +++ b/tests/adapters/claude_sdk/activity/test_permission.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from band.runtime.working_state import report_tool_activity +from band.testing.activity import ActivityRecorder +from tests.adapters.claude_sdk.helpers import OpenRoom, with_approvals +from tests.adapters.opencode.helpers import wait_for +from tests.baseline.decisions import ModelDecision, ToolCall + + +class TestClaudePermissionActivity: + async def test_overlapping_native_waits_include_claimed_notice_delivery( + self, claude_room: OpenRoom + ) -> None: + room = await claude_room(with_approvals(), emit=()) + recorder = ActivityRecorder() + room.tools._activity_observer = recorder + room.claude.script( + [ + ModelDecision( + tool_calls=( + ToolCall("Bash", {"command": "private-command"}), + ToolCall( + "Write", + {"file_path": "private-path", "content": "private-content"}, + ), + ) + ), + room.model_reply("Done"), + ] + ) + held = room.tools.hold_message("resolved") + await room.send("Do both") + await room.until_said("Approval requested", times=2) + await room.send("/approve a-1") + async with held: + assert recorder.waiting + await report_tool_activity(room.tools, "ordinary-tool") + await room.send("/approve a-2") + await wait_for( + lambda: len(room.adapter._pending_approvals[room.room_id]) == 1 + ) + assert recorder.waiting + await room.settled() + assert not recorder.waiting + assert not room.tool_call_names diff --git a/tests/adapters/claude_sdk/approvals/test_manual_approval.py b/tests/adapters/claude_sdk/approvals/test_manual_approval.py index 44ddfe6e0..d5afafecb 100644 --- a/tests/adapters/claude_sdk/approvals/test_manual_approval.py +++ b/tests/adapters/claude_sdk/approvals/test_manual_approval.py @@ -240,7 +240,7 @@ async def test_an_unanswered_ask_times_out_to_the_configured_decision( self, claude_room: OpenRoom, decision: str, bash_output: str ) -> None: """A reply landing while the timeout notice is still being sent is told - the token is gone, not "resolved".""" + the token is no longer pending, not "resolved".""" room = await claude_room( with_approvals(wait_timeout_s=60, timeout_decision=decision) ) @@ -254,7 +254,7 @@ async def test_an_unanswered_ask_times_out_to_the_configured_decision( assert room.chat == [ prompt("a-1", "Bash: `ls`"), - "Unknown approval token `a-1`. Available: none.", + "Approval `a-1` is no longer pending.", APPROVAL_TIMED_OUT_TEMPLATE.format(token="a-1", decision=decision), "Done waiting.", ] diff --git a/tests/adapters/codex/kit.py b/tests/adapters/codex/kit.py index cda37d249..02e01a01a 100644 --- a/tests/adapters/codex/kit.py +++ b/tests/adapters/codex/kit.py @@ -446,20 +446,21 @@ def ask( self, approval_id: str, *, request_id: int ) -> asyncio.Task[ApprovalDecision]: params = {"approvalId": approval_id, "command": "npm test"} - self.asks.append( - ask := asyncio.create_task( - self.adapter._resolve_manual_approval( - tools=self.tools, - msg=make_platform_message(room_id=ROOM_ID), - room_id=ROOM_ID, - event=event_request( - request_id, CodexApprovalMethod.COMMAND_EXECUTION, params - ), - summary="npm test", - params=params, - ) - ) - ) + + async def resolve() -> ApprovalDecision: + async with self.adapter._manual_approval( + tools=self.tools, + msg=make_platform_message(room_id=ROOM_ID), + room_id=ROOM_ID, + event=event_request( + request_id, CodexApprovalMethod.COMMAND_EXECUTION, params + ), + summary="npm test", + params=params, + ) as decision: + return decision + + self.asks.append(ask := asyncio.create_task(resolve())) return ask @asynccontextmanager diff --git a/tests/adapters/copilot_sdk/test_session_configuration.py b/tests/adapters/copilot_sdk/test_session_configuration.py index 2c7391ae6..79e67dd7a 100644 --- a/tests/adapters/copilot_sdk/test_session_configuration.py +++ b/tests/adapters/copilot_sdk/test_session_configuration.py @@ -90,7 +90,8 @@ async def ask_operator(request, context): await run_message(adapter, tools) kwargs = client.sessions[0].kwargs - assert kwargs["on_user_input_request"] is ask_operator + response = await kwargs["on_user_input_request"]({"question": "Question?"}, {}) + assert response == {"answer": "beta", "wasFreeform": False} # ask_user must be allowlisted alongside the bridged Band tools, # or the built-in tool stays filtered out and the handler is dead. assert "ask_user" in kwargs["available_tools"] diff --git a/tests/adapters/letta/__init__.py b/tests/adapters/letta/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/adapters/letta/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/letta/kit.py b/tests/adapters/letta/kit.py new file mode 100644 index 000000000..990a45d37 --- /dev/null +++ b/tests/adapters/letta/kit.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import json + +import pytest + +from band.core.types import ToolEventKey +from band.runtime.tools import BandTool +from band.testing import ( + FakeAgentTools, + events_of_type, +) + +REPLY_SETTLING_CALLS = [ + pytest.param( + BandTool.SEND_MESSAGE, {"content": "Hi", "mentions": ["@alice"]}, id="message" + ), + pytest.param( + BandTool.SEND_ROOM_FILE, + {"content": "notes", "filename": "notes.txt", "mentions": ["@alice"]}, + id="room-file", + ), + pytest.param(BandTool.NO_REPLY, {"reason": "FYI only"}, id="no-reply"), +] + + +def sent_contents(tools: FakeAgentTools) -> list[str]: + return [message["content"] for message in tools.messages_sent] + + +def reported_tool_names(tools: FakeAgentTools, event_type: str) -> list[str]: + return [ + json.loads(event["content"])[ToolEventKey.NAME] + for event in events_of_type(tools, event_type) + ] diff --git a/tests/adapters/letta/test_agentcreateoptions.py b/tests/adapters/letta/test_agentcreateoptions.py new file mode 100644 index 000000000..ca27c331f --- /dev/null +++ b/tests/adapters/letta/test_agentcreateoptions.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, + LettaMCPConfig, +) +from tests.adapters.lettakit import ( + make_mock_agent, +) + + +class TestAgentCreateOptions: + @pytest.mark.asyncio + async def test_embedding_passed_when_configured(self) -> None: + adapter = LettaAdapter( + config=LettaAdapterConfig(embedding="openai/text-embedding-3-small") + ) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + mock_client.agents.create.return_value = make_mock_agent() + + await adapter._create_agent("room-1") + + create_kwargs = mock_client.agents.create.call_args.kwargs + assert create_kwargs["embedding"] == "openai/text-embedding-3-small" + + @pytest.mark.asyncio + async def test_embedding_omitted_by_default(self) -> None: + """Letta Cloud picks its own default — None must not be sent.""" + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + mock_client.agents.create.return_value = make_mock_agent() + + await adapter._create_agent("room-1") + + assert "embedding" not in mock_client.agents.create.call_args.kwargs + + @pytest.mark.asyncio + async def test_per_room_persona_carries_room_id(self) -> None: + """Self-hosted MCP schemas require room_id per call; a per_room agent + gets it in its persona block.""" + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + mock_client.agents.create.return_value = make_mock_agent() + + await adapter._create_agent("room-42") + + blocks = mock_client.agents.create.call_args.kwargs["memory_blocks"] + persona = next(b for b in blocks if b["label"] == "persona") + assert "room-42" in persona["value"] + + @pytest.mark.asyncio + async def test_external_mode_persona_omits_room_id(self) -> None: + """External band-mcp tool schemas carry no room_id argument.""" + adapter = LettaAdapter( + config=LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")) + ) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + mock_client.agents.create.return_value = make_mock_agent() + + await adapter._create_agent("room-42") + + blocks = mock_client.agents.create.call_args.kwargs["memory_blocks"] + persona = next(b for b in blocks if b["label"] == "persona") + assert "room-42" not in persona["value"] diff --git a/tests/adapters/letta/test_autorelaydisabled.py b/tests/adapters/letta/test_autorelaydisabled.py new file mode 100644 index 000000000..7a5bd09a7 --- /dev/null +++ b/tests/adapters/letta/test_autorelaydisabled.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from band.adapters.letta import ( + LettaAdapterConfig, +) +from band.converters.letta import LettaSessionState +from band.core.protocols import ( + TurnResultAlreadyReported, +) +from band.core.types import Capability +from band.runtime.tools import BandTool +from band.testing import ( + MISSING_REPLY_FAILURE, + FakeAgentTools, + failure_reports, + reported_failures, +) +from tests.adapters.letta.kit import REPLY_SETTLING_CALLS, sent_contents +from tests.adapters.lettakit import ( + make_platform_message, + ready_letta_adapter, + scripted_letta_turn, +) +from tests.framework_conformance.turnprobes import ROOM_ID, turn_input, turn_tools + + +class TestAutoRelayDisabled: + @pytest.mark.asyncio + async def test_disabled_relay_drops_the_text_and_the_verdict_reports_it( + self, + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(auto_relay=False), room_id=ROOM_ID + ) + mock_client.agents.messages.create.side_effect = scripted_letta_turn( + adapter, room_id=ROOM_ID, tool_calls=[], final_text="I'll help you!" + ) + + tools = turn_tools() + with pytest.raises(TurnResultAlreadyReported): + await adapter.on_event(turn_input(tools)) + + assert tools.messages_sent == [] + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] + + @pytest.mark.asyncio + async def test_disabled_relay_completes_a_turn_that_did_real_work(self) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(auto_relay=False), room_id=ROOM_ID + ) + mock_client.agents.messages.create.side_effect = scripted_letta_turn( + adapter, + room_id=ROOM_ID, + tool_calls=[(BandTool.CREATE_CHATROOM, {})], + final_text="Created it.", + ) + + tools = turn_tools() + await adapter.on_event(turn_input(tools)) + + assert tools.messages_sent == [] + assert failure_reports(tools) == [] + + @pytest.mark.asyncio + @pytest.mark.parametrize(("reply_tool", "arguments"), REPLY_SETTLING_CALLS) + async def test_disabled_relay_quiet_when_a_reply_tool_settled_the_turn( + self, reply_tool: str, arguments: dict[str, Any] + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(auto_relay=False), capabilities={Capability.FILES} + ) + mock_client.agents.messages.create.side_effect = scripted_letta_turn( + adapter, + room_id="room-1", + tool_calls=[(reply_tool, arguments)], + final_text="Done!", + ) + + tools = FakeAgentTools() + await adapter.on_message( + make_platform_message(), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert "Done!" not in sent_contents(tools) + assert not reported_failures(tools) diff --git a/tests/adapters/letta/test_coldbootseeding.py b/tests/adapters/letta/test_coldbootseeding.py new file mode 100644 index 000000000..175edb21d --- /dev/null +++ b/tests/adapters/letta/test_coldbootseeding.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, + RoomContext, +) +from band.converters.letta import LettaSessionState +from band.core.protocols import ( + TurnResultAlreadyReported, +) +from band.testing import ( + FakeAgentTools, +) +from tests.adapters.lettakit import ( + make_assistant_message, + make_letta_response, + make_mock_agent, + make_mock_tool_page, + make_platform_message, +) + + +class TestColdBootSeeding: + @pytest.fixture + def adapter_with_client(self) -> tuple[LettaAdapter, AsyncMock]: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test prompt" + adapter._mcp.server_id = "mcp-server-1" + adapter._mcp.tool_ids = [] + return adapter, mock_client + + @pytest.mark.asyncio + async def test_new_agent_in_room_with_history_is_seeded( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = adapter_with_client + mock_client.agents.create.return_value = make_mock_agent("fresh-agent") + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Hi again!") + ) + + history = LettaSessionState( + replay_messages=["[Alice]: The secret word is kumquat.", "[Bot]: Noted!"] + ) + tools = FakeAgentTools() + await adapter.on_message( + make_platform_message(content="what was the secret word?"), + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + content = mock_client.agents.messages.create.call_args.kwargs["messages"][0][ + "content" + ] + assert "joining an ongoing conversation" in content + assert "[Alice]: The secret word is kumquat." in content + assert "what was the secret word?" in content + # Seed is delivered exactly once + assert adapter._rooms["room-1"].pending_seed == [] + + @pytest.mark.asyncio + async def test_reported_turn_consumes_delivered_seed(self) -> None: + """A completed provider call consumes its seed before response handling.""" + adapter = LettaAdapter(config=LettaAdapterConfig(auto_relay=False)) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test prompt" + adapter._mcp.server_id = "mcp-server-1" + adapter._mcp.tool_ids = [] + room_ctx = RoomContext( + agent_id="agent-1", pending_seed=["[Alice]: Earlier context"] + ) + adapter._rooms["room-1"] = room_ctx + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("The response was not sent through the tool.") + ) + + await adapter.on_message( + make_platform_message(), + FakeAgentTools(), + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert room_ctx.pending_seed == [] + assert room_ctx.last_interaction is not None + + @pytest.mark.asyncio + async def test_failed_first_turn_preserves_pending_seed( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + """Cold-boot history must survive a failed first turn — clearing the + seed at compose time would drop replay context forever.""" + adapter, mock_client = adapter_with_client + mock_client.agents.create.return_value = make_mock_agent("fresh-agent") + mock_client.agents.messages.create.side_effect = TimeoutError("slow letta") + + history = LettaSessionState( + replay_messages=["[Alice]: The secret word is kumquat."] + ) + tools = FakeAgentTools() + with pytest.raises(TurnResultAlreadyReported): + await adapter.on_message( + make_platform_message(content="what was the secret word?"), + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + assert adapter._rooms["room-1"].pending_seed == [ + "[Alice]: The secret word is kumquat." + ] + + @pytest.mark.asyncio + async def test_per_room_does_not_resume_config_agent_id( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + """config.agent_id is not a per-room resume target — each room without + persisted history must get its own Letta agent.""" + adapter, mock_client = adapter_with_client + adapter.config = adapter.config.model_copy( + update={"agent_id": "shared-bootstrap-id"} + ) + mock_client.agents.create.return_value = make_mock_agent("room-agent") + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Hello!") + ) + + await adapter.on_message( + make_platform_message(room_id="room-1"), + FakeAgentTools(), + LettaSessionState(), + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + mock_client.agents.retrieve.assert_not_called() + mock_client.agents.create.assert_awaited_once() + + @pytest.mark.asyncio + async def test_second_message_is_not_reseeded( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = adapter_with_client + mock_client.agents.create.return_value = make_mock_agent("fresh-agent") + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Hello!") + ) + + history = LettaSessionState(replay_messages=["[Alice]: earlier message"]) + tools = FakeAgentTools() + for _ in range(2): + await adapter.on_message( + make_platform_message(), + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + second_content = mock_client.agents.messages.create.call_args.kwargs[ + "messages" + ][0]["content"] + assert "[Alice]: earlier message" not in second_content + + @pytest.mark.asyncio + async def test_resumed_agent_is_not_seeded( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + """Resume-by-id stays the fast path — the live agent already carries + its own context, so replaying history would duplicate it.""" + adapter, mock_client = adapter_with_client + mock_client.agents.retrieve.return_value = make_mock_agent("live-agent") + mock_client.agents.tools.list.return_value = make_mock_tool_page() + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Resumed!") + ) + + history = LettaSessionState( + agent_id="live-agent", + replay_messages=["[Alice]: earlier message"], + ) + tools = FakeAgentTools() + await adapter.on_message( + make_platform_message(), + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + content = mock_client.agents.messages.create.call_args.kwargs["messages"][0][ + "content" + ] + assert "[Alice]: earlier message" not in content + mock_client.agents.create.assert_not_called() diff --git a/tests/adapters/letta/test_configenvsourcing.py b/tests/adapters/letta/test_configenvsourcing.py new file mode 100644 index 000000000..40233104c --- /dev/null +++ b/tests/adapters/letta/test_configenvsourcing.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import pytest + +from band.adapters.letta import ( + LettaAdapterConfig, +) + + +class TestConfigEnvSourcing: + """provider_key env sourcing: LETTA_* names only, never bare vars.""" + + @pytest.fixture(autouse=True) + def clean_env(self, monkeypatch: pytest.MonkeyPatch) -> None: + for var in ("API_KEY", "PROVIDER_KEY", "LETTA_API_KEY", "LETTA_PROVIDER_KEY"): + monkeypatch.delenv(var, raising=False) + + def test_bare_env_vars_never_populate_provider_key( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A foreign secret in API_KEY/PROVIDER_KEY must not reach Letta.""" + monkeypatch.setenv("API_KEY", "foreign-secret") + monkeypatch.setenv("PROVIDER_KEY", "foreign-secret") + + assert LettaAdapterConfig().provider_key is None + + def test_letta_api_key_env_populates_provider_key( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LETTA_API_KEY", "cloud-key") + + assert LettaAdapterConfig().provider_key == "cloud-key" + + def test_prefixed_field_name_env_populates_provider_key( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LETTA_PROVIDER_KEY", "prefixed-key") + + assert LettaAdapterConfig().provider_key == "prefixed-key" + + def test_explicit_kwarg_wins_over_env( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LETTA_API_KEY", "env-key") + + assert LettaAdapterConfig(provider_key="kwarg-key").provider_key == "kwarg-key" + + def test_prefixed_env_fills_a_field_and_an_explicit_value_beats_it( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("LETTA_MODEL", "openai/env-model") + + assert LettaAdapterConfig().model == "openai/env-model" + assert LettaAdapterConfig(model="openai/explicit").model == "openai/explicit" diff --git a/tests/adapters/letta/test_deleteagentsoncleanup.py b/tests/adapters/letta/test_deleteagentsoncleanup.py new file mode 100644 index 000000000..e5a2cf5b3 --- /dev/null +++ b/tests/adapters/letta/test_deleteagentsoncleanup.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, + RoomContext, +) + + +class TestDeleteAgentsOnCleanup: + @pytest.mark.asyncio + async def test_opt_in_deletes_agent_instead_of_consolidating(self) -> None: + adapter = LettaAdapter(config=LettaAdapterConfig(delete_agents_on_cleanup=True)) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + await adapter.on_cleanup("room-1") + + mock_client.agents.delete.assert_awaited_once_with("agent-1") + # No consolidation prompt when the agent is being deleted + mock_client.agents.messages.create.assert_not_called() + + @pytest.mark.asyncio + async def test_default_keeps_agent(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + await adapter.on_cleanup("room-1") + + mock_client.agents.delete.assert_not_called() + + @pytest.mark.asyncio + async def test_can_skip_consolidation_on_cleanup(self) -> None: + adapter = LettaAdapter( + config=LettaAdapterConfig(consolidate_memory_on_cleanup=False) + ) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + await adapter.on_cleanup("room-1") + + mock_client.agents.delete.assert_not_called() + mock_client.agents.messages.create.assert_not_called() + assert "room-1" not in adapter._rooms + + @pytest.mark.asyncio + async def test_delete_failure_does_not_propagate(self) -> None: + adapter = LettaAdapter(config=LettaAdapterConfig(delete_agents_on_cleanup=True)) + mock_client = AsyncMock() + adapter._client = mock_client + mock_client.agents.delete.side_effect = Exception("gone already") + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + await adapter.on_cleanup("room-1") # should not raise + assert "room-1" not in adapter._rooms diff --git a/tests/adapters/letta/test_executionreporting.py b/tests/adapters/letta/test_executionreporting.py new file mode 100644 index 000000000..6ae667c07 --- /dev/null +++ b/tests/adapters/letta/test_executionreporting.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +import json +from unittest.mock import AsyncMock + +import pytest +from letta_client.types.agents.text_content import TextContent +from letta_client.types.agents.tool_call import ToolCall +from letta_client.types.agents.tool_return import ToolReturn + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, + RoomContext, +) +from band.converters.letta import LettaSessionState +from band.core.types import Emit +from band.runtime.tools import BandTool +from band.testing import ( + FakeAgentTools, +) +from tests.adapters.letta.kit import reported_tool_names +from tests.adapters.lettakit import ( + make_assistant_message, + make_grouped_call_message, + make_grouped_return_message, + make_letta_response, + make_platform_message, + make_tool_call_message, + make_tool_return_message, + ready_letta_adapter, +) + + +class TestExecutionReporting: + @pytest.mark.asyncio + async def test_reports_non_silent_tool_calls(self) -> None: + config = LettaAdapterConfig() + adapter = LettaAdapter(config=config, emit=Emit.TOOL_CALLS) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_call_message("band_lookup_peers", "{}"), + make_tool_return_message("band_lookup_peers", '{"peers": []}'), + make_assistant_message("Done"), + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + tool_call_events = [ + e for e in tools.events_sent if e["message_type"] == "tool_call" + ] + tool_result_events = [ + e for e in tools.events_sent if e["message_type"] == "tool_result" + ] + assert len(tool_call_events) == 1 + assert len(tool_result_events) == 1 + + @pytest.mark.asyncio + async def test_reports_every_parallel_call_and_its_result(self) -> None: + adapter, mock_client = ready_letta_adapter(emit=Emit.TOOL_CALLS) + calls = [ + ToolCall(name=BandTool.GET_PARTICIPANTS, arguments="{}", tool_call_id="a"), + ToolCall(name=BandTool.LOOKUP_PEERS, arguments="{}", tool_call_id="b"), + ] + multimodal_output = [TextContent(type="text", text="no peers")] + returns = [ + ToolReturn( + type="tool", + status="success", + tool_call_id="b", + tool_return=multimodal_output, + ), + ToolReturn( + type="tool", status="success", tool_call_id="a", tool_return="[]" + ), + ] + mock_client.agents.messages.create.return_value = make_letta_response( + make_grouped_call_message(calls), + make_grouped_return_message(returns, name=calls[0].name), + make_assistant_message("Done"), + ) + tools = FakeAgentTools() + + await adapter.on_message( + make_platform_message(), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert reported_tool_names(tools, "tool_call") == [ + BandTool.GET_PARTICIPANTS, + BandTool.LOOKUP_PEERS, + ] + assert reported_tool_names(tools, "tool_result") == [ + BandTool.LOOKUP_PEERS, + BandTool.GET_PARTICIPANTS, + ] + + @pytest.mark.asyncio + async def test_silent_tools_not_reported(self) -> None: + config = LettaAdapterConfig() + adapter = LettaAdapter(config=config, emit=Emit.TOOL_CALLS) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_call_message("band_send_message"), + make_tool_return_message("band_send_message"), + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + tool_events = [ + e + for e in tools.events_sent + if e["message_type"] in ("tool_call", "tool_result") + ] + assert len(tool_events) == 0 + + @pytest.mark.asyncio + async def test_send_room_file_reports_content_placeholder_not_raw_bytes( + self, + ) -> None: + """ToolCall.arguments is a raw JSON string (letta_client's own wire + shape); the tool_call event must still redact band_send_room_file's + content field, not embed the real file bytes verbatim.""" + config = LettaAdapterConfig() + adapter = LettaAdapter(config=config, emit=Emit.TOOL_CALLS) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_call_message( + "band_send_room_file", + '{"content": "SECRET FILE BYTES", "filename": "f.txt"}', + ), + make_tool_return_message("band_send_room_file", '{"status": "ok"}'), + make_assistant_message("Done"), + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + tool_call_events = [ + e for e in tools.events_sent if e["message_type"] == "tool_call" + ] + assert len(tool_call_events) == 1 + reported_args = json.loads(tool_call_events[0]["content"])["args"] + assert "SECRET FILE BYTES" not in json.dumps(reported_args) + assert "byte file content" in reported_args["content"] + assert reported_args["filename"] == "f.txt" diff --git a/tests/adapters/letta/test_externaltoolrecording.py b/tests/adapters/letta/test_externaltoolrecording.py new file mode 100644 index 000000000..238fdcd86 --- /dev/null +++ b/tests/adapters/letta/test_externaltoolrecording.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Literal + +import pytest +from letta_client.types.tool_return_message import ToolReturnMessage + +from band.adapters.letta import ( + LettaAdapterConfig, + LettaMCPConfig, +) +from band.converters.letta import LettaSessionState +from band.core.protocols import ( + TurnResultAlreadyReported, +) +from band.runtime.tools import BandTool +from band.testing import ( + MISSING_REPLY_FAILURE, + FakeAgentTools, + failure_reports, +) +from tests.adapters.letta.kit import sent_contents +from tests.adapters.lettakit import ( + call_id, + make_assistant_message, + make_letta_response, + make_parallel_tool_messages, + make_platform_message, + make_tool_call_message, + make_tool_return_message, + ready_letta_adapter, +) +from tests.framework_conformance.turnprobes import ROOM_ID, turn_input, turn_tools + + +class TestExternalToolRecording: + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("mcp_mode", "status", "replied"), + [ + pytest.param("external", "success", True, id="external-success"), + pytest.param("external", "error", False, id="external-error"), + # Self-hosted calls run on the room's own tools, which record them. + pytest.param("self_host", "success", False, id="self-host"), + ], + ) + async def test_streamed_tool_return_records_only_for_external_band_mcp( + self, + mcp_mode: Literal["self_host", "external"], + status: Literal["success", "error"], + replied: bool, + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode=mcp_mode)) + ) + mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_call_message(BandTool.SEND_MESSAGE), + make_tool_return_message(BandTool.SEND_MESSAGE, status=status), + ) + tools = FakeAgentTools() + + await adapter.on_message( + make_platform_message(), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert tools.turn.replied is replied + + @pytest.mark.asyncio + async def test_ungrouped_tool_return_is_matched_to_its_call(self) -> None: + """A server that sends no ``tool_returns`` puts the one result on the + message itself.""" + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")) + ) + mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_call_message(BandTool.SEND_MESSAGE), + ToolReturnMessage( + id="message-return", + date=datetime.now(UTC), + message_type="tool_return_message", + status="success", + tool_call_id=call_id(BandTool.SEND_MESSAGE), + tool_return="result", + ), + ) + tools = FakeAgentTools() + + await adapter.on_message( + make_platform_message(), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert tools.turn.replied + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("effect_tool", "replied"), + [ + pytest.param(BandTool.ADD_PARTICIPANT, False, id="act"), + pytest.param(BandTool.SEND_MESSAGE, True, id="reply"), + pytest.param(BandTool.NO_REPLY, True, id="decline"), + ], + ) + @pytest.mark.parametrize("reverse_calls", [False, True]) + async def test_parallel_success_completes_turn_independent_of_order( + self, + effect_tool: str, + replied: bool, + reverse_calls: bool, + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), + room_id=ROOM_ID, + ) + results: list[tuple[str, Literal["success", "error"]]] = [ + (BandTool.GET_PARTICIPANTS, "success"), + (effect_tool, "success"), + ] + if reverse_calls: + results.reverse() + mock_client.agents.messages.create.return_value = make_letta_response( + *make_parallel_tool_messages(results, reverse_returns=False) + ) + tools = turn_tools() + + await adapter.on_event(turn_input(tools)) + + assert tools.turn.complete + assert tools.turn.replied is replied + assert tools.messages_sent == [] + assert failure_reports(tools) == [] + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "effect_tool", + [BandTool.ADD_PARTICIPANT, BandTool.SEND_MESSAGE, BandTool.NO_REPLY], + ) + @pytest.mark.parametrize("reverse_returns", [False, True]) + async def test_parallel_failed_effect_does_not_complete_turn( + self, effect_tool: str, reverse_returns: bool + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), + room_id=ROOM_ID, + ) + mock_client.agents.messages.create.return_value = make_letta_response( + *make_parallel_tool_messages( + [ + (BandTool.GET_PARTICIPANTS, "success"), + (effect_tool, "error"), + ], + reverse_returns=reverse_returns, + ) + ) + tools = turn_tools() + + with pytest.raises(TurnResultAlreadyReported): + await adapter.on_event(turn_input(tools)) + + assert not tools.turn.replied + assert tools.messages_sent == [] + assert failure_reports(tools) == [MISSING_REPLY_FAILURE] + + @pytest.mark.asyncio + @pytest.mark.parametrize("effect_tool", [BandTool.SEND_MESSAGE, BandTool.NO_REPLY]) + @pytest.mark.parametrize("reverse_returns", [False, True]) + async def test_parallel_failed_reply_does_not_suppress_fallback( + self, effect_tool: str, reverse_returns: bool + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")), + room_id=ROOM_ID, + ) + mock_client.agents.messages.create.return_value = make_letta_response( + *make_parallel_tool_messages( + [(effect_tool, "error"), (BandTool.GET_PARTICIPANTS, "success")], + reverse_returns=reverse_returns, + ), + make_assistant_message("Fallback reply"), + ) + tools = turn_tools() + + await adapter.on_event(turn_input(tools)) + + assert sent_contents(tools) == ["Fallback reply"] + assert failure_reports(tools) == [] + + @pytest.mark.asyncio + @pytest.mark.parametrize("reverse_returns", [False, True]) + async def test_parallel_results_match_call_ids_not_positions( + self, reverse_returns: bool + ) -> None: + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), + room_id=ROOM_ID, + ) + mock_client.agents.messages.create.return_value = make_letta_response( + *make_parallel_tool_messages( + [ + (BandTool.SEND_MESSAGE, "error"), + (BandTool.ADD_PARTICIPANT, "success"), + ], + reverse_returns=reverse_returns, + ) + ) + tools = turn_tools() + + await adapter.on_event(turn_input(tools)) + + assert tools.turn.complete + assert not tools.turn.replied + assert tools.messages_sent == [] + assert failure_reports(tools) == [] diff --git a/tests/adapters/letta/test_extractsummary.py b/tests/adapters/letta/test_extractsummary.py new file mode 100644 index 000000000..500c2d986 --- /dev/null +++ b/tests/adapters/letta/test_extractsummary.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from band.adapters.letta import ( + LettaAdapter, +) + + +class TestExtractSummary: + def test_short_text_kept_whole(self) -> None: + parts = ["Hello there. This is more text."] + assert "Hello there. This is more text." == LettaAdapter._extract_summary(parts) + + def test_truncates_on_word_boundary(self) -> None: + parts = ["alpha beta " * 30] + result = LettaAdapter._extract_summary(parts, max_length=50) + assert len(result) <= 53 # 50 + "..." + assert result.endswith("...") + assert not result.removesuffix("...").endswith("alph") # no mid-word cut + + def test_no_sentence_heuristics(self) -> None: + """Decimals must not be mistaken for sentence ends.""" + parts = ["pi is 3.14 and e is 2.71"] + assert "pi is 3.14 and e is 2.71" == LettaAdapter._extract_summary(parts) + + def test_empty(self) -> None: + assert "" == LettaAdapter._extract_summary([]) + + def test_multiple_parts_joined(self) -> None: + parts = ["First part.", "Second part."] + assert "First part. Second part." == LettaAdapter._extract_summary(parts) diff --git a/tests/adapters/letta/test_formattimeago.py b/tests/adapters/letta/test_formattimeago.py new file mode 100644 index 000000000..d1965a3c4 --- /dev/null +++ b/tests/adapters/letta/test_formattimeago.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from band.adapters.letta import ( + LettaAdapter, +) + + +class TestFormatTimeAgo: + def test_seconds(self) -> None: + dt = datetime.now(UTC) - timedelta(seconds=30) + assert "30s" == LettaAdapter._format_time_ago(dt) + + def test_minutes(self) -> None: + dt = datetime.now(UTC) - timedelta(minutes=5) + assert "5m" == LettaAdapter._format_time_ago(dt) + + def test_hours(self) -> None: + dt = datetime.now(UTC) - timedelta(hours=3) + assert "3h" == LettaAdapter._format_time_ago(dt) + + def test_one_hour(self) -> None: + dt = datetime.now(UTC) - timedelta(hours=1) + assert "1 hour" == LettaAdapter._format_time_ago(dt) + + def test_days(self) -> None: + dt = datetime.now(UTC) - timedelta(days=5) + assert "5d" == LettaAdapter._format_time_ago(dt) + + def test_one_day(self) -> None: + dt = datetime.now(UTC) - timedelta(days=1) + assert "1 day" == LettaAdapter._format_time_ago(dt) + + def test_naive_datetime_treated_as_utc(self) -> None: + dt = datetime.now(UTC).replace(tzinfo=None) - timedelta(minutes=10) + result = LettaAdapter._format_time_ago(dt) + assert "10m" == result diff --git a/tests/adapters/letta/test_instructionblockupdate.py b/tests/adapters/letta/test_instructionblockupdate.py new file mode 100644 index 000000000..f3f0cb1f2 --- /dev/null +++ b/tests/adapters/letta/test_instructionblockupdate.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, +) +from tests.adapters.lettakit import ( + default_enforcement, +) + + +class TestInstructionBlockUpdate: + @pytest.mark.asyncio + async def test_updates_persona_block(self) -> None: + adapter = LettaAdapter() + adapter._system_prompt = "Test system prompt" + mock_client = AsyncMock() + adapter._client = mock_client + + await adapter._update_instruction_block("agent-1", "room-1") + + # Default config self-hosts the MCP server in per_room mode, so the + # enforcement carries the room id the tool schemas require. + mock_client.agents.blocks.update.assert_called_once_with( + "persona", + agent_id="agent-1", + value=default_enforcement(room_id="room-1") + "Test system prompt", + ) + + @pytest.mark.asyncio + async def test_falls_back_to_next_label(self) -> None: + adapter = LettaAdapter() + adapter._system_prompt = "Test prompt" + mock_client = AsyncMock() + adapter._client = mock_client + + # First label fails, second succeeds + mock_client.agents.blocks.update.side_effect = [ + Exception("not found"), + None, + None, + ] + + await adapter._update_instruction_block("agent-1", "room-1") + + assert mock_client.agents.blocks.update.call_count == 2 + second_call = mock_client.agents.blocks.update.call_args_list[1] + assert second_call.args[0] == "custom_instructions" + + @pytest.mark.asyncio + async def test_creates_persona_when_all_labels_fail(self) -> None: + adapter = LettaAdapter() + adapter._system_prompt = "Test prompt" + mock_client = AsyncMock() + adapter._client = mock_client + + mock_client.agents.blocks.update.side_effect = Exception("not found") + mock_block = MagicMock() + mock_block.id = "block-1" + mock_client.blocks.create.return_value = mock_block + + await adapter._update_instruction_block("agent-1", "room-1") + + mock_client.blocks.create.assert_called_once() + assert mock_client.blocks.create.call_args.kwargs["label"] == "persona" + mock_client.agents.blocks.attach.assert_called_once_with( + "block-1", + agent_id="agent-1", + ) diff --git a/tests/adapters/letta/test_lettaadapterinit.py b/tests/adapters/letta/test_lettaadapterinit.py new file mode 100644 index 000000000..99a5b535e --- /dev/null +++ b/tests/adapters/letta/test_lettaadapterinit.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, + LettaMCPConfig, +) + + +class TestLettaAdapterInit: + # Default values are not asserted here: that would only restate the + # dataclass definition, and the frozen-config conformance suite already + # pins ``LettaAdapter().config == LettaAdapterConfig()``. + + def test_custom_config_reaches_adapter(self) -> None: + config = LettaAdapterConfig( + base_url="http://custom:8283", + provider_key="sk-test", + mode="shared", + mcp=LettaMCPConfig(mode="external", server_url="http://mcp:9000/sse"), + ) + adapter = LettaAdapter(config=config) + assert adapter.config is config + + def test_no_leaked_adapter_config_env_vars( + self, assert_no_leaked_adapter_config_env: None + ) -> None: + """Requesting the fixture is the assertion — see its docstring.""" diff --git a/tests/adapters/letta/test_lettaadapteroncleanup.py b/tests/adapters/letta/test_lettaadapteroncleanup.py new file mode 100644 index 000000000..b2ffff17b --- /dev/null +++ b/tests/adapters/letta/test_lettaadapteroncleanup.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + RoomContext, +) + + +class TestLettaAdapterOnCleanup: + @pytest.mark.asyncio + async def test_cleanup_removes_room_state(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + await adapter.on_cleanup("room-1") + + assert "room-1" not in adapter._rooms + # Memory consolidation was attempted + mock_client.agents.messages.create.assert_called_once() + + @pytest.mark.asyncio + async def test_cleanup_idempotent(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + + # Room doesn't exist + await adapter.on_cleanup("nonexistent") + mock_client.agents.messages.create.assert_not_called() + + @pytest.mark.asyncio + async def test_cleanup_twice(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + await adapter.on_cleanup("room-1") + await adapter.on_cleanup("room-1") # Should not raise + + assert "room-1" not in adapter._rooms + + @pytest.mark.asyncio + async def test_cleanup_multi_room(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + adapter._rooms["room-2"] = RoomContext(agent_id="agent-2") + + await adapter.on_cleanup("room-1") + + assert "room-1" not in adapter._rooms + assert "room-2" in adapter._rooms + + @pytest.mark.asyncio + async def test_cleanup_without_client(self) -> None: + adapter = LettaAdapter() + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + # No client — should not raise + await adapter.on_cleanup("room-1") + assert "room-1" not in adapter._rooms + + @pytest.mark.asyncio + async def test_cleanup_before_started(self) -> None: + adapter = LettaAdapter() + # Completely uninitialized — no client, no rooms + await adapter.on_cleanup("room-1") + assert "room-1" not in adapter._rooms diff --git a/tests/adapters/letta/test_lettaadapteronmessageperroom.py b/tests/adapters/letta/test_lettaadapteronmessageperroom.py new file mode 100644 index 000000000..ea6b9c0a4 --- /dev/null +++ b/tests/adapters/letta/test_lettaadapteronmessageperroom.py @@ -0,0 +1,440 @@ +from __future__ import annotations + +import asyncio +import logging +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, + RoomContext, +) +from band.converters.letta import LettaSessionState +from band.core.protocols import ( + GENERIC_PROVIDER_FAILURE_MESSAGE, + TurnResultAlreadyReported, +) +from band.core.types import Capability, Emit +from band.testing import ( + FakeAgentTools, + reported_failures, +) +from tests.adapters.letta.kit import REPLY_SETTLING_CALLS, sent_contents +from tests.adapters.lettakit import ( + make_assistant_message, + make_letta_response, + make_mock_agent, + make_mock_tool_page, + make_platform_message, + make_tool_call_message, + make_tool_return_message, + ready_letta_adapter, + scripted_letta_turn, +) + + +class TestLettaAdapterOnMessagePerRoom: + """Tests for per_room mode message handling.""" + + @pytest.fixture + def adapter_with_client(self) -> tuple[LettaAdapter, AsyncMock]: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test prompt" + adapter._mcp.tool_ids = ["t1", "t2"] + adapter._mcp.server_id = "mcp-server-1" + return adapter, mock_client + + @pytest.mark.asyncio + async def test_basic_message_creates_agent_and_sends( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = adapter_with_client + + mock_agent = make_mock_agent("agent-1") + mock_client.agents.create.return_value = mock_agent + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Hello!") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + # Agent was created + mock_client.agents.create.assert_called_once() + # MCP tools were attached + assert mock_client.agents.tools.attach.call_count == 2 + # Message was sent via direct agent API (not conversations) + mock_client.agents.messages.create.assert_called_once() + call_kwargs = mock_client.agents.messages.create.call_args.kwargs + assert call_kwargs["agent_id"] == "agent-1" + assert "conversation_id" not in call_kwargs + + @pytest.mark.asyncio + async def test_configured_memory_blocks_follow_the_persona_block( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = adapter_with_client + human = {"label": "human", "value": "Prefers short answers."} + adapter.config = LettaAdapterConfig(memory_blocks=(human,)) + mock_client.agents.create.return_value = make_mock_agent("agent-1") + + await adapter._create_agent("room-1") + + blocks = mock_client.agents.create.call_args.kwargs["memory_blocks"] + assert [block["label"] for block in blocks] == ["persona", "human"] + assert blocks[1] == human + + @pytest.mark.asyncio + async def test_auto_relay_when_no_send_message( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = adapter_with_client + + # Setup room with agent + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("I'll help you!") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + # Auto-relay should have sent the message + assert len(tools.messages_sent) == 1 + assert tools.messages_sent[0]["content"] == "I'll help you!" + + @pytest.mark.asyncio + async def test_send_message_failure_is_not_reported_as_provider_failure( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + """The Letta agent answered fine; the room POST is what failed. That + must not surface as a Letta AgentFailure -- deliver_reply's + DeliveryFailedError must be recognized and left unreported here.""" + adapter, mock_client = adapter_with_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("I'll help you!") + ) + + tools = FakeAgentTools() + + async def _raise(*args: Any, **kwargs: Any) -> None: + raise RuntimeError("platform rejected the message") + + tools.send_message = _raise # type: ignore[method-assign] + + msg = make_platform_message() + history = LettaSessionState() + + with pytest.raises(RuntimeError, match="platform rejected the message"): + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert not reported_failures(tools) + + @pytest.mark.asyncio + @pytest.mark.parametrize(("reply_tool", "arguments"), REPLY_SETTLING_CALLS) + async def test_skip_auto_relay_when_a_reply_tool_settled_the_turn( + self, reply_tool: str, arguments: dict[str, Any] + ) -> None: + adapter, mock_client = ready_letta_adapter(capabilities={Capability.FILES}) + mock_client.agents.messages.create.side_effect = scripted_letta_turn( + adapter, + room_id="room-1", + tool_calls=[(reply_tool, arguments)], + final_text="Done!", + ) + tools = FakeAgentTools() + + await adapter.on_message( + make_platform_message(), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert "Done!" not in sent_contents(tools) + + @pytest.mark.asyncio + async def test_timeout_reports_error( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = adapter_with_client + adapter.config = adapter.config.model_copy(update={"turn_timeout_s": 0.01}) + + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + async def slow_response(**kwargs: Any) -> MagicMock: + await asyncio.sleep(1) + return make_letta_response() + + mock_client.agents.messages.create.side_effect = slow_response + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + with pytest.raises(TurnResultAlreadyReported): + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + failures = reported_failures(tools) + assert len(failures) == 1 + assert "timed out" in failures[0]["message"] + assert failures[0]["provider"] == "letta" + assert failures[0]["code"] == "timeout" + + @pytest.mark.asyncio + async def test_slow_delivery_is_not_misreported_as_provider_timeout( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + """turn_timeout_s bounds only the Letta round-trip. A room POST that + is merely slow (not the Letta call) must not be misreported as a + Letta provider timeout -- it must be given time to complete.""" + adapter, mock_client = adapter_with_client + adapter.config = adapter.config.model_copy(update={"turn_timeout_s": 0.05}) + + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + # The Letta call itself responds instantly, well inside turn_timeout_s. + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("I'll help you!") + ) + + tools = FakeAgentTools() + real_send_message = tools.send_message + + async def _slow_send_message(*args: Any, **kwargs: Any) -> Any: + # Longer than turn_timeout_s: only the provider call may race it. + await asyncio.sleep(0.15) + return await real_send_message(*args, **kwargs) + + tools.send_message = _slow_send_message # type: ignore[method-assign] + + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert len(tools.messages_sent) == 1 + assert not reported_failures(tools) + + @pytest.mark.asyncio + async def test_send_event_timeout_is_not_misreported_as_provider_timeout( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """A TimeoutError from tool-event reporting (send_event) -- not the + Letta round-trip itself -- must fall through to the generic failure + path, never be mislabeled as a Letta provider timeout. (send_failure + itself best-effort-swallows the same broken channel here, same as + production, so the room never receives a failure event either way -- + what this guards is which branch is taken/logged.)""" + config = LettaAdapterConfig() + adapter = LettaAdapter(config=config, emit=Emit.TOOL_CALLS) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + # The Letta round-trip itself returns instantly -- well inside + # turn_timeout_s -- so any TimeoutError must come from elsewhere. + mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_call_message("band_lookup_peers", "{}"), + make_tool_return_message("band_lookup_peers", '{"peers": []}'), + ) + + tools = FakeAgentTools() + tools.send_event_error = TimeoutError("event POST hiccup") + msg = make_platform_message() + history = LettaSessionState() + + with ( + caplog.at_level(logging.ERROR, logger="band.adapters.letta"), + pytest.raises(TimeoutError), + ): + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert not any("timed out" in r.message for r in caplog.records) + assert any("Error during Letta turn" in r.message for r in caplog.records) + + @pytest.mark.asyncio + async def test_generic_exception_reports_and_propagates( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + """A bare exception from the Letta client (not a timeout, not a + delivery failure) must still surface via the generic fallback.""" + adapter, mock_client = adapter_with_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + mock_client.agents.messages.create.side_effect = ConnectionError( + "letta connection reset" + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + with pytest.raises(ConnectionError, match="letta connection reset"): + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + failure = reported_failures(tools)[0] + assert failure["provider"] == "letta" + assert failure["message"] == GENERIC_PROVIDER_FAILURE_MESSAGE + assert "letta connection reset" not in failure["message"] + + @pytest.mark.asyncio + async def test_participants_and_contacts_injected( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = adapter_with_client + + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Got it.") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + "Participants: Alice, Bob", + "Contacts: Charlie", + is_session_bootstrap=False, + room_id="room-1", + ) + + call_kwargs = mock_client.agents.messages.create.call_args.kwargs + content = call_kwargs["messages"][0]["content"] + assert "[System]: Participants: Alice, Bob" in content + assert "[System]: Contacts: Charlie" in content + + @pytest.mark.asyncio + async def test_agent_resume_from_history( + self, adapter_with_client: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = adapter_with_client + + mock_client.agents.retrieve.return_value = make_mock_agent("history-agent") + mock_client.agents.tools.list.return_value = make_mock_tool_page() + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Resumed!") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState(agent_id="history-agent") + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + # Agent was retrieved, not created + mock_client.agents.retrieve.assert_called_once_with("history-agent") + mock_client.agents.create.assert_not_called() + + @pytest.mark.asyncio + async def test_uninitialized_client_reports_error(self) -> None: + adapter = LettaAdapter() + # _client is None + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + with pytest.raises(RuntimeError, match="not initialized"): + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + failures = reported_failures(tools) + assert len(failures) == 1 + assert failures[0]["provider"] == "letta" + assert "not initialized" in failures[0]["message"] diff --git a/tests/adapters/letta/test_lettaadaptersharedmode.py b/tests/adapters/letta/test_lettaadaptersharedmode.py new file mode 100644 index 000000000..f8323225c --- /dev/null +++ b/tests/adapters/letta/test_lettaadaptersharedmode.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, + RoomContext, +) +from band.converters.letta import LettaSessionState +from band.testing import ( + FakeAgentTools, +) +from tests.adapters.lettakit import ( + make_assistant_message, + make_mock_agent, + make_mock_async_stream, + make_mock_conversation, + make_mock_tool_page, + make_platform_message, +) + + +class TestLettaAdapterSharedMode: + """Tests for shared mode with Conversations API.""" + + @pytest.fixture + def shared_adapter(self) -> tuple[LettaAdapter, AsyncMock]: + config = LettaAdapterConfig(mode="shared") + adapter = LettaAdapter(config=config) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test prompt" + adapter._mcp.tool_ids = ["t1", "t2"] + adapter._mcp.server_id = "mcp-server-1" + return adapter, mock_client + + @pytest.mark.asyncio + async def test_shared_mode_creates_agent_and_conversation( + self, shared_adapter: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = shared_adapter + + mock_agent = make_mock_agent("shared-agent") + mock_client.agents.create.return_value = mock_agent + mock_conv = make_mock_conversation("conv-room1") + mock_client.conversations.create.return_value = mock_conv + mock_client.conversations.messages.create.return_value = make_mock_async_stream( + make_assistant_message("Hi from shared!") + ) + + tools = FakeAgentTools() + msg = make_platform_message(room_id="room-1") + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + # One agent created + mock_client.agents.create.assert_called_once() + # Conversation created for this room + mock_client.conversations.create.assert_called_once_with( + agent_id="shared-agent", + ) + # Message sent through room-scoped conversation endpoint + call_kwargs = mock_client.conversations.messages.create.call_args.kwargs + assert call_kwargs["conversation_id"] == "conv-room1" + assert adapter._shared_agent_id == "shared-agent" + + @pytest.mark.asyncio + async def test_shared_mode_reuses_agent_for_second_room( + self, shared_adapter: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = shared_adapter + + # Setup: first room already connected + adapter._shared_agent_id = "shared-agent" + adapter._rooms["room-1"] = RoomContext( + agent_id="shared-agent", conversation_id="conv-1" + ) + + mock_conv2 = make_mock_conversation("conv-room2") + mock_client.conversations.create.return_value = mock_conv2 + mock_client.conversations.messages.create.return_value = make_mock_async_stream( + make_assistant_message("Hi room 2!") + ) + + tools = FakeAgentTools() + msg = make_platform_message(room_id="room-2") + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-2", + ) + + # Agent was NOT created again + mock_client.agents.create.assert_not_called() + # But a new conversation was created + mock_client.conversations.create.assert_called_once_with( + agent_id="shared-agent", + ) + assert adapter._rooms["room-2"].conversation_id == "conv-room2" + + @pytest.mark.asyncio + async def test_shared_mode_resumes_existing_agent( + self, shared_adapter: tuple[LettaAdapter, AsyncMock] + ) -> None: + adapter, mock_client = shared_adapter + adapter.config = adapter.config.model_copy( + update={"agent_id": "pre-existing-agent"} + ) + + mock_client.agents.retrieve.return_value = make_mock_agent("pre-existing-agent") + mock_client.agents.tools.list.return_value = make_mock_tool_page() + mock_conv = make_mock_conversation("conv-1") + mock_client.conversations.create.return_value = mock_conv + mock_client.conversations.messages.create.return_value = make_mock_async_stream( + make_assistant_message("Resumed shared!") + ) + + tools = FakeAgentTools() + msg = make_platform_message(room_id="room-1") + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + mock_client.agents.retrieve.assert_called_once_with("pre-existing-agent") + mock_client.agents.create.assert_not_called() + assert adapter._shared_agent_id == "pre-existing-agent" + + @pytest.mark.asyncio + async def test_shared_mode_resumes_persisted_conversation( + self, shared_adapter: tuple[LettaAdapter, AsyncMock] + ) -> None: + """A restart with a persisted conversation reuses it — a fresh one + would silently drop the room's conversational context.""" + adapter, mock_client = shared_adapter + adapter._shared_agent_id = "shared-agent" + + mock_client.conversations.retrieve.return_value = make_mock_conversation( + "conv-persisted", agent_id="shared-agent" + ) + mock_client.conversations.messages.create.return_value = make_mock_async_stream( + make_assistant_message("Still here!") + ) + + history = LettaSessionState( + agent_id="shared-agent", + conversation_id="conv-persisted", + replay_messages=["[Alice]: earlier"], + ) + tools = FakeAgentTools() + await adapter.on_message( + make_platform_message(room_id="room-1"), + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + mock_client.conversations.retrieve.assert_awaited_once_with("conv-persisted") + mock_client.conversations.create.assert_not_called() + assert adapter._rooms["room-1"].conversation_id == "conv-persisted" + # A resumed conversation already has its context — no seeding. + content = mock_client.conversations.messages.create.call_args.kwargs[ + "messages" + ][0]["content"] + assert "[Alice]: earlier" not in content + + @pytest.mark.asyncio + async def test_shared_mode_rejects_foreign_conversation( + self, shared_adapter: tuple[LettaAdapter, AsyncMock] + ) -> None: + """A persisted conversation must belong to the current shared agent.""" + adapter, mock_client = shared_adapter + adapter._shared_agent_id = "shared-agent" + + mock_client.conversations.retrieve.return_value = make_mock_conversation( + "conv-other", agent_id="other-agent" + ) + mock_client.conversations.create.return_value = make_mock_conversation( + "conv-fresh", agent_id="shared-agent" + ) + mock_client.conversations.messages.create.return_value = make_mock_async_stream( + make_assistant_message("Fresh thread") + ) + + history = LettaSessionState( + agent_id="shared-agent", + conversation_id="conv-other", + ) + await adapter.on_message( + make_platform_message(room_id="room-1"), + FakeAgentTools(), + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + mock_client.conversations.create.assert_awaited_once_with( + agent_id="shared-agent" + ) + assert adapter._rooms["room-1"].conversation_id == "conv-fresh" + + @pytest.mark.asyncio + async def test_shared_mode_seeds_fresh_conversation_from_history( + self, shared_adapter: tuple[LettaAdapter, AsyncMock] + ) -> None: + """A dead persisted conversation falls back to create + seed.""" + adapter, mock_client = shared_adapter + adapter._shared_agent_id = "shared-agent" + + mock_client.conversations.retrieve.side_effect = Exception("gone") + mock_client.conversations.create.return_value = make_mock_conversation( + "conv-fresh" + ) + mock_client.conversations.messages.create.return_value = make_mock_async_stream( + make_assistant_message("Rebuilt!") + ) + + history = LettaSessionState( + agent_id="shared-agent", + conversation_id="conv-dead", + replay_messages=["[Alice]: the secret is kumquat"], + ) + tools = FakeAgentTools() + await adapter.on_message( + make_platform_message(room_id="room-1"), + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + assert adapter._rooms["room-1"].conversation_id == "conv-fresh" + content = mock_client.conversations.messages.create.call_args.kwargs[ + "messages" + ][0]["content"] + assert "[Alice]: the secret is kumquat" in content + + @pytest.mark.asyncio + async def test_shared_mode_injects_room_id_per_message( + self, shared_adapter: tuple[LettaAdapter, AsyncMock] + ) -> None: + """Self-host + shared: the (shared) persona cannot carry a room id, so + every message reminds the agent which room_id to pass to tools.""" + adapter, mock_client = shared_adapter + adapter._shared_agent_id = "shared-agent" + adapter._rooms["room-42"] = RoomContext( + agent_id="shared-agent", conversation_id="conv-1" + ) + + mock_client.conversations.messages.create.return_value = make_mock_async_stream( + make_assistant_message("Hi!") + ) + + tools = FakeAgentTools() + await adapter.on_message( + make_platform_message(room_id="room-42"), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-42", + ) + + content = mock_client.conversations.messages.create.call_args.kwargs[ + "messages" + ][0]["content"] + assert "Current chat_id: room-42" in content diff --git a/tests/adapters/letta/test_memoryconsolidation.py b/tests/adapters/letta/test_memoryconsolidation.py new file mode 100644 index 000000000..b49139c80 --- /dev/null +++ b/tests/adapters/letta/test_memoryconsolidation.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + RoomContext, +) + + +class TestMemoryConsolidation: + @pytest.mark.asyncio + async def test_consolidation_on_cleanup(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + await adapter.on_cleanup("room-1") + + mock_client.agents.messages.create.assert_called_once() + call_kwargs = mock_client.agents.messages.create.call_args.kwargs + assert "Consolidate" in call_kwargs["messages"][0]["content"] + + @pytest.mark.asyncio + async def test_consolidation_failure_does_not_propagate(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + mock_client.agents.messages.create.side_effect = Exception("API error") + + # Should not raise + await adapter.on_cleanup("room-1") + assert "room-1" not in adapter._rooms diff --git a/tests/adapters/letta/test_rejoincontext.py b/tests/adapters/letta/test_rejoincontext.py new file mode 100644 index 000000000..4c564cae4 --- /dev/null +++ b/tests/adapters/letta/test_rejoincontext.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + RoomContext, +) +from band.converters.letta import LettaSessionState +from band.testing import ( + FakeAgentTools, +) +from tests.adapters.lettakit import ( + make_assistant_message, + make_letta_response, + make_platform_message, +) + + +class TestRejoinContext: + @pytest.mark.asyncio + async def test_rejoin_injects_time_away(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + + last_time = datetime.now(UTC) - timedelta(hours=2) + adapter._rooms["room-1"] = RoomContext( + agent_id="agent-1", + last_interaction=last_time, + summary="Discussed project plan", + ) + + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("I'm back!") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + call_kwargs = mock_client.agents.messages.create.call_args.kwargs + content = call_kwargs["messages"][0]["content"] + assert "rejoined" in content + assert "2h" in content + assert "Discussed project plan" in content diff --git a/tests/adapters/letta/test_sendtoolresolution.py b/tests/adapters/letta/test_sendtoolresolution.py new file mode 100644 index 000000000..0ab2d06fe --- /dev/null +++ b/tests/adapters/letta/test_sendtoolresolution.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, + LettaMCPConfig, +) +from band.converters.letta import LettaSessionState +from band.testing import ( + FakeAgentTools, +) +from tests.adapters.lettakit import ( + make_assistant_message, + make_letta_response, + make_platform_message, + make_tool_call_message, + make_tool_return_message, + ready_letta_adapter, +) + + +class TestSendToolResolution: + def test_resolves_external_band_mcp_names(self) -> None: + adapter = LettaAdapter() + adapter._mcp.resolve_send_tools( + ["create_agent_chat_message", "create_agent_chat_event", "health_check"] + ) + assert adapter._mcp.send_message_tool == "create_agent_chat_message" + assert adapter._mcp.send_event_tool == "create_agent_chat_event" + assert adapter._mcp.silent_reporting_tools == { + "create_agent_chat_message", + "create_agent_chat_event", + } + + def test_prefers_band_names_when_both_present(self) -> None: + adapter = LettaAdapter() + adapter._mcp.resolve_send_tools( + ["band_send_message", "create_agent_chat_message", "band_send_event"] + ) + assert adapter._mcp.send_message_tool == "band_send_message" + assert adapter._mcp.send_event_tool == "band_send_event" + + def test_falls_back_to_band_names_when_none_discovered(self) -> None: + adapter = LettaAdapter() + adapter._mcp.resolve_send_tools(["some_unrelated_tool"]) + assert adapter._mcp.send_message_tool == "band_send_message" + assert adapter._mcp.send_event_tool == "band_send_event" + + def test_enforcement_prompt_uses_resolved_names(self) -> None: + adapter = LettaAdapter() + adapter._system_prompt = "Base prompt" + adapter._mcp.resolve_send_tools( + ["create_agent_chat_message", "create_agent_chat_event"] + ) + text = adapter._instruction_text("room-1") + assert "create_agent_chat_message" in text + assert "band_send_message" not in text + + @pytest.mark.asyncio + async def test_relay_detection_uses_resolved_name(self) -> None: + """A send via the resolved (external) tool name suppresses auto-relay.""" + adapter, mock_client = ready_letta_adapter( + LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")) + ) + adapter._mcp.resolve_send_tools( + ["create_agent_chat_message", "create_agent_chat_event"] + ) + mock_client.agents.messages.create.return_value = make_letta_response( + make_tool_call_message("create_agent_chat_message"), + make_tool_return_message("create_agent_chat_message"), + make_assistant_message("Done!"), + ) + + tools = FakeAgentTools() + await adapter.on_message( + make_platform_message(), + tools, + LettaSessionState(), + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + assert len(tools.messages_sent) == 0 diff --git a/tests/adapters/letta/test_summarystorage.py b/tests/adapters/letta/test_summarystorage.py new file mode 100644 index 000000000..7a5d2f7b3 --- /dev/null +++ b/tests/adapters/letta/test_summarystorage.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + RoomContext, +) +from band.converters.letta import LettaSessionState +from band.testing import ( + FakeAgentTools, +) +from tests.adapters.lettakit import ( + make_assistant_message, + make_letta_response, + make_platform_message, +) + + +class TestSummaryStorage: + @pytest.mark.asyncio + async def test_summary_stored_after_turn(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") + + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("The weather is sunny. More details follow.") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=False, + room_id="room-1", + ) + + room_ctx = adapter._rooms["room-1"] + assert room_ctx.summary == "The weather is sunny. More details follow." + assert room_ctx.last_interaction is not None diff --git a/tests/adapters/letta/test_taskevents.py b/tests/adapters/letta/test_taskevents.py new file mode 100644 index 000000000..93206615f --- /dev/null +++ b/tests/adapters/letta/test_taskevents.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from band.adapters.letta import ( + LettaAdapter, + LettaAdapterConfig, +) +from band.converters.letta import LettaSessionState +from band.testing import ( + FakeAgentTools, +) +from tests.adapters.lettakit import ( + make_assistant_message, + make_letta_response, + make_mock_agent, + make_mock_async_stream, + make_mock_conversation, + make_platform_message, +) + + +class TestTaskEvents: + @pytest.mark.asyncio + async def test_emits_task_event_on_agent_creation(self) -> None: + adapter = LettaAdapter() + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + + mock_agent = make_mock_agent("new-agent") + mock_client.agents.create.return_value = mock_agent + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Hi!") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + task_events = [e for e in tools.events_sent if e["message_type"] == "task"] + assert len(task_events) == 1 + metadata = task_events[0]["metadata"] + assert metadata["letta_agent_id"] == "new-agent" + assert metadata["letta_room_id"] == "room-1" + + @pytest.mark.asyncio + async def test_task_events_disabled(self) -> None: + config = LettaAdapterConfig() + adapter = LettaAdapter(config=config, emit=()) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + + mock_agent = make_mock_agent("new-agent") + mock_client.agents.create.return_value = mock_agent + mock_client.agents.messages.create.return_value = make_letta_response( + make_assistant_message("Hi!") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + task_events = [e for e in tools.events_sent if e["message_type"] == "task"] + assert len(task_events) == 0 + + @pytest.mark.asyncio + async def test_shared_mode_emits_conversation_id(self) -> None: + config = LettaAdapterConfig(mode="shared") + adapter = LettaAdapter(config=config) + mock_client = AsyncMock() + adapter._client = mock_client + adapter._system_prompt = "Test" + adapter._mcp.tool_ids = [] + adapter._mcp.server_id = "mcp-server-1" + + mock_agent = make_mock_agent("shared-agent") + mock_client.agents.create.return_value = mock_agent + mock_conv = make_mock_conversation("conv-123") + mock_client.conversations.create.return_value = mock_conv + mock_client.conversations.messages.create.return_value = make_mock_async_stream( + make_assistant_message("Hi!") + ) + + tools = FakeAgentTools() + msg = make_platform_message() + history = LettaSessionState() + + await adapter.on_message( + msg, + tools, + history, + None, + None, + is_session_bootstrap=True, + room_id="room-1", + ) + + task_events = [e for e in tools.events_sent if e["message_type"] == "task"] + assert len(task_events) == 1 + assert task_events[0]["metadata"]["letta_conversation_id"] == "conv-123" diff --git a/tests/adapters/lettakit.py b/tests/adapters/lettakit.py index b9b7f8103..264493c39 100644 --- a/tests/adapters/lettakit.py +++ b/tests/adapters/lettakit.py @@ -2,8 +2,7 @@ Mock builders for Letta API objects (messages, responses, agents, conversations, MCP servers/tools, async streams) and platform messages, used -by ``test_letta_adapter.py``, ``test_letta_mcp.py``, and -``test_letta_orgscope.py``. +by the Letta adapter and MCP suites. """ from __future__ import annotations @@ -16,6 +15,7 @@ from uuid import uuid4 from letta_client import AsyncLetta +from letta_client.types.agents.letta_streaming_response import LettaUsageStatistics from letta_client.types.agents.tool_call import ToolCall from letta_client.types.agents.tool_call_message import ToolCallMessage from letta_client.types.agents.tool_return import ToolReturn @@ -149,6 +149,21 @@ def make_letta_response(*messages: Any) -> MagicMock: """Create a fake Letta API response.""" resp = MagicMock() resp.messages = list(messages) + + async def stream() -> Any: + for message in resp.messages: + yield message + if usage := resp.__dict__.get("usage"): + values = { + name: value + for name in LettaUsageStatistics.model_fields + if isinstance(value := getattr(usage, name, None), int) + } + yield LettaUsageStatistics(message_type="usage_statistics", **values) + + resp.__aiter__.side_effect = stream + resp.__aenter__.return_value = resp + resp.__aexit__.return_value = None return resp @@ -323,12 +338,4 @@ def mock_org_user_provisioned( def make_mock_async_stream(*messages: MagicMock) -> Any: """Create a mock async stream yielding Letta messages.""" - class _AsyncStream: - def __init__(self, stream_messages: list[MagicMock]) -> None: - self._messages = stream_messages - - async def __aiter__(self) -> Any: - for stream_message in self._messages: - yield stream_message - - return _AsyncStream(list(messages)) + return make_letta_response(*messages) diff --git a/tests/adapters/strands/__init__.py b/tests/adapters/strands/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/adapters/strands/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/strands/activity/__init__.py b/tests/adapters/strands/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/adapters/strands/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/strands/activity/test_hooks.py b/tests/adapters/strands/activity/test_hooks.py new file mode 100644 index 000000000..b662fd503 --- /dev/null +++ b/tests/adapters/strands/activity/test_hooks.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import pytest + +pytest.importorskip("strands") + +from strands.hooks.events import AfterToolCallEvent, BeforeToolCallEvent + +from band.adapters.strands import BandTurnHooks +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder + + +class TestStrandsActivity: + @pytest.mark.parametrize("status", ["success", "error"]) + async def test_native_hook_reports_without_narration(self, status: str) -> None: + tools = FakeAgentTools() + recorder = ActivityRecorder() + tools._activity_observer = recorder + hooks = BandTurnHooks(tools, emit_execution=False, custom_effects={}) + call = { + "name": "shell", + "input": {"command": "private-argument"}, + "toolUseId": "call", + } + await hooks._on_before_tool( + BeforeToolCallEvent( + agent=None, invocation_state={}, tool_use=call, selected_tool=None + ) + ) + assert recorder.events == [{"kind": "tool_call", "name": "shell"}] + await hooks._on_after_tool( + AfterToolCallEvent( + agent=None, + selected_tool=None, + invocation_state={}, + tool_use=call, + result={ + "toolUseId": "call", + "status": status, + "content": [{"text": "private-result"}], + }, + ) + ) + assert recorder.events == [ + {"kind": "tool_call", "name": "shell"}, + {"kind": "reasoning"}, + ] + assert not tools.events_sent diff --git a/tests/adapters/test_letta_adapter.py b/tests/adapters/test_letta_adapter.py deleted file mode 100644 index 5b0c07cf0..000000000 --- a/tests/adapters/test_letta_adapter.py +++ /dev/null @@ -1,2123 +0,0 @@ -"""Tests for LettaAdapter. - -Message-path and agent-lifecycle coverage. MCP wiring/lifecycle tests live -in ``test_letta_mcp.py``; the shared mock factories in ``lettakit.py``. -""" - -from __future__ import annotations - -import asyncio -import json -import logging -from datetime import UTC, datetime, timedelta -from typing import Any, Literal -from unittest.mock import AsyncMock, MagicMock - -import pytest -from letta_client.types.agents.text_content import TextContent -from letta_client.types.agents.tool_call import ToolCall -from letta_client.types.agents.tool_return import ToolReturn -from letta_client.types.tool_return_message import ToolReturnMessage - -from band.adapters.letta import ( - LettaAdapter, - LettaAdapterConfig, - LettaMCPConfig, - RoomContext, -) -from band.converters.letta import LettaSessionState -from band.core.protocols import ( - GENERIC_PROVIDER_FAILURE_MESSAGE, - TurnResultAlreadyReported, -) -from band.core.types import Capability, Emit, ToolEventKey -from band.runtime.tools import BandTool -from band.testing import ( - MISSING_REPLY_FAILURE, - FakeAgentTools, - events_of_type, - failure_reports, - reported_failures, -) -from tests.adapters.lettakit import ( - call_id, - default_enforcement, - make_assistant_message, - make_grouped_call_message, - make_grouped_return_message, - make_letta_response, - make_mock_agent, - make_mock_async_stream, - make_mock_conversation, - make_mock_tool_page, - make_parallel_tool_messages, - make_platform_message, - make_tool_call_message, - make_tool_return_message, - ready_letta_adapter, - scripted_letta_turn, -) -from tests.framework_conformance.turnprobes import ROOM_ID, turn_input, turn_tools - -# Band tool calls that settle the turn's reply, so no assistant text is relayed. -REPLY_SETTLING_CALLS = [ - pytest.param( - BandTool.SEND_MESSAGE, {"content": "Hi", "mentions": ["@alice"]}, id="message" - ), - pytest.param( - BandTool.SEND_ROOM_FILE, - {"content": "notes", "filename": "notes.txt", "mentions": ["@alice"]}, - id="room-file", - ), - pytest.param(BandTool.NO_REPLY, {"reason": "FYI only"}, id="no-reply"), -] - - -def sent_contents(tools: FakeAgentTools) -> list[str]: - return [message["content"] for message in tools.messages_sent] - - -def reported_tool_names(tools: FakeAgentTools, event_type: str) -> list[str]: - return [ - json.loads(event["content"])[ToolEventKey.NAME] - for event in events_of_type(tools, event_type) - ] - - -# ────────────────────────────────────────────────────────────────────── -# Initialization -# ────────────────────────────────────────────────────────────────────── - - -class TestLettaAdapterInit: - # Default values are not asserted here: that would only restate the - # dataclass definition, and the frozen-config conformance suite already - # pins ``LettaAdapter().config == LettaAdapterConfig()``. - - def test_custom_config_reaches_adapter(self) -> None: - config = LettaAdapterConfig( - base_url="http://custom:8283", - provider_key="sk-test", - mode="shared", - mcp=LettaMCPConfig(mode="external", server_url="http://mcp:9000/sse"), - ) - adapter = LettaAdapter(config=config) - assert adapter.config is config - - def test_no_leaked_adapter_config_env_vars( - self, assert_no_leaked_adapter_config_env: None - ) -> None: - """Requesting the fixture is the assertion — see its docstring.""" - - -# ────────────────────────────────────────────────────────────────────── -# on_message (per_room mode) -# ────────────────────────────────────────────────────────────────────── - - -class TestLettaAdapterOnMessagePerRoom: - """Tests for per_room mode message handling.""" - - @pytest.fixture - def adapter_with_client(self) -> tuple[LettaAdapter, AsyncMock]: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test prompt" - adapter._mcp.tool_ids = ["t1", "t2"] - adapter._mcp.server_id = "mcp-server-1" - return adapter, mock_client - - @pytest.mark.asyncio - async def test_basic_message_creates_agent_and_sends( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = adapter_with_client - - mock_agent = make_mock_agent("agent-1") - mock_client.agents.create.return_value = mock_agent - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Hello!") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - # Agent was created - mock_client.agents.create.assert_called_once() - # MCP tools were attached - assert mock_client.agents.tools.attach.call_count == 2 - # Message was sent via direct agent API (not conversations) - mock_client.agents.messages.create.assert_called_once() - call_kwargs = mock_client.agents.messages.create.call_args.kwargs - assert call_kwargs["agent_id"] == "agent-1" - assert "conversation_id" not in call_kwargs - - @pytest.mark.asyncio - async def test_configured_memory_blocks_follow_the_persona_block( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = adapter_with_client - human = {"label": "human", "value": "Prefers short answers."} - adapter.config = LettaAdapterConfig(memory_blocks=(human,)) - mock_client.agents.create.return_value = make_mock_agent("agent-1") - - await adapter._create_agent("room-1") - - blocks = mock_client.agents.create.call_args.kwargs["memory_blocks"] - assert [block["label"] for block in blocks] == ["persona", "human"] - assert blocks[1] == human - - @pytest.mark.asyncio - async def test_auto_relay_when_no_send_message( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = adapter_with_client - - # Setup room with agent - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("I'll help you!") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - # Auto-relay should have sent the message - assert len(tools.messages_sent) == 1 - assert tools.messages_sent[0]["content"] == "I'll help you!" - - @pytest.mark.asyncio - async def test_send_message_failure_is_not_reported_as_provider_failure( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - """The Letta agent answered fine; the room POST is what failed. That - must not surface as a Letta AgentFailure -- deliver_reply's - DeliveryFailedError must be recognized and left unreported here.""" - adapter, mock_client = adapter_with_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("I'll help you!") - ) - - tools = FakeAgentTools() - - async def _raise(*args: Any, **kwargs: Any) -> None: - raise RuntimeError("platform rejected the message") - - tools.send_message = _raise # type: ignore[method-assign] - - msg = make_platform_message() - history = LettaSessionState() - - with pytest.raises(RuntimeError, match="platform rejected the message"): - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert not reported_failures(tools) - - @pytest.mark.asyncio - @pytest.mark.parametrize(("reply_tool", "arguments"), REPLY_SETTLING_CALLS) - async def test_skip_auto_relay_when_a_reply_tool_settled_the_turn( - self, reply_tool: str, arguments: dict[str, Any] - ) -> None: - adapter, mock_client = ready_letta_adapter(capabilities={Capability.FILES}) - mock_client.agents.messages.create.side_effect = scripted_letta_turn( - adapter, - room_id="room-1", - tool_calls=[(reply_tool, arguments)], - final_text="Done!", - ) - tools = FakeAgentTools() - - await adapter.on_message( - make_platform_message(), - tools, - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert "Done!" not in sent_contents(tools) - - @pytest.mark.asyncio - async def test_timeout_reports_error( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = adapter_with_client - adapter.config = adapter.config.model_copy(update={"turn_timeout_s": 0.01}) - - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - async def slow_response(**kwargs: Any) -> MagicMock: - await asyncio.sleep(1) - return make_letta_response() - - mock_client.agents.messages.create.side_effect = slow_response - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - with pytest.raises(TurnResultAlreadyReported): - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - failures = reported_failures(tools) - assert len(failures) == 1 - assert "timed out" in failures[0]["message"] - assert failures[0]["provider"] == "letta" - assert failures[0]["code"] == "timeout" - - @pytest.mark.asyncio - async def test_slow_delivery_is_not_misreported_as_provider_timeout( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - """turn_timeout_s bounds only the Letta round-trip. A room POST that - is merely slow (not the Letta call) must not be misreported as a - Letta provider timeout -- it must be given time to complete.""" - adapter, mock_client = adapter_with_client - adapter.config = adapter.config.model_copy(update={"turn_timeout_s": 0.05}) - - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - # The Letta call itself responds instantly, well inside turn_timeout_s. - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("I'll help you!") - ) - - tools = FakeAgentTools() - real_send_message = tools.send_message - - async def _slow_send_message(*args: Any, **kwargs: Any) -> Any: - # Longer than turn_timeout_s: only the provider call may race it. - await asyncio.sleep(0.15) - return await real_send_message(*args, **kwargs) - - tools.send_message = _slow_send_message # type: ignore[method-assign] - - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert len(tools.messages_sent) == 1 - assert not reported_failures(tools) - - @pytest.mark.asyncio - async def test_send_event_timeout_is_not_misreported_as_provider_timeout( - self, caplog: pytest.LogCaptureFixture - ) -> None: - """A TimeoutError from tool-event reporting (send_event) -- not the - Letta round-trip itself -- must fall through to the generic failure - path, never be mislabeled as a Letta provider timeout. (send_failure - itself best-effort-swallows the same broken channel here, same as - production, so the room never receives a failure event either way -- - what this guards is which branch is taken/logged.)""" - config = LettaAdapterConfig() - adapter = LettaAdapter(config=config, emit=Emit.TOOL_CALLS) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - # The Letta round-trip itself returns instantly -- well inside - # turn_timeout_s -- so any TimeoutError must come from elsewhere. - mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_call_message("band_lookup_peers", "{}"), - make_tool_return_message("band_lookup_peers", '{"peers": []}'), - ) - - tools = FakeAgentTools() - tools.send_event_error = TimeoutError("event POST hiccup") - msg = make_platform_message() - history = LettaSessionState() - - with ( - caplog.at_level(logging.ERROR, logger="band.adapters.letta"), - pytest.raises(TimeoutError), - ): - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert not any("timed out" in r.message for r in caplog.records) - assert any("Error during Letta turn" in r.message for r in caplog.records) - - @pytest.mark.asyncio - async def test_generic_exception_reports_and_propagates( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - """A bare exception from the Letta client (not a timeout, not a - delivery failure) must still surface via the generic fallback.""" - adapter, mock_client = adapter_with_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - mock_client.agents.messages.create.side_effect = ConnectionError( - "letta connection reset" - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - with pytest.raises(ConnectionError, match="letta connection reset"): - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - failure = reported_failures(tools)[0] - assert failure["provider"] == "letta" - assert failure["message"] == GENERIC_PROVIDER_FAILURE_MESSAGE - assert "letta connection reset" not in failure["message"] - - @pytest.mark.asyncio - async def test_participants_and_contacts_injected( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = adapter_with_client - - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Got it.") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - "Participants: Alice, Bob", - "Contacts: Charlie", - is_session_bootstrap=False, - room_id="room-1", - ) - - call_kwargs = mock_client.agents.messages.create.call_args.kwargs - content = call_kwargs["messages"][0]["content"] - assert "[System]: Participants: Alice, Bob" in content - assert "[System]: Contacts: Charlie" in content - - @pytest.mark.asyncio - async def test_agent_resume_from_history( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = adapter_with_client - - mock_client.agents.retrieve.return_value = make_mock_agent("history-agent") - mock_client.agents.tools.list.return_value = make_mock_tool_page() - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Resumed!") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState(agent_id="history-agent") - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - # Agent was retrieved, not created - mock_client.agents.retrieve.assert_called_once_with("history-agent") - mock_client.agents.create.assert_not_called() - - @pytest.mark.asyncio - async def test_uninitialized_client_reports_error(self) -> None: - adapter = LettaAdapter() - # _client is None - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - with pytest.raises(RuntimeError, match="not initialized"): - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - failures = reported_failures(tools) - assert len(failures) == 1 - assert failures[0]["provider"] == "letta" - assert "not initialized" in failures[0]["message"] - - -# ────────────────────────────────────────────────────────────────────── -# on_message (shared mode) -# ────────────────────────────────────────────────────────────────────── - - -class TestLettaAdapterSharedMode: - """Tests for shared mode with Conversations API.""" - - @pytest.fixture - def shared_adapter(self) -> tuple[LettaAdapter, AsyncMock]: - config = LettaAdapterConfig(mode="shared") - adapter = LettaAdapter(config=config) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test prompt" - adapter._mcp.tool_ids = ["t1", "t2"] - adapter._mcp.server_id = "mcp-server-1" - return adapter, mock_client - - @pytest.mark.asyncio - async def test_shared_mode_creates_agent_and_conversation( - self, shared_adapter: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = shared_adapter - - mock_agent = make_mock_agent("shared-agent") - mock_client.agents.create.return_value = mock_agent - mock_conv = make_mock_conversation("conv-room1") - mock_client.conversations.create.return_value = mock_conv - mock_client.conversations.messages.create.return_value = make_mock_async_stream( - make_assistant_message("Hi from shared!") - ) - - tools = FakeAgentTools() - msg = make_platform_message(room_id="room-1") - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - # One agent created - mock_client.agents.create.assert_called_once() - # Conversation created for this room - mock_client.conversations.create.assert_called_once_with( - agent_id="shared-agent", - ) - # Message sent through room-scoped conversation endpoint - call_kwargs = mock_client.conversations.messages.create.call_args.kwargs - assert call_kwargs["conversation_id"] == "conv-room1" - assert adapter._shared_agent_id == "shared-agent" - - @pytest.mark.asyncio - async def test_shared_mode_reuses_agent_for_second_room( - self, shared_adapter: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = shared_adapter - - # Setup: first room already connected - adapter._shared_agent_id = "shared-agent" - adapter._rooms["room-1"] = RoomContext( - agent_id="shared-agent", conversation_id="conv-1" - ) - - mock_conv2 = make_mock_conversation("conv-room2") - mock_client.conversations.create.return_value = mock_conv2 - mock_client.conversations.messages.create.return_value = make_mock_async_stream( - make_assistant_message("Hi room 2!") - ) - - tools = FakeAgentTools() - msg = make_platform_message(room_id="room-2") - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-2", - ) - - # Agent was NOT created again - mock_client.agents.create.assert_not_called() - # But a new conversation was created - mock_client.conversations.create.assert_called_once_with( - agent_id="shared-agent", - ) - assert adapter._rooms["room-2"].conversation_id == "conv-room2" - - @pytest.mark.asyncio - async def test_shared_mode_resumes_existing_agent( - self, shared_adapter: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = shared_adapter - adapter.config = adapter.config.model_copy( - update={"agent_id": "pre-existing-agent"} - ) - - mock_client.agents.retrieve.return_value = make_mock_agent("pre-existing-agent") - mock_client.agents.tools.list.return_value = make_mock_tool_page() - mock_conv = make_mock_conversation("conv-1") - mock_client.conversations.create.return_value = mock_conv - mock_client.conversations.messages.create.return_value = make_mock_async_stream( - make_assistant_message("Resumed shared!") - ) - - tools = FakeAgentTools() - msg = make_platform_message(room_id="room-1") - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - mock_client.agents.retrieve.assert_called_once_with("pre-existing-agent") - mock_client.agents.create.assert_not_called() - assert adapter._shared_agent_id == "pre-existing-agent" - - @pytest.mark.asyncio - async def test_shared_mode_resumes_persisted_conversation( - self, shared_adapter: tuple[LettaAdapter, AsyncMock] - ) -> None: - """A restart with a persisted conversation reuses it — a fresh one - would silently drop the room's conversational context.""" - adapter, mock_client = shared_adapter - adapter._shared_agent_id = "shared-agent" - - mock_client.conversations.retrieve.return_value = make_mock_conversation( - "conv-persisted", agent_id="shared-agent" - ) - mock_client.conversations.messages.create.return_value = make_mock_async_stream( - make_assistant_message("Still here!") - ) - - history = LettaSessionState( - agent_id="shared-agent", - conversation_id="conv-persisted", - replay_messages=["[Alice]: earlier"], - ) - tools = FakeAgentTools() - await adapter.on_message( - make_platform_message(room_id="room-1"), - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - mock_client.conversations.retrieve.assert_awaited_once_with("conv-persisted") - mock_client.conversations.create.assert_not_called() - assert adapter._rooms["room-1"].conversation_id == "conv-persisted" - # A resumed conversation already has its context — no seeding. - content = mock_client.conversations.messages.create.call_args.kwargs[ - "messages" - ][0]["content"] - assert "[Alice]: earlier" not in content - - @pytest.mark.asyncio - async def test_shared_mode_rejects_foreign_conversation( - self, shared_adapter: tuple[LettaAdapter, AsyncMock] - ) -> None: - """A persisted conversation must belong to the current shared agent.""" - adapter, mock_client = shared_adapter - adapter._shared_agent_id = "shared-agent" - - mock_client.conversations.retrieve.return_value = make_mock_conversation( - "conv-other", agent_id="other-agent" - ) - mock_client.conversations.create.return_value = make_mock_conversation( - "conv-fresh", agent_id="shared-agent" - ) - mock_client.conversations.messages.create.return_value = make_mock_async_stream( - make_assistant_message("Fresh thread") - ) - - history = LettaSessionState( - agent_id="shared-agent", - conversation_id="conv-other", - ) - await adapter.on_message( - make_platform_message(room_id="room-1"), - FakeAgentTools(), - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - mock_client.conversations.create.assert_awaited_once_with( - agent_id="shared-agent" - ) - assert adapter._rooms["room-1"].conversation_id == "conv-fresh" - - @pytest.mark.asyncio - async def test_shared_mode_seeds_fresh_conversation_from_history( - self, shared_adapter: tuple[LettaAdapter, AsyncMock] - ) -> None: - """A dead persisted conversation falls back to create + seed.""" - adapter, mock_client = shared_adapter - adapter._shared_agent_id = "shared-agent" - - mock_client.conversations.retrieve.side_effect = Exception("gone") - mock_client.conversations.create.return_value = make_mock_conversation( - "conv-fresh" - ) - mock_client.conversations.messages.create.return_value = make_mock_async_stream( - make_assistant_message("Rebuilt!") - ) - - history = LettaSessionState( - agent_id="shared-agent", - conversation_id="conv-dead", - replay_messages=["[Alice]: the secret is kumquat"], - ) - tools = FakeAgentTools() - await adapter.on_message( - make_platform_message(room_id="room-1"), - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - assert adapter._rooms["room-1"].conversation_id == "conv-fresh" - content = mock_client.conversations.messages.create.call_args.kwargs[ - "messages" - ][0]["content"] - assert "[Alice]: the secret is kumquat" in content - - @pytest.mark.asyncio - async def test_shared_mode_injects_room_id_per_message( - self, shared_adapter: tuple[LettaAdapter, AsyncMock] - ) -> None: - """Self-host + shared: the (shared) persona cannot carry a room id, so - every message reminds the agent which room_id to pass to tools.""" - adapter, mock_client = shared_adapter - adapter._shared_agent_id = "shared-agent" - adapter._rooms["room-42"] = RoomContext( - agent_id="shared-agent", conversation_id="conv-1" - ) - - mock_client.conversations.messages.create.return_value = make_mock_async_stream( - make_assistant_message("Hi!") - ) - - tools = FakeAgentTools() - await adapter.on_message( - make_platform_message(room_id="room-42"), - tools, - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-42", - ) - - content = mock_client.conversations.messages.create.call_args.kwargs[ - "messages" - ][0]["content"] - assert "Current chat_id: room-42" in content - - -# ────────────────────────────────────────────────────────────────────── -# Execution reporting (observation only) -# ────────────────────────────────────────────────────────────────────── - - -class TestExecutionReporting: - @pytest.mark.asyncio - async def test_reports_non_silent_tool_calls(self) -> None: - config = LettaAdapterConfig() - adapter = LettaAdapter(config=config, emit=Emit.TOOL_CALLS) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_call_message("band_lookup_peers", "{}"), - make_tool_return_message("band_lookup_peers", '{"peers": []}'), - make_assistant_message("Done"), - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - tool_call_events = [ - e for e in tools.events_sent if e["message_type"] == "tool_call" - ] - tool_result_events = [ - e for e in tools.events_sent if e["message_type"] == "tool_result" - ] - assert len(tool_call_events) == 1 - assert len(tool_result_events) == 1 - - @pytest.mark.asyncio - async def test_reports_every_parallel_call_and_its_result(self) -> None: - adapter, mock_client = ready_letta_adapter(emit=Emit.TOOL_CALLS) - calls = [ - ToolCall(name=BandTool.GET_PARTICIPANTS, arguments="{}", tool_call_id="a"), - ToolCall(name=BandTool.LOOKUP_PEERS, arguments="{}", tool_call_id="b"), - ] - multimodal_output = [TextContent(type="text", text="no peers")] - returns = [ - ToolReturn( - type="tool", - status="success", - tool_call_id="b", - tool_return=multimodal_output, - ), - ToolReturn( - type="tool", status="success", tool_call_id="a", tool_return="[]" - ), - ] - mock_client.agents.messages.create.return_value = make_letta_response( - make_grouped_call_message(calls), - make_grouped_return_message(returns, name=calls[0].name), - make_assistant_message("Done"), - ) - tools = FakeAgentTools() - - await adapter.on_message( - make_platform_message(), - tools, - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert reported_tool_names(tools, "tool_call") == [ - BandTool.GET_PARTICIPANTS, - BandTool.LOOKUP_PEERS, - ] - assert reported_tool_names(tools, "tool_result") == [ - BandTool.LOOKUP_PEERS, - BandTool.GET_PARTICIPANTS, - ] - - @pytest.mark.asyncio - async def test_silent_tools_not_reported(self) -> None: - config = LettaAdapterConfig() - adapter = LettaAdapter(config=config, emit=Emit.TOOL_CALLS) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_call_message("band_send_message"), - make_tool_return_message("band_send_message"), - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - tool_events = [ - e - for e in tools.events_sent - if e["message_type"] in ("tool_call", "tool_result") - ] - assert len(tool_events) == 0 - - @pytest.mark.asyncio - async def test_send_room_file_reports_content_placeholder_not_raw_bytes( - self, - ) -> None: - """ToolCall.arguments is a raw JSON string (letta_client's own wire - shape); the tool_call event must still redact band_send_room_file's - content field, not embed the real file bytes verbatim.""" - config = LettaAdapterConfig() - adapter = LettaAdapter(config=config, emit=Emit.TOOL_CALLS) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_call_message( - "band_send_room_file", - '{"content": "SECRET FILE BYTES", "filename": "f.txt"}', - ), - make_tool_return_message("band_send_room_file", '{"status": "ok"}'), - make_assistant_message("Done"), - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - tool_call_events = [ - e for e in tools.events_sent if e["message_type"] == "tool_call" - ] - assert len(tool_call_events) == 1 - reported_args = json.loads(tool_call_events[0]["content"])["args"] - assert "SECRET FILE BYTES" not in json.dumps(reported_args) - assert "byte file content" in reported_args["content"] - assert reported_args["filename"] == "f.txt" - - -# ────────────────────────────────────────────────────────────────────── -# on_cleanup -# ────────────────────────────────────────────────────────────────────── - - -class TestLettaAdapterOnCleanup: - @pytest.mark.asyncio - async def test_cleanup_removes_room_state(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - await adapter.on_cleanup("room-1") - - assert "room-1" not in adapter._rooms - # Memory consolidation was attempted - mock_client.agents.messages.create.assert_called_once() - - @pytest.mark.asyncio - async def test_cleanup_idempotent(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - - # Room doesn't exist - await adapter.on_cleanup("nonexistent") - mock_client.agents.messages.create.assert_not_called() - - @pytest.mark.asyncio - async def test_cleanup_twice(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - await adapter.on_cleanup("room-1") - await adapter.on_cleanup("room-1") # Should not raise - - assert "room-1" not in adapter._rooms - - @pytest.mark.asyncio - async def test_cleanup_multi_room(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - adapter._rooms["room-2"] = RoomContext(agent_id="agent-2") - - await adapter.on_cleanup("room-1") - - assert "room-1" not in adapter._rooms - assert "room-2" in adapter._rooms - - @pytest.mark.asyncio - async def test_cleanup_without_client(self) -> None: - adapter = LettaAdapter() - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - # No client — should not raise - await adapter.on_cleanup("room-1") - assert "room-1" not in adapter._rooms - - @pytest.mark.asyncio - async def test_cleanup_before_started(self) -> None: - adapter = LettaAdapter() - # Completely uninitialized — no client, no rooms - await adapter.on_cleanup("room-1") - assert "room-1" not in adapter._rooms - - -# ────────────────────────────────────────────────────────────────────── -# Instruction block update -# ────────────────────────────────────────────────────────────────────── - - -class TestInstructionBlockUpdate: - @pytest.mark.asyncio - async def test_updates_persona_block(self) -> None: - adapter = LettaAdapter() - adapter._system_prompt = "Test system prompt" - mock_client = AsyncMock() - adapter._client = mock_client - - await adapter._update_instruction_block("agent-1", "room-1") - - # Default config self-hosts the MCP server in per_room mode, so the - # enforcement carries the room id the tool schemas require. - mock_client.agents.blocks.update.assert_called_once_with( - "persona", - agent_id="agent-1", - value=default_enforcement(room_id="room-1") + "Test system prompt", - ) - - @pytest.mark.asyncio - async def test_falls_back_to_next_label(self) -> None: - adapter = LettaAdapter() - adapter._system_prompt = "Test prompt" - mock_client = AsyncMock() - adapter._client = mock_client - - # First label fails, second succeeds - mock_client.agents.blocks.update.side_effect = [ - Exception("not found"), - None, - None, - ] - - await adapter._update_instruction_block("agent-1", "room-1") - - assert mock_client.agents.blocks.update.call_count == 2 - second_call = mock_client.agents.blocks.update.call_args_list[1] - assert second_call.args[0] == "custom_instructions" - - @pytest.mark.asyncio - async def test_creates_persona_when_all_labels_fail(self) -> None: - adapter = LettaAdapter() - adapter._system_prompt = "Test prompt" - mock_client = AsyncMock() - adapter._client = mock_client - - mock_client.agents.blocks.update.side_effect = Exception("not found") - mock_block = MagicMock() - mock_block.id = "block-1" - mock_client.blocks.create.return_value = mock_block - - await adapter._update_instruction_block("agent-1", "room-1") - - mock_client.blocks.create.assert_called_once() - assert mock_client.blocks.create.call_args.kwargs["label"] == "persona" - mock_client.agents.blocks.attach.assert_called_once_with( - "block-1", - agent_id="agent-1", - ) - - -# ────────────────────────────────────────────────────────────────────── -# Rejoin context -# ────────────────────────────────────────────────────────────────────── - - -class TestRejoinContext: - @pytest.mark.asyncio - async def test_rejoin_injects_time_away(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - - last_time = datetime.now(UTC) - timedelta(hours=2) - adapter._rooms["room-1"] = RoomContext( - agent_id="agent-1", - last_interaction=last_time, - summary="Discussed project plan", - ) - - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("I'm back!") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - call_kwargs = mock_client.agents.messages.create.call_args.kwargs - content = call_kwargs["messages"][0]["content"] - assert "rejoined" in content - assert "2h" in content - assert "Discussed project plan" in content - - -# ────────────────────────────────────────────────────────────────────── -# Task events -# ────────────────────────────────────────────────────────────────────── - - -class TestTaskEvents: - @pytest.mark.asyncio - async def test_emits_task_event_on_agent_creation(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - - mock_agent = make_mock_agent("new-agent") - mock_client.agents.create.return_value = mock_agent - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Hi!") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - task_events = [e for e in tools.events_sent if e["message_type"] == "task"] - assert len(task_events) == 1 - metadata = task_events[0]["metadata"] - assert metadata["letta_agent_id"] == "new-agent" - assert metadata["letta_room_id"] == "room-1" - - @pytest.mark.asyncio - async def test_task_events_disabled(self) -> None: - config = LettaAdapterConfig() - adapter = LettaAdapter(config=config, emit=()) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - - mock_agent = make_mock_agent("new-agent") - mock_client.agents.create.return_value = mock_agent - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Hi!") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - task_events = [e for e in tools.events_sent if e["message_type"] == "task"] - assert len(task_events) == 0 - - @pytest.mark.asyncio - async def test_shared_mode_emits_conversation_id(self) -> None: - config = LettaAdapterConfig(mode="shared") - adapter = LettaAdapter(config=config) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - - mock_agent = make_mock_agent("shared-agent") - mock_client.agents.create.return_value = mock_agent - mock_conv = make_mock_conversation("conv-123") - mock_client.conversations.create.return_value = mock_conv - mock_client.conversations.messages.create.return_value = make_mock_async_stream( - make_assistant_message("Hi!") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - task_events = [e for e in tools.events_sent if e["message_type"] == "task"] - assert len(task_events) == 1 - assert task_events[0]["metadata"]["letta_conversation_id"] == "conv-123" - - -# ────────────────────────────────────────────────────────────────────── -# Memory consolidation -# ────────────────────────────────────────────────────────────────────── - - -class TestMemoryConsolidation: - @pytest.mark.asyncio - async def test_consolidation_on_cleanup(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - await adapter.on_cleanup("room-1") - - mock_client.agents.messages.create.assert_called_once() - call_kwargs = mock_client.agents.messages.create.call_args.kwargs - assert "Consolidate" in call_kwargs["messages"][0]["content"] - - @pytest.mark.asyncio - async def test_consolidation_failure_does_not_propagate(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.side_effect = Exception("API error") - - # Should not raise - await adapter.on_cleanup("room-1") - assert "room-1" not in adapter._rooms - - -# ────────────────────────────────────────────────────────────────────── -# Static helpers -# ────────────────────────────────────────────────────────────────────── - - -class TestFormatTimeAgo: - def test_seconds(self) -> None: - dt = datetime.now(UTC) - timedelta(seconds=30) - assert "30s" == LettaAdapter._format_time_ago(dt) - - def test_minutes(self) -> None: - dt = datetime.now(UTC) - timedelta(minutes=5) - assert "5m" == LettaAdapter._format_time_ago(dt) - - def test_hours(self) -> None: - dt = datetime.now(UTC) - timedelta(hours=3) - assert "3h" == LettaAdapter._format_time_ago(dt) - - def test_one_hour(self) -> None: - dt = datetime.now(UTC) - timedelta(hours=1) - assert "1 hour" == LettaAdapter._format_time_ago(dt) - - def test_days(self) -> None: - dt = datetime.now(UTC) - timedelta(days=5) - assert "5d" == LettaAdapter._format_time_ago(dt) - - def test_one_day(self) -> None: - dt = datetime.now(UTC) - timedelta(days=1) - assert "1 day" == LettaAdapter._format_time_ago(dt) - - def test_naive_datetime_treated_as_utc(self) -> None: - dt = datetime.now(UTC).replace(tzinfo=None) - timedelta(minutes=10) - result = LettaAdapter._format_time_ago(dt) - assert "10m" == result - - -class TestExtractSummary: - def test_short_text_kept_whole(self) -> None: - parts = ["Hello there. This is more text."] - assert "Hello there. This is more text." == LettaAdapter._extract_summary(parts) - - def test_truncates_on_word_boundary(self) -> None: - parts = ["alpha beta " * 30] - result = LettaAdapter._extract_summary(parts, max_length=50) - assert len(result) <= 53 # 50 + "..." - assert result.endswith("...") - assert not result.removesuffix("...").endswith("alph") # no mid-word cut - - def test_no_sentence_heuristics(self) -> None: - """Decimals must not be mistaken for sentence ends.""" - parts = ["pi is 3.14 and e is 2.71"] - assert "pi is 3.14 and e is 2.71" == LettaAdapter._extract_summary(parts) - - def test_empty(self) -> None: - assert "" == LettaAdapter._extract_summary([]) - - def test_multiple_parts_joined(self) -> None: - parts = ["First part.", "Second part."] - assert "First part. Second part." == LettaAdapter._extract_summary(parts) - - -# ────────────────────────────────────────────────────────────────────── -# Summary storage -# ────────────────────────────────────────────────────────────────────── - - -class TestSummaryStorage: - @pytest.mark.asyncio - async def test_summary_stored_after_turn(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("The weather is sunny. More details follow.") - ) - - tools = FakeAgentTools() - msg = make_platform_message() - history = LettaSessionState() - - await adapter.on_message( - msg, - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - room_ctx = adapter._rooms["room-1"] - assert room_ctx.summary == "The weather is sunny. More details follow." - assert room_ctx.last_interaction is not None - - -# ────────────────────────────────────────────────────────────────────── -# Send-tool name resolution (derived from discovered MCP tools) -# ────────────────────────────────────────────────────────────────────── - - -class TestSendToolResolution: - def test_resolves_external_band_mcp_names(self) -> None: - adapter = LettaAdapter() - adapter._mcp.resolve_send_tools( - ["create_agent_chat_message", "create_agent_chat_event", "health_check"] - ) - assert adapter._mcp.send_message_tool == "create_agent_chat_message" - assert adapter._mcp.send_event_tool == "create_agent_chat_event" - assert adapter._mcp.silent_reporting_tools == { - "create_agent_chat_message", - "create_agent_chat_event", - } - - def test_prefers_band_names_when_both_present(self) -> None: - adapter = LettaAdapter() - adapter._mcp.resolve_send_tools( - ["band_send_message", "create_agent_chat_message", "band_send_event"] - ) - assert adapter._mcp.send_message_tool == "band_send_message" - assert adapter._mcp.send_event_tool == "band_send_event" - - def test_falls_back_to_band_names_when_none_discovered(self) -> None: - adapter = LettaAdapter() - adapter._mcp.resolve_send_tools(["some_unrelated_tool"]) - assert adapter._mcp.send_message_tool == "band_send_message" - assert adapter._mcp.send_event_tool == "band_send_event" - - def test_enforcement_prompt_uses_resolved_names(self) -> None: - adapter = LettaAdapter() - adapter._system_prompt = "Base prompt" - adapter._mcp.resolve_send_tools( - ["create_agent_chat_message", "create_agent_chat_event"] - ) - text = adapter._instruction_text("room-1") - assert "create_agent_chat_message" in text - assert "band_send_message" not in text - - @pytest.mark.asyncio - async def test_relay_detection_uses_resolved_name(self) -> None: - """A send via the resolved (external) tool name suppresses auto-relay.""" - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")) - ) - adapter._mcp.resolve_send_tools( - ["create_agent_chat_message", "create_agent_chat_event"] - ) - mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_call_message("create_agent_chat_message"), - make_tool_return_message("create_agent_chat_message"), - make_assistant_message("Done!"), - ) - - tools = FakeAgentTools() - await adapter.on_message( - make_platform_message(), - tools, - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert len(tools.messages_sent) == 0 - - -# ────────────────────────────────────────────────────────────────────── -# Turn recording of out-of-process tool calls -# ────────────────────────────────────────────────────────────────────── - - -class TestExternalToolRecording: - @pytest.mark.asyncio - @pytest.mark.parametrize( - ("mcp_mode", "status", "replied"), - [ - pytest.param("external", "success", True, id="external-success"), - pytest.param("external", "error", False, id="external-error"), - # Self-hosted calls run on the room's own tools, which record them. - pytest.param("self_host", "success", False, id="self-host"), - ], - ) - async def test_streamed_tool_return_records_only_for_external_band_mcp( - self, - mcp_mode: Literal["self_host", "external"], - status: Literal["success", "error"], - replied: bool, - ) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(mcp=LettaMCPConfig(mode=mcp_mode)) - ) - mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_call_message(BandTool.SEND_MESSAGE), - make_tool_return_message(BandTool.SEND_MESSAGE, status=status), - ) - tools = FakeAgentTools() - - await adapter.on_message( - make_platform_message(), - tools, - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert tools.turn.replied is replied - - @pytest.mark.asyncio - async def test_ungrouped_tool_return_is_matched_to_its_call(self) -> None: - """A server that sends no ``tool_returns`` puts the one result on the - message itself.""" - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")) - ) - mock_client.agents.messages.create.return_value = make_letta_response( - make_tool_call_message(BandTool.SEND_MESSAGE), - ToolReturnMessage( - id="message-return", - date=datetime.now(UTC), - message_type="tool_return_message", - status="success", - tool_call_id=call_id(BandTool.SEND_MESSAGE), - tool_return="result", - ), - ) - tools = FakeAgentTools() - - await adapter.on_message( - make_platform_message(), - tools, - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert tools.turn.replied - - @pytest.mark.asyncio - @pytest.mark.parametrize( - ("effect_tool", "replied"), - [ - pytest.param(BandTool.ADD_PARTICIPANT, False, id="act"), - pytest.param(BandTool.SEND_MESSAGE, True, id="reply"), - pytest.param(BandTool.NO_REPLY, True, id="decline"), - ], - ) - @pytest.mark.parametrize("reverse_calls", [False, True]) - async def test_parallel_success_completes_turn_independent_of_order( - self, - effect_tool: str, - replied: bool, - reverse_calls: bool, - ) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), - room_id=ROOM_ID, - ) - results: list[tuple[str, Literal["success", "error"]]] = [ - (BandTool.GET_PARTICIPANTS, "success"), - (effect_tool, "success"), - ] - if reverse_calls: - results.reverse() - mock_client.agents.messages.create.return_value = make_letta_response( - *make_parallel_tool_messages(results, reverse_returns=False) - ) - tools = turn_tools() - - await adapter.on_event(turn_input(tools)) - - assert tools.turn.complete - assert tools.turn.replied is replied - assert tools.messages_sent == [] - assert failure_reports(tools) == [] - - @pytest.mark.asyncio - @pytest.mark.parametrize( - "effect_tool", - [BandTool.ADD_PARTICIPANT, BandTool.SEND_MESSAGE, BandTool.NO_REPLY], - ) - @pytest.mark.parametrize("reverse_returns", [False, True]) - async def test_parallel_failed_effect_does_not_complete_turn( - self, effect_tool: str, reverse_returns: bool - ) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), - room_id=ROOM_ID, - ) - mock_client.agents.messages.create.return_value = make_letta_response( - *make_parallel_tool_messages( - [ - (BandTool.GET_PARTICIPANTS, "success"), - (effect_tool, "error"), - ], - reverse_returns=reverse_returns, - ) - ) - tools = turn_tools() - - with pytest.raises(TurnResultAlreadyReported): - await adapter.on_event(turn_input(tools)) - - assert not tools.turn.replied - assert tools.messages_sent == [] - assert failure_reports(tools) == [MISSING_REPLY_FAILURE] - - @pytest.mark.asyncio - @pytest.mark.parametrize("effect_tool", [BandTool.SEND_MESSAGE, BandTool.NO_REPLY]) - @pytest.mark.parametrize("reverse_returns", [False, True]) - async def test_parallel_failed_reply_does_not_suppress_fallback( - self, effect_tool: str, reverse_returns: bool - ) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")), - room_id=ROOM_ID, - ) - mock_client.agents.messages.create.return_value = make_letta_response( - *make_parallel_tool_messages( - [(effect_tool, "error"), (BandTool.GET_PARTICIPANTS, "success")], - reverse_returns=reverse_returns, - ), - make_assistant_message("Fallback reply"), - ) - tools = turn_tools() - - await adapter.on_event(turn_input(tools)) - - assert sent_contents(tools) == ["Fallback reply"] - assert failure_reports(tools) == [] - - @pytest.mark.asyncio - @pytest.mark.parametrize("reverse_returns", [False, True]) - async def test_parallel_results_match_call_ids_not_positions( - self, reverse_returns: bool - ) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(mcp=LettaMCPConfig(mode="external"), auto_relay=False), - room_id=ROOM_ID, - ) - mock_client.agents.messages.create.return_value = make_letta_response( - *make_parallel_tool_messages( - [ - (BandTool.SEND_MESSAGE, "error"), - (BandTool.ADD_PARTICIPANT, "success"), - ], - reverse_returns=reverse_returns, - ) - ) - tools = turn_tools() - - await adapter.on_event(turn_input(tools)) - - assert tools.turn.complete - assert not tools.turn.replied - assert tools.messages_sent == [] - assert failure_reports(tools) == [] - - -# ────────────────────────────────────────────────────────────────────── -# Auto-relay knob -# ────────────────────────────────────────────────────────────────────── - - -class TestAutoRelayDisabled: - @pytest.mark.asyncio - async def test_disabled_relay_drops_the_text_and_the_verdict_reports_it( - self, - ) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(auto_relay=False), room_id=ROOM_ID - ) - mock_client.agents.messages.create.side_effect = scripted_letta_turn( - adapter, room_id=ROOM_ID, tool_calls=[], final_text="I'll help you!" - ) - - tools = turn_tools() - with pytest.raises(TurnResultAlreadyReported): - await adapter.on_event(turn_input(tools)) - - assert tools.messages_sent == [] - assert failure_reports(tools) == [MISSING_REPLY_FAILURE] - - @pytest.mark.asyncio - async def test_disabled_relay_completes_a_turn_that_did_real_work(self) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(auto_relay=False), room_id=ROOM_ID - ) - mock_client.agents.messages.create.side_effect = scripted_letta_turn( - adapter, - room_id=ROOM_ID, - tool_calls=[(BandTool.CREATE_CHATROOM, {})], - final_text="Created it.", - ) - - tools = turn_tools() - await adapter.on_event(turn_input(tools)) - - assert tools.messages_sent == [] - assert failure_reports(tools) == [] - - @pytest.mark.asyncio - @pytest.mark.parametrize(("reply_tool", "arguments"), REPLY_SETTLING_CALLS) - async def test_disabled_relay_quiet_when_a_reply_tool_settled_the_turn( - self, reply_tool: str, arguments: dict[str, Any] - ) -> None: - adapter, mock_client = ready_letta_adapter( - LettaAdapterConfig(auto_relay=False), capabilities={Capability.FILES} - ) - mock_client.agents.messages.create.side_effect = scripted_letta_turn( - adapter, - room_id="room-1", - tool_calls=[(reply_tool, arguments)], - final_text="Done!", - ) - - tools = FakeAgentTools() - await adapter.on_message( - make_platform_message(), - tools, - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert "Done!" not in sent_contents(tools) - assert not reported_failures(tools) - - -# ────────────────────────────────────────────────────────────────────── -# Cold-boot history seeding -# ────────────────────────────────────────────────────────────────────── - - -class TestColdBootSeeding: - @pytest.fixture - def adapter_with_client(self) -> tuple[LettaAdapter, AsyncMock]: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test prompt" - adapter._mcp.server_id = "mcp-server-1" - adapter._mcp.tool_ids = [] - return adapter, mock_client - - @pytest.mark.asyncio - async def test_new_agent_in_room_with_history_is_seeded( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = adapter_with_client - mock_client.agents.create.return_value = make_mock_agent("fresh-agent") - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Hi again!") - ) - - history = LettaSessionState( - replay_messages=["[Alice]: The secret word is kumquat.", "[Bot]: Noted!"] - ) - tools = FakeAgentTools() - await adapter.on_message( - make_platform_message(content="what was the secret word?"), - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - content = mock_client.agents.messages.create.call_args.kwargs["messages"][0][ - "content" - ] - assert "joining an ongoing conversation" in content - assert "[Alice]: The secret word is kumquat." in content - assert "what was the secret word?" in content - # Seed is delivered exactly once - assert adapter._rooms["room-1"].pending_seed == [] - - @pytest.mark.asyncio - async def test_reported_turn_consumes_delivered_seed(self) -> None: - """A completed provider call consumes its seed before response handling.""" - adapter = LettaAdapter(config=LettaAdapterConfig(auto_relay=False)) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test prompt" - adapter._mcp.server_id = "mcp-server-1" - adapter._mcp.tool_ids = [] - room_ctx = RoomContext( - agent_id="agent-1", pending_seed=["[Alice]: Earlier context"] - ) - adapter._rooms["room-1"] = room_ctx - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("The response was not sent through the tool.") - ) - - await adapter.on_message( - make_platform_message(), - FakeAgentTools(), - LettaSessionState(), - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - assert room_ctx.pending_seed == [] - assert room_ctx.last_interaction is not None - - @pytest.mark.asyncio - async def test_failed_first_turn_preserves_pending_seed( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - """Cold-boot history must survive a failed first turn — clearing the - seed at compose time would drop replay context forever.""" - adapter, mock_client = adapter_with_client - mock_client.agents.create.return_value = make_mock_agent("fresh-agent") - mock_client.agents.messages.create.side_effect = TimeoutError("slow letta") - - history = LettaSessionState( - replay_messages=["[Alice]: The secret word is kumquat."] - ) - tools = FakeAgentTools() - with pytest.raises(TurnResultAlreadyReported): - await adapter.on_message( - make_platform_message(content="what was the secret word?"), - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - assert adapter._rooms["room-1"].pending_seed == [ - "[Alice]: The secret word is kumquat." - ] - - @pytest.mark.asyncio - async def test_per_room_does_not_resume_config_agent_id( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - """config.agent_id is not a per-room resume target — each room without - persisted history must get its own Letta agent.""" - adapter, mock_client = adapter_with_client - adapter.config = adapter.config.model_copy( - update={"agent_id": "shared-bootstrap-id"} - ) - mock_client.agents.create.return_value = make_mock_agent("room-agent") - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Hello!") - ) - - await adapter.on_message( - make_platform_message(room_id="room-1"), - FakeAgentTools(), - LettaSessionState(), - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - mock_client.agents.retrieve.assert_not_called() - mock_client.agents.create.assert_awaited_once() - - @pytest.mark.asyncio - async def test_second_message_is_not_reseeded( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - adapter, mock_client = adapter_with_client - mock_client.agents.create.return_value = make_mock_agent("fresh-agent") - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Hello!") - ) - - history = LettaSessionState(replay_messages=["[Alice]: earlier message"]) - tools = FakeAgentTools() - for _ in range(2): - await adapter.on_message( - make_platform_message(), - tools, - history, - None, - None, - is_session_bootstrap=False, - room_id="room-1", - ) - - second_content = mock_client.agents.messages.create.call_args.kwargs[ - "messages" - ][0]["content"] - assert "[Alice]: earlier message" not in second_content - - @pytest.mark.asyncio - async def test_resumed_agent_is_not_seeded( - self, adapter_with_client: tuple[LettaAdapter, AsyncMock] - ) -> None: - """Resume-by-id stays the fast path — the live agent already carries - its own context, so replaying history would duplicate it.""" - adapter, mock_client = adapter_with_client - mock_client.agents.retrieve.return_value = make_mock_agent("live-agent") - mock_client.agents.tools.list.return_value = make_mock_tool_page() - mock_client.agents.messages.create.return_value = make_letta_response( - make_assistant_message("Resumed!") - ) - - history = LettaSessionState( - agent_id="live-agent", - replay_messages=["[Alice]: earlier message"], - ) - tools = FakeAgentTools() - await adapter.on_message( - make_platform_message(), - tools, - history, - None, - None, - is_session_bootstrap=True, - room_id="room-1", - ) - - content = mock_client.agents.messages.create.call_args.kwargs["messages"][0][ - "content" - ] - assert "[Alice]: earlier message" not in content - mock_client.agents.create.assert_not_called() - - -# ────────────────────────────────────────────────────────────────────── -# Agent-create options -# ────────────────────────────────────────────────────────────────────── - - -class TestAgentCreateOptions: - @pytest.mark.asyncio - async def test_embedding_passed_when_configured(self) -> None: - adapter = LettaAdapter( - config=LettaAdapterConfig(embedding="openai/text-embedding-3-small") - ) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - mock_client.agents.create.return_value = make_mock_agent() - - await adapter._create_agent("room-1") - - create_kwargs = mock_client.agents.create.call_args.kwargs - assert create_kwargs["embedding"] == "openai/text-embedding-3-small" - - @pytest.mark.asyncio - async def test_embedding_omitted_by_default(self) -> None: - """Letta Cloud picks its own default — None must not be sent.""" - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - mock_client.agents.create.return_value = make_mock_agent() - - await adapter._create_agent("room-1") - - assert "embedding" not in mock_client.agents.create.call_args.kwargs - - @pytest.mark.asyncio - async def test_per_room_persona_carries_room_id(self) -> None: - """Self-hosted MCP schemas require room_id per call; a per_room agent - gets it in its persona block.""" - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - mock_client.agents.create.return_value = make_mock_agent() - - await adapter._create_agent("room-42") - - blocks = mock_client.agents.create.call_args.kwargs["memory_blocks"] - persona = next(b for b in blocks if b["label"] == "persona") - assert "room-42" in persona["value"] - - @pytest.mark.asyncio - async def test_external_mode_persona_omits_room_id(self) -> None: - """External band-mcp tool schemas carry no room_id argument.""" - adapter = LettaAdapter( - config=LettaAdapterConfig(mcp=LettaMCPConfig(mode="external")) - ) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._system_prompt = "Test" - adapter._mcp.tool_ids = [] - adapter._mcp.server_id = "mcp-server-1" - mock_client.agents.create.return_value = make_mock_agent() - - await adapter._create_agent("room-42") - - blocks = mock_client.agents.create.call_args.kwargs["memory_blocks"] - persona = next(b for b in blocks if b["label"] == "persona") - assert "room-42" not in persona["value"] - - -# ────────────────────────────────────────────────────────────────────── -# delete_agents_on_cleanup -# ────────────────────────────────────────────────────────────────────── - - -class TestDeleteAgentsOnCleanup: - @pytest.mark.asyncio - async def test_opt_in_deletes_agent_instead_of_consolidating(self) -> None: - adapter = LettaAdapter(config=LettaAdapterConfig(delete_agents_on_cleanup=True)) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - await adapter.on_cleanup("room-1") - - mock_client.agents.delete.assert_awaited_once_with("agent-1") - # No consolidation prompt when the agent is being deleted - mock_client.agents.messages.create.assert_not_called() - - @pytest.mark.asyncio - async def test_default_keeps_agent(self) -> None: - adapter = LettaAdapter() - mock_client = AsyncMock() - adapter._client = mock_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - await adapter.on_cleanup("room-1") - - mock_client.agents.delete.assert_not_called() - - @pytest.mark.asyncio - async def test_can_skip_consolidation_on_cleanup(self) -> None: - adapter = LettaAdapter( - config=LettaAdapterConfig(consolidate_memory_on_cleanup=False) - ) - mock_client = AsyncMock() - adapter._client = mock_client - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - await adapter.on_cleanup("room-1") - - mock_client.agents.delete.assert_not_called() - mock_client.agents.messages.create.assert_not_called() - assert "room-1" not in adapter._rooms - - @pytest.mark.asyncio - async def test_delete_failure_does_not_propagate(self) -> None: - adapter = LettaAdapter(config=LettaAdapterConfig(delete_agents_on_cleanup=True)) - mock_client = AsyncMock() - adapter._client = mock_client - mock_client.agents.delete.side_effect = Exception("gone already") - adapter._rooms["room-1"] = RoomContext(agent_id="agent-1") - - await adapter.on_cleanup("room-1") # should not raise - assert "room-1" not in adapter._rooms - - -class TestConfigEnvSourcing: - """provider_key env sourcing: LETTA_* names only, never bare vars.""" - - @pytest.fixture(autouse=True) - def clean_env(self, monkeypatch: pytest.MonkeyPatch) -> None: - for var in ("API_KEY", "PROVIDER_KEY", "LETTA_API_KEY", "LETTA_PROVIDER_KEY"): - monkeypatch.delenv(var, raising=False) - - def test_bare_env_vars_never_populate_provider_key( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - """A foreign secret in API_KEY/PROVIDER_KEY must not reach Letta.""" - monkeypatch.setenv("API_KEY", "foreign-secret") - monkeypatch.setenv("PROVIDER_KEY", "foreign-secret") - - assert LettaAdapterConfig().provider_key is None - - def test_letta_api_key_env_populates_provider_key( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setenv("LETTA_API_KEY", "cloud-key") - - assert LettaAdapterConfig().provider_key == "cloud-key" - - def test_prefixed_field_name_env_populates_provider_key( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setenv("LETTA_PROVIDER_KEY", "prefixed-key") - - assert LettaAdapterConfig().provider_key == "prefixed-key" - - def test_explicit_kwarg_wins_over_env( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setenv("LETTA_API_KEY", "env-key") - - assert LettaAdapterConfig(provider_key="kwarg-key").provider_key == "kwarg-key" - - def test_prefixed_env_fills_a_field_and_an_explicit_value_beats_it( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - monkeypatch.setenv("LETTA_MODEL", "openai/env-model") - - assert LettaAdapterConfig().model == "openai/env-model" - assert LettaAdapterConfig(model="openai/explicit").model == "openai/explicit" diff --git a/tests/e2e/baseline/smoke/activity/__init__.py b/tests/e2e/baseline/smoke/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/e2e/baseline/smoke/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/e2e/baseline/smoke/activity/conftest.py b/tests/e2e/baseline/smoke/activity/conftest.py new file mode 100644 index 000000000..2af5bb2f0 --- /dev/null +++ b/tests/e2e/baseline/smoke/activity/conftest.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from collections.abc import AsyncIterator + +import pytest + +from tests.e2e.baseline.settings import BaselineSettings +from tests.e2e.baseline.smoke.activity.kit import HeldLookup, HeldNativePeer + + +@pytest.fixture +async def held_lookup(baseline_settings: BaselineSettings) -> AsyncIterator[HeldLookup]: + held = HeldLookup(baseline_settings.e2e_timeout) + try: + yield held + finally: + held.release.set() + + +@pytest.fixture +async def native_peer( + baseline_settings: BaselineSettings, +) -> AsyncIterator[HeldNativePeer]: + async with HeldNativePeer(baseline_settings.e2e_timeout).running() as peer: + yield peer diff --git a/tests/e2e/baseline/smoke/activity/kit.py b/tests/e2e/baseline/smoke/activity/kit.py new file mode 100644 index 000000000..96f98f69f --- /dev/null +++ b/tests/e2e/baseline/smoke/activity/kit.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import asyncio +import os +import shlex +import sys +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager, suppress +from pathlib import Path +from tempfile import TemporaryDirectory +from uuid import uuid4 + +from pydantic import BaseModel + +from band.adapters.claude_sdk import ( + ClaudeApprovalOptions, + ClaudeSDKAdapter, + ClaudeSDKAdapterConfig, +) +from band.core.types import MessageType +from tests.e2e.baseline.smoke.samples.sample_tools import LookupInput +from tests.e2e.baseline.toolkit.capture import ReplyCapture +from tests.e2e.baseline.toolkit.provisioning import ( + AdapterCell, + ProvisionedAgent, + running_provisioned_agent, +) +from tests.e2e.baseline.toolkit.tools import ToolSpec + + +class HeldLookup: + def __init__(self, timeout_s: float) -> None: + self.timeout_s = timeout_s + self.argument_sentinel = f"argument-{uuid4().hex}" + self.result_sentinel = f"result-{uuid4().hex}" + self.entered = asyncio.Event() + self.release = asyncio.Event() + self.arguments: list[LookupInput] = [] + self.spec = ToolSpec(LookupInput, self.lookup) + + async def lookup(self, args: BaseModel) -> str: + assert isinstance(args, LookupInput) + self.arguments.append(args) + self.entered.set() + async with asyncio.timeout(self.timeout_s): + await self.release.wait() + return self.result_sentinel + + +class HeldNativePeer: + def __init__(self, timeout_s: float) -> None: + self.timeout_s = timeout_s + self.argument_sentinel = f"argument-{uuid4().hex}" + self.result_sentinel = f"result-{uuid4().hex}" + self.entered = asyncio.Event() + self.release = asyncio.Event() + self.requests: list[str] = [] + self.writers: set[asyncio.StreamWriter] = set() + self.handlers: set[asyncio.Task[object]] = set() + self.url = "" + + async def handle( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + task = asyncio.current_task() + assert task is not None + self.handlers.add(task) + self.writers.add(writer) + try: + async with asyncio.timeout(self.timeout_s): + request = await reader.readuntil(b"\r\n\r\n") + self.requests.append(request.split(b"\r\n", 1)[0].decode()) + self.entered.set() + await self.release.wait() + body = self.result_sentinel.encode() + writer.write( + b"HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: " + + str(len(body)).encode() + + b"\r\n\r\n" + + body + ) + await writer.drain() + except (TimeoutError, ConnectionError, asyncio.IncompleteReadError): + self.entered.set() + finally: + writer.close() + with suppress(ConnectionError): + await writer.wait_closed() + self.writers.discard(writer) + self.handlers.discard(task) + + @property + def command(self) -> str: + script = f"import sys, urllib.request; sys.stdout.write(urllib.request.urlopen({self.url!r}, timeout={self.timeout_s!r}).read().decode())" + if os.name == "nt": + executable = sys.executable.replace("'", "''") + escaped_script = script.replace("'", "''") + return f"& '{executable}' -c '{escaped_script}'" + return f"{shlex.quote(sys.executable)} -c {shlex.quote(script)}" + + @asynccontextmanager + async def running(self) -> AsyncIterator[HeldNativePeer]: + server = await asyncio.start_server(self.handle, "127.0.0.1", 0) + port = server.sockets[0].getsockname()[1] + self.url = f"http://127.0.0.1:{port}/{self.argument_sentinel}" + try: + yield self + finally: + self.release.set() + server.close() + await server.wait_closed() + for writer in tuple(self.writers): + writer.close() + await asyncio.gather(*self.handlers, return_exceptions=True) + + +@asynccontextmanager +async def native_activity_agent(cell: AdapterCell) -> AsyncIterator[ProvisionedAgent]: + with TemporaryDirectory(prefix="band-activity-native-") as directory: + adapter = ClaudeSDKAdapter( + ClaudeSDKAdapterConfig( + model=cell.settings.llm_models.anthropic_model, + cwd=str(Path(directory).resolve()), + approvals=ClaudeApprovalOptions( + mode="auto_accept", text_notifications=False + ), + custom_section="Use your native Bash or PowerShell tool to execute the user's exact test command. Report its actual output using band_send_message. Do not substitute a different tool or narrate the command.", + ), + emit=(), + ) + async with running_provisioned_agent( + adapter, cell.resources, label="activity-native" + ) as agent: + yield agent + + +async def assert_no_tool_narration(capture: ReplyCapture, agent_id: str) -> None: + for kind in (MessageType.TOOL_CALL, MessageType.TOOL_RESULT): + events = await capture.events(kind, sender_id=agent_id) + events.assert_none() diff --git a/tests/e2e/baseline/smoke/activity/test_native_reporting.py b/tests/e2e/baseline/smoke/activity/test_native_reporting.py new file mode 100644 index 000000000..e95ef9c7d --- /dev/null +++ b/tests/e2e/baseline/smoke/activity/test_native_reporting.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from tests.baseline.adapter import Adapter +from tests.e2e.baseline.agents import per_adapter +from tests.e2e.baseline.smoke.activity.kit import ( + HeldNativePeer, + assert_no_tool_narration, + native_activity_agent, +) +from tests.e2e.baseline.smoke.samples.approvals import CLAUDE_SHELL_TOOLS +from tests.e2e.baseline.toolkit.capture import CaptureFactory +from tests.e2e.baseline.toolkit.provisioning import AdapterCell +from tests.e2e.baseline.toolkit.user_ops import UserOps + + +class TestNativeActivity: + @per_adapter(Adapter.CLAUDE_SDK) + @pytest.mark.asyncio(loop_scope="session") + @pytest.mark.timeout(extra=120) + async def test_native_shell_reports_privately_without_narration( + self, + cell: AdapterCell, + native_peer: HeldNativePeer, + user_ops: UserOps, + reply_capture: CaptureFactory, + ) -> None: + async with native_activity_agent(cell) as agent: + try: + room_id = await cell.resources.provision_room( + title="activity-native", participants=[agent.id] + ) + async with reply_capture(room_id, observe_activity=True) as capture: + mid = await user_ops.send_message( + room_id, + f"Execute this exact command once through native Bash/PowerShell, then send me its actual output: {native_peer.command}", + mention_id=agent.id, + mention_name=agent.name, + ) + async with asyncio.timeout(cell.settings.e2e_timeout): + await native_peer.entered.wait() + await capture.wait_for_tool_activity(agent.id, CLAUDE_SHELL_TOOLS) + mark = capture.activities.snapshot() + native_peer.release.set() + replies = await capture.wait_for_reply(mid, agent.id) + await capture.wait_for_activity_stop(agent.id, since=mark) + assert ( + f"GET /{native_peer.argument_sentinel} HTTP/1.1" + in native_peer.requests + ) + replies.assert_contains_any([native_peer.result_sentinel]) + capture.activities.assert_names_only( + native_peer.argument_sentinel, native_peer.result_sentinel + ) + await assert_no_tool_narration(capture, agent.id) + finally: + native_peer.release.set() diff --git a/tests/e2e/baseline/smoke/activity/test_sdk_reporting.py b/tests/e2e/baseline/smoke/activity/test_sdk_reporting.py new file mode 100644 index 000000000..23aa76cba --- /dev/null +++ b/tests/e2e/baseline/smoke/activity/test_sdk_reporting.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from band.core.types import AdapterFeatures +from tests.baseline.adapter import Adapter +from tests.e2e.baseline.agents import per_adapter +from tests.e2e.baseline.smoke.activity.kit import HeldLookup, assert_no_tool_narration +from tests.e2e.baseline.smoke.samples.sample_tools import ( + LOOKUP_PROMPT, + lookup_code_instruction, +) +from tests.e2e.baseline.toolkit.capture import CaptureFactory +from tests.e2e.baseline.toolkit.provisioning import AdapterCell +from tests.e2e.baseline.toolkit.user_ops import UserOps + + +class TestSDKActivity: + @per_adapter( + Adapter.ANTHROPIC, prompt=LOOKUP_PROMPT, features=AdapterFeatures(emit=()) + ) + @pytest.mark.asyncio(loop_scope="session") + @pytest.mark.timeout(extra=120) + async def test_held_sdk_tool_reports_privately_without_narration( + self, + cell: AdapterCell, + held_lookup: HeldLookup, + user_ops: UserOps, + reply_capture: CaptureFactory, + ) -> None: + async with cell.running(tools=[held_lookup.spec]) as agent: + try: + room_id = await cell.resources.provision_room( + title="activity-sdk", participants=[agent.id] + ) + async with reply_capture(room_id, observe_activity=True) as capture: + mid = await user_ops.send_message( + room_id, + f"{lookup_code_instruction('alpha')}. Pass note='{held_lookup.argument_sentinel}' exactly; relay the tool's returned value.", + mention_id=agent.id, + mention_name=agent.name, + ) + async with asyncio.timeout(cell.settings.e2e_timeout): + await held_lookup.entered.wait() + await capture.wait_for_tool_activity( + agent.id, {held_lookup.spec.name} + ) + mark = capture.activities.snapshot() + held_lookup.release.set() + replies = await capture.wait_for_reply(mid, agent.id) + await capture.wait_for_activity_stop(agent.id, since=mark) + replies.assert_contains_any([held_lookup.result_sentinel]) + assert ( + held_lookup.arguments[0].note == held_lookup.argument_sentinel + ) + capture.activities.assert_names_only( + held_lookup.argument_sentinel, held_lookup.result_sentinel + ) + await assert_no_tool_narration(capture, agent.id) + finally: + held_lookup.release.set() diff --git a/tests/e2e/baseline/toolkit/activity.py b/tests/e2e/baseline/toolkit/activity.py new file mode 100644 index 000000000..4c3fc872b --- /dev/null +++ b/tests/e2e/baseline/toolkit/activity.py @@ -0,0 +1,87 @@ +"""Opt-in test observation of ephemeral room activity, independent of SDK receiving.""" + +from __future__ import annotations + +import json +from collections.abc import AsyncIterator, Awaitable, Callable, Collection +from contextlib import asynccontextmanager +from dataclasses import dataclass +from enum import StrEnum +from typing import Any + +from phoenix_channels_python_client.phx_messages import PHXMessage + +from band.client.streaming import WebSocketClient +from tests.e2e.baseline.toolkit.ws import TrackingWebSocketClient + + +class ActivityEvent(StrEnum): + STARTED = "agent_activity_started" + CHANGED = "agent_activity_changed" + STOPPED = "agent_activity_stopped" + + +def activity_topic(room_id: str) -> str: + return f"room_activity:{room_id}" + + +@dataclass(frozen=True) +class ActivityFrame: + event: ActivityEvent + payload: dict[str, Any] + + @property + def agent_id(self) -> str: + return str(self.payload.get("agent_id", "")) + + @property + def detail(self) -> dict[str, Any]: + detail = self.payload.get("activity") + return detail if isinstance(detail, dict) else {} + + +class Activities(list[ActivityFrame]): + def has_tool(self, agent_id: str, names: Collection[str]) -> bool: + return any( + frame.agent_id == agent_id + and frame.detail.get("kind") == "tool_call" + and frame.detail.get("name") in names + for frame in self + ) + + def snapshot(self) -> int: + return len(self) + + def has_stopped(self, agent_id: str, *, since: int = 0) -> bool: + return any( + frame.agent_id == agent_id and frame.event == ActivityEvent.STOPPED + for frame in self[since:] + ) + + def assert_names_only(self, *private_values: str) -> None: + for frame in self: + assert set(frame.detail) <= {"kind", "name"}, frame.detail + serialized = json.dumps([frame.payload for frame in self]) + for value in private_values: + assert value not in serialized, ( + "Private tool data appeared in room activity" + ) + + +@asynccontextmanager +async def activity_subscription( + ws: WebSocketClient | TrackingWebSocketClient, + room_id: str, + handler: Callable[[PHXMessage], Awaitable[None]], +) -> AsyncIterator[None]: + raw = ws.ws if isinstance(ws, TrackingWebSocketClient) else ws + client = raw.client + if client is None: + raise RuntimeError("Activity observation requires a connected WebSocket") + topic = activity_topic(room_id) + try: + await client.subscribe_to_topic(topic, handler) + yield + finally: + if topic in raw.joined_topics(): + await client.unsubscribe_from_topic(topic) diff --git a/tests/e2e/baseline/toolkit/capture.py b/tests/e2e/baseline/toolkit/capture.py index 0402cb306..400cb3a97 100644 --- a/tests/e2e/baseline/toolkit/capture.py +++ b/tests/e2e/baseline/toolkit/capture.py @@ -39,12 +39,13 @@ import sys from collections import defaultdict from collections.abc import AsyncIterator, Callable, Iterable -from contextlib import AbstractAsyncContextManager, asynccontextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager, nullcontext from datetime import UTC, datetime -from typing import Any +from typing import Any, Protocol from band_rest import AsyncRestClient from phoenix_channels_python_client.exceptions import PHXClientError +from phoenix_channels_python_client.phx_messages import PHXMessage from band.client.streaming import ( DeliveryStatus, @@ -53,6 +54,12 @@ ) from band.core.types import MessageType, metadata_to_dict from tests.e2e.baseline.settings import BaselineSettings +from tests.e2e.baseline.toolkit.activity import ( + Activities, + ActivityEvent, + ActivityFrame, + activity_subscription, +) from tests.e2e.baseline.toolkit.observations import ( Errors, Events, @@ -116,6 +123,7 @@ def __init__( # One agent-auth REST client per agent id, reused across memory reads. self._agent_clients: dict[str, AsyncRestClient] = {} self.messages: Replies = Replies() + self.activities = Activities() # message_id -> {recipient_id: delivery-state dict}, fed by # ``message_updated`` events. The authoritative "agent finished this # message" signal, independent of any LLM reply text. @@ -252,6 +260,21 @@ async def _log_delivery_timeout(self, message_id: str, recipient_id: str) -> Non error, ) + async def wait_for_tool_activity( + self, agent_id: str, names: Iterable[str] + ) -> Activities: + accepted = frozenset(names) + await self.wait_until(lambda _: self.activities.has_tool(agent_id, accepted)) + return self.activities + + async def wait_for_activity_stop( + self, agent_id: str, *, since: int = 0 + ) -> Activities: + await self.wait_until( + lambda _: self.activities.has_stopped(agent_id, since=since) + ) + return self.activities + async def wait_until( self, predicate: Callable[[list[MessageCreatedPayload]], bool], @@ -645,6 +668,7 @@ async def reply_capture( user_ops: UserOps | None = None, settings: BaselineSettings | None = None, deadline_s: float = DEFAULT_DEADLINE_S, + observe_activity: bool = False, ) -> AsyncIterator[ReplyCapture]: """Subscribe to a room before sending, yield a capture, leave on exit.""" capture = ReplyCapture( @@ -657,9 +681,22 @@ async def handler(payload: MessageCreatedPayload) -> None: async def updated_handler(payload: MessageCreatedPayload) -> None: capture._on_message_updated(payload) + async def activity_handler(message: PHXMessage) -> None: + try: + event = ActivityEvent(message.event) + except ValueError: + return + capture.activities.append(ActivityFrame(event, dict(message.payload))) + capture._nudge.set() + await ws.join_chat_room_channel(room_id, handler, updated_handler) try: - yield capture + async with ( + activity_subscription(ws, room_id, activity_handler) + if observe_activity + else nullcontext() + ): + yield capture finally: # Best-effort, but only for the transport: a leave the server never acks # (``PHXTopicError``) or one attempted while the socket is down ( @@ -680,4 +717,7 @@ async def updated_handler(payload: MessageCreatedPayload) -> None: # Type of the ``reply_capture`` fixture: call with a room id, get a # ``ReplyCapture`` async context manager. Shared so smokes type their fixture # parameter without each redefining the same alias. -CaptureFactory = Callable[[str], AbstractAsyncContextManager[ReplyCapture]] +class CaptureFactory(Protocol): + def __call__( + self, room_id: str, *, observe_activity: bool = False + ) -> AbstractAsyncContextManager[ReplyCapture]: ... diff --git a/tests/e2e/baseline/toolkit/tools.py b/tests/e2e/baseline/toolkit/tools.py index 5bb005401..e2ec4b57b 100644 --- a/tests/e2e/baseline/toolkit/tools.py +++ b/tests/e2e/baseline/toolkit/tools.py @@ -12,7 +12,7 @@ from __future__ import annotations -from collections.abc import Callable +from collections.abc import Awaitable, Callable from dataclasses import dataclass from typing import Any @@ -31,7 +31,7 @@ class ToolSpec: """A custom tool: a Pydantic input ``model`` plus a ``handler(model) -> str``.""" model: type[BaseModel] - handler: Callable[[BaseModel], str] + handler: Callable[[BaseModel], str | Awaitable[str]] @property def name(self) -> str: diff --git a/tests/framework_conformance/crewaicoverage/__init__.py b/tests/framework_conformance/crewaicoverage/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/framework_conformance/crewaicoverage/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/framework_conformance/crewaicoverage/kit.py b/tests/framework_conformance/crewaicoverage/kit.py new file mode 100644 index 000000000..218788b9e --- /dev/null +++ b/tests/framework_conformance/crewaicoverage/kit.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from tests.framework_conformance import venv_job_coverage as vjc + +_CREWAI_ONLY_MODULES = { + "crewai": "crewai", + "nest-asyncio": "nest_asyncio", + "pillow": "PIL", +} + + +def _job_command() -> str: + return vjc.job_command("test-crewai", "Run crewai tests") + + +def activity_command() -> str: + return vjc.job_command("test-crewai", "Run CrewAI activity tests") diff --git a/tests/framework_conformance/crewaicoverage/test_collection.py b/tests/framework_conformance/crewaicoverage/test_collection.py new file mode 100644 index 000000000..b68ecb2b2 --- /dev/null +++ b/tests/framework_conformance/crewaicoverage/test_collection.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from tests.framework_conformance import venv_job_coverage as vjc +from tests.framework_conformance.crewaicoverage.kit import ( + _CREWAI_ONLY_MODULES, + _job_command, + activity_command, +) + + +class TestCrewaiJobRunsEveryTestThatNeedsItsVenv: + def test_crewai_job_runs_every_test_that_needs_its_venv(self) -> None: + pattern = vjc.needs_venv_pattern(frozenset(_CREWAI_ONLY_MODULES.values())) + needed = vjc.tests_needing_venv(pattern) + command = _job_command() + listed = vjc.job_paths(command) + missing = sorted( + str(path) + for path in needed + if not vjc.covered_by(path, listed, command) + and not vjc.covered_by( + path, vjc.job_paths(activity_command()), activity_command() + ) + ) + assert not missing, ( + "these test files need a dev-crewai-only dependency but ci.yml's crewai " + f"job never collects them, so they run nowhere: {missing}" + ) diff --git a/tests/framework_conformance/crewaicoverage/test_dependencies.py b/tests/framework_conformance/crewaicoverage/test_dependencies.py new file mode 100644 index 000000000..1bf150144 --- /dev/null +++ b/tests/framework_conformance/crewaicoverage/test_dependencies.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from tests.framework_conformance import venv_job_coverage as vjc +from tests.framework_conformance.crewaicoverage.kit import ( + _CREWAI_ONLY_MODULES, +) + + +class TestCrewaiOnlyDependencySetMatchesPyproject: + def test_crewai_only_dependency_set_matches_pyproject(self) -> None: + """The detected module set is derived from a real dep list, not a guess.""" + assert vjc.only_distributions("dev-crewai") == set(_CREWAI_ONLY_MODULES), ( + "the dev-crewai-only dependencies changed; update _CREWAI_ONLY_MODULES so " + "this guard still detects every test that needs that venv" + ) diff --git a/tests/framework_conformance/crewaicoverage/test_detection.py b/tests/framework_conformance/crewaicoverage/test_detection.py new file mode 100644 index 000000000..438a639ca --- /dev/null +++ b/tests/framework_conformance/crewaicoverage/test_detection.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from pathlib import Path + +from tests.framework_conformance import venv_job_coverage as vjc +from tests.framework_conformance.crewaicoverage.kit import ( + _CREWAI_ONLY_MODULES, +) + + +class TestDetectionFindsTheTestsThatOnlyTheCrewaiVenvCanRun: + def test_detection_finds_the_tests_that_only_the_crewai_venv_can_run(self) -> None: + """Guard the guard: if the pattern drifts, the tests above pass empty. + + These are the whole set today — `phase3` for its nest_asyncio cases, + `test_files_image_passthrough_matrix.py` for its guarded `import crewai` + (real-package probe, `_CREWAI_AVAILABLE` skip elsewhere), the rest for + importing crewai itself. + """ + pattern = vjc.needs_venv_pattern(frozenset(_CREWAI_ONLY_MODULES.values())) + needed = vjc.tests_needing_venv(pattern) + assert needed == { + Path("tests/integrations/crewai/activity/test_execution.py"), + Path("tests/adapters/test_crewai_flow_phase3.py"), + Path("tests/integrations/test_crewai_flow_real_sdk.py"), + Path("tests/integrations/test_crewai_real_tools.py"), + Path("tests/integrations/test_crewai_completion.py"), + Path("tests/test_capability_gating_e2e.py"), + Path("tests/framework_conformance/test_files_image_passthrough_matrix.py"), + } diff --git a/tests/framework_conformance/crewaicoverage/test_paths.py b/tests/framework_conformance/crewaicoverage/test_paths.py new file mode 100644 index 000000000..2ab18018c --- /dev/null +++ b/tests/framework_conformance/crewaicoverage/test_paths.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from tests.framework_conformance import venv_job_coverage as vjc +from tests.framework_conformance.crewaicoverage.kit import ( + _job_command, + activity_command, +) +from tests.paths import REPO_ROOT + + +class TestCrewaiJobPathsAllExist: + def test_crewai_job_paths_all_exist(self) -> None: + """The other drift direction: a listed path that was renamed or deleted is a + silently empty target, since pytest is given the whole list at once.""" + listed = vjc.job_paths(_job_command()) | vjc.job_paths(activity_command()) + stale = sorted(str(path) for path in listed if not (REPO_ROOT / path).exists()) + assert not stale, ( + f"ci.yml's crewai job lists paths that no longer exist: {stale}" + ) diff --git a/tests/framework_conformance/crewaicoverage/test_strictness.py b/tests/framework_conformance/crewaicoverage/test_strictness.py new file mode 100644 index 000000000..1b2e069b7 --- /dev/null +++ b/tests/framework_conformance/crewaicoverage/test_strictness.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +import pytest + +from tests.framework_configs.sentinel import StrictnessSettings + + +class TestMissingFrameworkOptoutIsParsedAsABoolean: + @pytest.mark.parametrize("flag", ["1", "0", ""]) + def test_missing_framework_optout_is_parsed_as_a_boolean( + self, flag: str, monkeypatch: pytest.MonkeyPatch + ) -> None: + """``BAND_ALLOW_MISSING_FRAMEWORKS=0`` must keep strict mode ON. + + The flag is set explicitly rather than conditionally: e2e.yml sends "0" for every + lane that should stay strict, and only ci.yml's crewai/parlant jobs send "1". A + presence check reads that "0" as an opt-out — so the value has to be parsed as a + boolean, or a cell asking to stay strict silently disables the fail-loud guard + instead. + """ + monkeypatch.setenv("CI", "true") + monkeypatch.setenv("BAND_ALLOW_MISSING_FRAMEWORKS", flag) + + settings = StrictnessSettings() + strict = settings.ci and not settings.band_allow_missing_frameworks + assert strict is (flag != "1") diff --git a/tests/framework_conformance/parlantcoverage/__init__.py b/tests/framework_conformance/parlantcoverage/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/framework_conformance/parlantcoverage/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/framework_conformance/parlantcoverage/kit.py b/tests/framework_conformance/parlantcoverage/kit.py new file mode 100644 index 000000000..9c797de12 --- /dev/null +++ b/tests/framework_conformance/parlantcoverage/kit.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from tests.framework_conformance import venv_job_coverage as vjc + +_PARLANT_ONLY_MODULES = { + "parlant": "parlant", + "werkzeug": "werkzeug", +} + + +def _job_command() -> str: + return vjc.job_command("test-parlant", "Run parlant tests") + + +def activity_command() -> str: + return vjc.job_command("test-parlant", "Run Parlant activity tests") diff --git a/tests/framework_conformance/parlantcoverage/test_collection.py b/tests/framework_conformance/parlantcoverage/test_collection.py new file mode 100644 index 000000000..8155bd5d4 --- /dev/null +++ b/tests/framework_conformance/parlantcoverage/test_collection.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from tests.framework_conformance import venv_job_coverage as vjc +from tests.framework_conformance.parlantcoverage.kit import ( + _PARLANT_ONLY_MODULES, + _job_command, + activity_command, +) + + +class TestParlantJobRunsEveryTestThatNeedsItsVenv: + def test_parlant_job_runs_every_test_that_needs_its_venv(self) -> None: + pattern = vjc.needs_venv_pattern(frozenset(_PARLANT_ONLY_MODULES.values())) + needed = vjc.tests_needing_venv(pattern) + command = _job_command() + listed = vjc.job_paths(command) + missing = sorted( + str(path) + for path in needed + if not vjc.covered_by(path, listed, command) + and not vjc.covered_by( + path, vjc.job_paths(activity_command()), activity_command() + ) + ) + assert not missing, ( + "these test files need a dev-parlant-only dependency but ci.yml's parlant " + f"job never collects them, so they run nowhere: {missing}" + ) diff --git a/tests/framework_conformance/parlantcoverage/test_dependencies.py b/tests/framework_conformance/parlantcoverage/test_dependencies.py new file mode 100644 index 000000000..465d6cbc7 --- /dev/null +++ b/tests/framework_conformance/parlantcoverage/test_dependencies.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +from tests.framework_conformance import venv_job_coverage as vjc +from tests.framework_conformance.parlantcoverage.kit import ( + _PARLANT_ONLY_MODULES, +) + + +class TestParlantOnlyDependencySetMatchesPyproject: + def test_parlant_only_dependency_set_matches_pyproject(self) -> None: + """The detected module set is derived from a real dep list, not a guess.""" + assert vjc.only_distributions("dev-parlant") == set(_PARLANT_ONLY_MODULES), ( + "the dev-parlant-only dependencies changed; update _PARLANT_ONLY_MODULES so " + "this guard still detects every test that needs that venv" + ) diff --git a/tests/framework_conformance/parlantcoverage/test_detection.py b/tests/framework_conformance/parlantcoverage/test_detection.py new file mode 100644 index 000000000..c1ce325b9 --- /dev/null +++ b/tests/framework_conformance/parlantcoverage/test_detection.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from pathlib import Path + +from tests.framework_conformance import venv_job_coverage as vjc +from tests.framework_conformance.parlantcoverage.kit import ( + _PARLANT_ONLY_MODULES, +) + + +class TestDetectionFindsTheTestsThatOnlyTheParlantVenvCanRun: + def test_detection_finds_the_tests_that_only_the_parlant_venv_can_run(self) -> None: + """Guard the guard: if the pattern drifts, the tests above pass empty.""" + pattern = vjc.needs_venv_pattern(frozenset(_PARLANT_ONLY_MODULES.values())) + needed = vjc.tests_needing_venv(pattern) + assert needed == { + Path("tests/integrations/parlant/activity/test_execution.py"), + Path("tests/integrations/parlant/test_catalog.py"), + Path("tests/integrations/parlant/test_chattools.py"), + Path("tests/integrations/parlant/test_contacttools.py"), + Path("tests/integrations/parlant/test_customtools.py"), + Path("tests/integrations/parlant/test_filetools.py"), + Path("tests/integrations/parlant/test_guard.py"), + Path("tests/integrations/parlant/test_server.py"), + Path("tests/integrations/parlant/test_tasktools.py"), + Path("tests/integrations/parlant/test_vocabulary.py"), + } diff --git a/tests/framework_conformance/parlantcoverage/test_paths.py b/tests/framework_conformance/parlantcoverage/test_paths.py new file mode 100644 index 000000000..f2c960630 --- /dev/null +++ b/tests/framework_conformance/parlantcoverage/test_paths.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from tests.framework_conformance import venv_job_coverage as vjc +from tests.framework_conformance.parlantcoverage.kit import ( + _job_command, + activity_command, +) +from tests.paths import REPO_ROOT + + +class TestParlantJobPathsAllExist: + def test_parlant_job_paths_all_exist(self) -> None: + """The other drift direction: a listed path that was renamed or deleted is a + silently empty target, since pytest is given the whole list at once.""" + listed = vjc.job_paths(_job_command()) | vjc.job_paths(activity_command()) + stale = sorted(str(path) for path in listed if not (REPO_ROOT / path).exists()) + assert not stale, ( + f"ci.yml's parlant job lists paths that no longer exist: {stale}" + ) diff --git a/tests/framework_conformance/test_crewai_job_coverage.py b/tests/framework_conformance/test_crewai_job_coverage.py deleted file mode 100644 index 4406115bd..000000000 --- a/tests/framework_conformance/test_crewai_job_coverage.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Guard against drift in ci.yml's hand-listed crewai test paths. - -See `venv_job_coverage`'s module docstring for the general shape of this guard. -""" - -from __future__ import annotations - -from pathlib import Path - -import pytest - -from tests.framework_configs.sentinel import StrictnessSettings -from tests.framework_conformance import venv_job_coverage as vjc -from tests.paths import REPO_ROOT - -# Import names of the distributions only `dev-crewai` installs. Kept as a map so -# the drift test below can prove it still matches pyproject: a new dev-crewai-only -# dep fails there rather than silently narrowing what this guard detects. -_CREWAI_ONLY_MODULES = { - "crewai": "crewai", - "nest-asyncio": "nest_asyncio", - "pillow": "PIL", -} - - -def _job_command() -> str: - return vjc.job_command("test-crewai", "Run crewai tests") - - -def test_crewai_job_runs_every_test_that_needs_its_venv() -> None: - pattern = vjc.needs_venv_pattern(frozenset(_CREWAI_ONLY_MODULES.values())) - needed = vjc.tests_needing_venv(pattern) - command = _job_command() - listed = vjc.job_paths(command) - missing = sorted( - str(path) for path in needed if not vjc.covered_by(path, listed, command) - ) - assert not missing, ( - "these test files need a dev-crewai-only dependency but ci.yml's crewai " - f"job never collects them, so they run nowhere: {missing}" - ) - - -def test_crewai_job_paths_all_exist() -> None: - """The other drift direction: a listed path that was renamed or deleted is a - silently empty target, since pytest is given the whole list at once.""" - listed = vjc.job_paths(_job_command()) - stale = sorted(str(path) for path in listed if not (REPO_ROOT / path).exists()) - assert not stale, f"ci.yml's crewai job lists paths that no longer exist: {stale}" - - -def test_crewai_only_dependency_set_matches_pyproject() -> None: - """The detected module set is derived from a real dep list, not a guess.""" - assert vjc.only_distributions("dev-crewai") == set(_CREWAI_ONLY_MODULES), ( - "the dev-crewai-only dependencies changed; update _CREWAI_ONLY_MODULES so " - "this guard still detects every test that needs that venv" - ) - - -def test_detection_finds_the_tests_that_only_the_crewai_venv_can_run() -> None: - """Guard the guard: if the pattern drifts, the tests above pass empty. - - These are the whole set today — `phase3` for its nest_asyncio cases, - `test_files_image_passthrough_matrix.py` for its guarded `import crewai` - (real-package probe, `_CREWAI_AVAILABLE` skip elsewhere), the rest for - importing crewai itself. - """ - pattern = vjc.needs_venv_pattern(frozenset(_CREWAI_ONLY_MODULES.values())) - needed = vjc.tests_needing_venv(pattern) - assert needed == { - Path("tests/adapters/test_crewai_flow_phase3.py"), - Path("tests/integrations/test_crewai_flow_real_sdk.py"), - Path("tests/integrations/test_crewai_real_tools.py"), - Path("tests/integrations/test_crewai_completion.py"), - Path("tests/test_capability_gating_e2e.py"), - Path("tests/framework_conformance/test_files_image_passthrough_matrix.py"), - } - - -@pytest.mark.parametrize("flag", ["1", "0", ""]) -def test_missing_framework_optout_is_parsed_as_a_boolean( - flag: str, monkeypatch: pytest.MonkeyPatch -) -> None: - """``BAND_ALLOW_MISSING_FRAMEWORKS=0`` must keep strict mode ON. - - The flag is set explicitly rather than conditionally: e2e.yml sends "0" for every - lane that should stay strict, and only ci.yml's crewai/parlant jobs send "1". A - presence check reads that "0" as an opt-out — so the value has to be parsed as a - boolean, or a cell asking to stay strict silently disables the fail-loud guard - instead. - """ - monkeypatch.setenv("CI", "true") - monkeypatch.setenv("BAND_ALLOW_MISSING_FRAMEWORKS", flag) - - settings = StrictnessSettings() - strict = settings.ci and not settings.band_allow_missing_frameworks - assert strict is (flag != "1") diff --git a/tests/framework_conformance/test_parlant_job_coverage.py b/tests/framework_conformance/test_parlant_job_coverage.py deleted file mode 100644 index d6e5d883f..000000000 --- a/tests/framework_conformance/test_parlant_job_coverage.py +++ /dev/null @@ -1,72 +0,0 @@ -"""Guard against drift in ci.yml's hand-listed parlant test paths. - -See `venv_job_coverage`'s module docstring for the general shape of this guard. -``BAND_ALLOW_MISSING_FRAMEWORKS`` boolean-parsing is shared, non-parlant-specific -machinery — covered once in ``test_crewai_job_coverage.py``, not duplicated here. -""" - -from __future__ import annotations - -from pathlib import Path - -from tests.framework_conformance import venv_job_coverage as vjc -from tests.paths import REPO_ROOT - -# Import names of the distributions only `dev-parlant` installs. Kept as a map so -# the drift test below can prove it still matches pyproject: a new dev-parlant-only -# dep fails there rather than silently narrowing what this guard detects. -_PARLANT_ONLY_MODULES = { - "parlant": "parlant", - "werkzeug": "werkzeug", -} - - -def _job_command() -> str: - return vjc.job_command("test-parlant", "Run parlant tests") - - -def test_parlant_job_runs_every_test_that_needs_its_venv() -> None: - pattern = vjc.needs_venv_pattern(frozenset(_PARLANT_ONLY_MODULES.values())) - needed = vjc.tests_needing_venv(pattern) - command = _job_command() - listed = vjc.job_paths(command) - missing = sorted( - str(path) for path in needed if not vjc.covered_by(path, listed, command) - ) - assert not missing, ( - "these test files need a dev-parlant-only dependency but ci.yml's parlant " - f"job never collects them, so they run nowhere: {missing}" - ) - - -def test_parlant_job_paths_all_exist() -> None: - """The other drift direction: a listed path that was renamed or deleted is a - silently empty target, since pytest is given the whole list at once.""" - listed = vjc.job_paths(_job_command()) - stale = sorted(str(path) for path in listed if not (REPO_ROOT / path).exists()) - assert not stale, f"ci.yml's parlant job lists paths that no longer exist: {stale}" - - -def test_parlant_only_dependency_set_matches_pyproject() -> None: - """The detected module set is derived from a real dep list, not a guess.""" - assert vjc.only_distributions("dev-parlant") == set(_PARLANT_ONLY_MODULES), ( - "the dev-parlant-only dependencies changed; update _PARLANT_ONLY_MODULES so " - "this guard still detects every test that needs that venv" - ) - - -def test_detection_finds_the_tests_that_only_the_parlant_venv_can_run() -> None: - """Guard the guard: if the pattern drifts, the tests above pass empty.""" - pattern = vjc.needs_venv_pattern(frozenset(_PARLANT_ONLY_MODULES.values())) - needed = vjc.tests_needing_venv(pattern) - assert needed == { - Path("tests/integrations/parlant/test_catalog.py"), - Path("tests/integrations/parlant/test_chattools.py"), - Path("tests/integrations/parlant/test_contacttools.py"), - Path("tests/integrations/parlant/test_customtools.py"), - Path("tests/integrations/parlant/test_filetools.py"), - Path("tests/integrations/parlant/test_guard.py"), - Path("tests/integrations/parlant/test_server.py"), - Path("tests/integrations/parlant/test_tasktools.py"), - Path("tests/integrations/parlant/test_vocabulary.py"), - } diff --git a/tests/framework_conformance/test_strands_nooa_job_coverage.py b/tests/framework_conformance/test_strands_nooa_job_coverage.py index d639dd29e..715f21408 100644 --- a/tests/framework_conformance/test_strands_nooa_job_coverage.py +++ b/tests/framework_conformance/test_strands_nooa_job_coverage.py @@ -42,6 +42,11 @@ def test_job_selects_every_required_node(self, framework_environment: None) -> N coverage.job_command("test-strands-nooa", "Run Strands and NOOA tests") ) selected = coverage.selected_nodes(arguments) + activity_arguments = coverage.pytest_arguments( + coverage.job_command("test-strands-nooa", "Run Strands activity tests") + ) + assert "-k" not in activity_arguments + selected |= coverage.selected_nodes(activity_arguments) # Directory collection honors NOOA's supported-platform conftest gate. nooa_tests = REPO_ROOT / "tests/adapters/nooa" targets = { diff --git a/tests/integrations/crewai/__init__.py b/tests/integrations/crewai/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/integrations/crewai/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/integrations/crewai/activity/__init__.py b/tests/integrations/crewai/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/integrations/crewai/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/integrations/crewai/activity/kit.py b/tests/integrations/crewai/activity/kit.py new file mode 100644 index 000000000..77e0cfd42 --- /dev/null +++ b/tests/integrations/crewai/activity/kit.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from pydantic import BaseModel + + +class LookupInput(BaseModel): + """Look up a private value.""" + + note: str diff --git a/tests/integrations/crewai/activity/test_execution.py b/tests/integrations/crewai/activity/test_execution.py new file mode 100644 index 000000000..0207c60f1 --- /dev/null +++ b/tests/integrations/crewai/activity/test_execution.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import json + +import pytest + +from band.integrations.crewai.tools import ( + CrewAIToolContext, + NoopReporter, + build_band_crewai_tools, +) +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder +from tests.integrations.crewai.activity.kit import LookupInput + +pytest.importorskip("crewai") + + +class TestCrewAIActivity: + @pytest.mark.parametrize("failed", [False, True]) + def test_real_base_tool_reports_without_narration_and_ends_on_error( + self, failed: bool + ) -> None: + recorder = ActivityRecorder() + tools = FakeAgentTools() + tools._activity_observer = recorder + + async def lookup(args: LookupInput) -> str: + assert recorder.events == [{"kind": "tool_call", "name": "lookup"}] + assert args.note == "private-argument" + if failed: + raise ValueError("private-error") + return "private-result" + + built = build_band_crewai_tools( + get_context=lambda: CrewAIToolContext(room_id="room", tools=tools), + reporter=NoopReporter(), + custom_tools=[(LookupInput, lookup)], + ) + lookup_tool = next(tool for tool in built if tool.name == "lookup") + result = json.loads(lookup_tool.run(note="private-argument")) + assert result["status"] == ("error" if failed else "success") + assert recorder.events == [ + {"kind": "tool_call", "name": "lookup"}, + {"kind": "reasoning"}, + ] + assert not tools.events_sent diff --git a/tests/integrations/parlant/activity/__init__.py b/tests/integrations/parlant/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/integrations/parlant/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/integrations/parlant/activity/kit.py b/tests/integrations/parlant/activity/kit.py new file mode 100644 index 000000000..77e0cfd42 --- /dev/null +++ b/tests/integrations/parlant/activity/kit.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from pydantic import BaseModel + + +class LookupInput(BaseModel): + """Look up a private value.""" + + note: str diff --git a/tests/integrations/parlant/activity/test_execution.py b/tests/integrations/parlant/activity/test_execution.py new file mode 100644 index 000000000..eb689ff79 --- /dev/null +++ b/tests/integrations/parlant/activity/test_execution.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import asyncio + +import pytest + +pytest.importorskip("parlant.sdk") + +from parlant.core.tools import ToolContext + +from band.integrations.parlant.customtools import build_custom_tool +from band.integrations.parlant.sessiontools import set_session_tools +from band.integrations.parlant.tools import create_parlant_tools +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder +from tests.integrations.parlant.activity.kit import LookupInput +from tests.integrations.parlant.helpers import ToolServer + + +class TestParlantActivity: + @pytest.mark.parametrize("failed", [False, True]) + async def test_real_server_reports_only_calling_session_and_always_ends( + self, plugin_server: ToolServer, failed: bool + ) -> None: + tools, other = FakeAgentTools(), FakeAgentTools() + recorder, quiet = ActivityRecorder(), ActivityRecorder() + tools._activity_observer = recorder + other._activity_observer = quiet + set_session_tools("calling", tools) + set_session_tools("other", other) + entered, release = asyncio.Event(), asyncio.Event() + + async def lookup(args: LookupInput) -> str: + assert args.note == "private-argument" + entered.set() + await release.wait() + if failed: + raise ValueError("private-error") + return "private-result" + + await plugin_server.enable( + create_parlant_tools(custom_tools=[(LookupInput, lookup)]) + ) + call = asyncio.create_task( + plugin_server.call( + "lookup", session_id="calling", arguments={"note": "private-argument"} + ) + ) + try: + async with asyncio.timeout(5): + await entered.wait() + assert recorder.events == [{"kind": "tool_call", "name": "lookup"}] + assert not quiet.events + finally: + release.set() + result = await call + assert result == ( + "Error running lookup: private-error" if failed else "private-result" + ) + assert recorder.events == [ + {"kind": "tool_call", "name": "lookup"}, + {"kind": "reasoning"}, + ] + assert not tools.events_sent + + async def test_custom_handler_cancellation_ends_activity(self) -> None: + tools = FakeAgentTools() + recorder = ActivityRecorder() + tools._activity_observer = recorder + set_session_tools("cancelled", tools) + entered = asyncio.Event() + + async def lookup(args: LookupInput) -> str: + entered.set() + await asyncio.Event().wait() + return "unreachable" + + entry = build_custom_tool((LookupInput, lookup)) + task = asyncio.create_task( + entry.function( + ToolContext( + agent_id="agent", session_id="cancelled", customer_id="user" + ), + note="private-argument", + ) + ) + await entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert recorder.events == [ + {"kind": "tool_call", "name": "lookup"}, + {"kind": "reasoning"}, + ] diff --git a/tests/runtime/activityownership/kit.py b/tests/runtime/activityownership/kit.py index 7750c1ea5..702be2119 100644 --- a/tests/runtime/activityownership/kit.py +++ b/tests/runtime/activityownership/kit.py @@ -17,6 +17,8 @@ def __init__(self) -> None: self.hold_next_activity = False self.activity_entered = asyncio.Event() self.activity_release = asyncio.Event() + self.reject_detail = False + self.event_bodies: list[dict[str, Any]] = [] @property def working_transitions(self) -> list[bool]: @@ -35,6 +37,11 @@ async def respond(self, request: httpx.Request) -> httpx.Response: def answer(self, request: httpx.Request) -> httpx.Response: tail = request.url.path.rsplit("/", 1)[-1] + if tail == "events" and request.method == "POST": + self.event_bodies.append(json.loads(request.content)) if tail == "activity": - self.activity_bodies.append(json.loads(request.content)) + body = json.loads(request.content) + self.activity_bodies.append(body) + if self.reject_detail and "activity" in body: + return httpx.Response(422, json={"error": "unsupported activity"}) return super().answer(request) diff --git a/tests/runtime/activityownership/test_claude_native.py b/tests/runtime/activityownership/test_claude_native.py new file mode 100644 index 000000000..b37a1cbaa --- /dev/null +++ b/tests/runtime/activityownership/test_claude_native.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import json +from collections.abc import Callable + +import pytest + +from band.adapters.claude_sdk import ClaudeSDKAdapter +from band.converters.claude_sdk import ClaudeSDKSessionState +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from band.runtime.tools.registry import MCP_TOOL_PREFIX +from tests.adapters.claude_sdk.fakecli import scripted_claude +from tests.adapters.opencode.helpers import make_platform_message +from tests.baseline.decisions import ModelDecision +from tests.conftest import make_message_event +from tests.runtime.activityownership.kit import ActivityPlatform +from tests.runtime.helpers import ROOM_ID + + +class TestClaudeNativeWireActivity: + @pytest.mark.httpx_mock(should_mock=lambda request: request.url.host != "127.0.0.1") + async def test_real_context_observes_native_tool_without_tool_narration( + self, execution: Callable[..., ExecutionContext], peer: ActivityPlatform + ) -> None: + with scripted_claude() as cli: + cli.script( + [ + ModelDecision.call("Bash", command="private-argument"), + ModelDecision.call( + f"{MCP_TOOL_PREFIX}band_send_message", + content="private-result", + mentions=["@alice"], + ), + ] + ) + adapter = ClaudeSDKAdapter(emit=()) + await adapter.on_started("Test", "Test") + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + await adapter.on_message( + make_platform_message( + room_id=ROOM_ID, sender_id="user-456", sender_name="Alice" + ), + AgentTools.from_context(ctx), + ClaudeSDKSessionState(), + None, + None, + is_session_bootstrap=True, + room_id=ROOM_ID, + ) + + ctx = execution(handler) + try: + await ctx._process_event(make_message_event()) + assert { + "working": True, + "activity": {"kind": "tool_call", "name": "Bash"}, + } in peer.activity_bodies + assert peer.activity_bodies[-1] == {"working": False} + assert not any( + body.get("message_type") in {"tool_call", "tool_result"} + for body in peer.event_bodies + ) + assert "private-argument" not in json.dumps(peer.activity_bodies) + assert "private-result" not in json.dumps(peer.activity_bodies) + finally: + await adapter.cleanup_all() diff --git a/tests/runtime/activityownership/test_detached_detail_feature.py b/tests/runtime/activityownership/test_detached_detail_feature.py new file mode 100644 index 000000000..33a02435f --- /dev/null +++ b/tests/runtime/activityownership/test_detached_detail_feature.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +import asyncio +import json +from collections.abc import Callable +from typing import Any + +import pytest +from pytest_httpx import HTTPXMock + +from band.adapters.opencode import OpencodeAdapter +from band.core.types import Emit +from band.integrations.opencode.types import OpencodeSessionState +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from band.runtime.working_state import report_tool_activity +from tests.adapters.opencode.helpers import ( + FakeOpencodeClient, + event_permission, + event_session_idle, + event_tool_part, + make_platform_message, +) +from tests.conftest import make_message_event +from tests.mcpbackends import backends_created_by +from tests.runtime.activityownership.kit import ActivityPlatform +from tests.runtime.helpers import ROOM_ID + + +class TestDetachedActivityFeature: + @pytest.mark.looptime + @pytest.mark.parametrize( + "variant", ["normal", "silent", "downgrade", "cap", "disabled", "hub"] + ) + async def test_parked_approval_owns_detail_until_native_completion( + self, + execution: Callable[..., ExecutionContext], + peer: ActivityPlatform, + httpx_mock: HTTPXMock, + variant: str, + ) -> None: + peer.reject_detail = variant == "downgrade" + resumed, release = asyncio.Event(), asyncio.Event() + captured: list[AgentTools] = [] + + async def executing() -> None: + resumed.set() + await release.wait() + + def tool(status: str) -> dict[str, Any]: + return event_tool_part( + "sess-1", + "native", + tool="bash", + call_id="call", + status=status, + input_data={"command": "private-argument"}, + output="private-result", + ) + + client = FakeOpencodeClient( + prompt_event_sequences=[ + [tool("pending"), event_permission("sess-1", "ask")] + ], + reply_permission_events={ + "ask": [ + tool("running"), + executing, + tool("completed"), + event_session_idle("sess-1"), + ] + }, + ) + adapter = OpencodeAdapter( + client_factory=lambda _: client, + emit=() if variant == "silent" else (Emit.TOOL_CALLS,), + ) + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + captured.append(tools) + await adapter.on_message( + make_platform_message(room_id=ROOM_ID, sender_id="user-456"), + tools, + OpencodeSessionState(), + None, + None, + is_session_bootstrap=True, + room_id=ROOM_ID, + ) + + settings = {} + if variant == "cap": + settings["max_working_state_seconds"] = 5.0 + if variant == "disabled": + settings["enable_working_state"] = False + if variant == "hub": + settings["hub_room_id"] = ROOM_ID + with backends_created_by(): + await adapter.on_started("Test", "Test") + ctx = execution(handler, **settings) + try: + await ctx._process_event(make_message_event(msg_id="original")) + owner = captured[0]._scope + assert owner is not None + workers = tuple(owner.workers) + await ctx._process_event(make_message_event(msg_id="busy")) + await asyncio.sleep(9.1) + parked = list(peer.activity_bodies) + match variant: + case "disabled" | "hub": + assert not parked + assert not httpx_mock.get_requests() + case "downgrade": + assert parked[0].get("activity") == {"kind": "reasoning"} + assert len(parked[1:]) >= 3 + assert all(body == {"working": True} for body in parked[1:]) + case "cap": + assert parked[-1] == { + "working": True, + "activity": {"kind": "permission"}, + } + assert len(parked) == 4 + case _: + assert ( + parked[-3:] + == [{"working": True, "activity": {"kind": "permission"}}] + * 3 + ) + assert peer.marked("processed") == ["original"] + assert await adapter.try_handle_control( + make_platform_message( + room_id=ROOM_ID, content="approve ask", sender_id="user-456" + ), + captured[-1], + ) + await resumed.wait() + if variant in {"normal", "silent"}: + assert peer.activity_bodies[-1] == { + "working": True, + "activity": {"kind": "tool_call", "name": "bash"}, + } + if variant == "cap": + assert peer.activity_bodies == parked + release.set() + await asyncio.gather(*(worker.task for worker in workers)) + await ctx._join_activity_completion() + assert not client.aborted_sessions + assert "private-argument" not in json.dumps(peer.activity_bodies) + assert "private-result" not in json.dumps(peer.activity_bodies) + before = list(peer.activity_bodies) + await report_tool_activity(captured[0], "late-old-callback") + assert peer.activity_bodies == before + if variant == "silent": + assert not any( + event.get("message_type") in {"tool_call", "tool_result"} + for event in peer.event_bodies + ) + finally: + release.set() + await adapter.on_cleanup(ROOM_ID) diff --git a/tests/toolkit/activity/__init__.py b/tests/toolkit/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/toolkit/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/toolkit/activity/test_subscription.py b/tests/toolkit/activity/test_subscription.py new file mode 100644 index 000000000..8a9d2b7b5 --- /dev/null +++ b/tests/toolkit/activity/test_subscription.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import pytest +from phoenix_channels_python_client.exceptions import PHXTopicError + +from band.client.streaming import WebSocketClient +from band.testing import JoinOutcome, fake_phoenix_server +from tests.e2e.baseline.toolkit.activity import ActivityEvent, activity_topic +from tests.e2e.baseline.toolkit.capture import reply_capture + + +class TestActivitySubscription: + async def test_joins_before_trigger_dispatches_and_leaves(self) -> None: + async with ( + fake_phoenix_server() as server, + WebSocketClient(ws_url=server.url, api_key="test", agent_id=None) as ws, + ): + async with reply_capture( + ws, "room", observe_activity=True, deadline_s=1 + ) as capture: + topic = activity_topic("room") + assert topic in server.joined_topics + await server.push( + topic, + ActivityEvent.CHANGED, + { + "agent_id": "agent", + "working": True, + "activity": {"kind": "tool_call", "name": "Bash"}, + }, + ) + await capture.wait_for_tool_activity("agent", {"Bash"}) + mark = capture.activities.snapshot() + await server.push( + topic, + ActivityEvent.STOPPED, + {"agent_id": "agent", "working": False}, + ) + await capture.wait_for_activity_stop("agent", since=mark) + capture.activities.assert_names_only("private-input", "private-output") + assert topic not in ws.joined_topics() + assert "chat_room:room" not in ws.joined_topics() + + async def test_rejected_activity_join_fails_and_releases_ordinary_capture( + self, + ) -> None: + topic = activity_topic("room") + async with ( + fake_phoenix_server( + join_outcomes={topic: [JoinOutcome.REJECTED]} + ) as server, + WebSocketClient(ws_url=server.url, api_key="test", agent_id=None) as ws, + ): + with pytest.raises(PHXTopicError): + async with reply_capture(ws, "room", observe_activity=True): + pytest.fail( + "Rejected activity subscription must not run a scenario" + ) + assert "chat_room:room" not in ws.joined_topics() From 831b8517b54a2cf343d32fdf997a565670d406d3 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 18 Oct 2026 11:09:08 +0300 Subject: [PATCH 12/47] refactor: Consolidate activity reset and streaming usage ownership --- src/band/adapters/letta.py | 16 -- src/band/runtime/working_state.py | 13 +- tests/adapters/activity/kit.py | 2 +- tests/adapters/activity/test_letta_stream.py | 7 +- tests/adapters/test_usage_mapping.py | 270 ------------------ tests/adapters/usagemapping/__init__.py | 1 + .../usagemapping/test_agnousagemapping.py | 31 ++ .../test_claudesdkusagemapping.py | 33 +++ .../usagemapping/test_codexusagemapping.py | 26 ++ .../usagemapping/test_geminiusagemapping.py | 31 ++ .../test_googleadkusagemapping.py | 31 ++ .../usagemapping/test_isusageevent.py | 25 ++ .../usagemapping/test_opencodeusagemapping.py | 36 +++ .../test_turnusageconstructors.py | 52 ++++ 14 files changed, 280 insertions(+), 294 deletions(-) delete mode 100644 tests/adapters/test_usage_mapping.py create mode 100644 tests/adapters/usagemapping/__init__.py create mode 100644 tests/adapters/usagemapping/test_agnousagemapping.py create mode 100644 tests/adapters/usagemapping/test_claudesdkusagemapping.py create mode 100644 tests/adapters/usagemapping/test_codexusagemapping.py create mode 100644 tests/adapters/usagemapping/test_geminiusagemapping.py create mode 100644 tests/adapters/usagemapping/test_googleadkusagemapping.py create mode 100644 tests/adapters/usagemapping/test_isusageevent.py create mode 100644 tests/adapters/usagemapping/test_opencodeusagemapping.py create mode 100644 tests/adapters/usagemapping/test_turnusageconstructors.py diff --git a/src/band/adapters/letta.py b/src/band/adapters/letta.py index 7e4342267..33628f57f 100644 --- a/src/band/adapters/letta.py +++ b/src/band/adapters/letta.py @@ -1209,22 +1209,6 @@ async def _consolidate_memory(self, agent_id: str, room_id: str) -> None: # Helpers # ------------------------------------------------------------------ - @staticmethod - def _usage_from_response(response: Any) -> TurnUsage: - """Map a Letta ``LettaResponse.usage`` (a ``Usage``) onto TurnUsage. - - Field names verified against letta-client: ``prompt_tokens`` / - ``completion_tokens`` / ``cached_input_tokens`` / ``cache_write_tokens``. - A missing ``usage`` yields empty usage. - """ - return TurnUsage.from_object( - getattr(response, "usage", None), - input="prompt_tokens", - output="completion_tokens", - cache_read="cached_input_tokens", - cache_write="cache_write_tokens", - ) - @staticmethod def _format_time_ago(dt: datetime) -> str: """Format a datetime as a human-readable time-ago string.""" diff --git a/src/band/runtime/working_state.py b/src/band/runtime/working_state.py index 23661f17c..e05aff083 100644 --- a/src/band/runtime/working_state.py +++ b/src/band/runtime/working_state.py @@ -138,8 +138,7 @@ async def start(self, *, owner: object | None = None) -> None: self._active = True cycle = owner if owner is not None else object() self._cycle = cycle - self._ordinary = AgentActivityDetail(kind="reasoning") - self._manual_wait = False + self._reset_detail() self._started_at = asyncio.get_running_loop().time() await self._report_locked(True) if self._cycle is cycle and self._active: @@ -177,8 +176,7 @@ async def update( async def clear_detail(self, *, current: Callable[[], bool]) -> None: async with self._lock: if current(): - self._ordinary = AgentActivityDetail(kind="reasoning") - self._manual_wait = False + self._reset_detail() def defer_false_report(self) -> None: """Start ``working: false`` and return without waiting for it. @@ -221,11 +219,14 @@ async def stop(self) -> None: task.exception() await self._finish_false_send() + def _reset_detail(self) -> None: + self._ordinary = AgentActivityDetail(kind="reasoning") + self._manual_wait = False + def _arm_false_report(self) -> asyncio.Task[None] | None: self._active = False self._cycle = None - self._ordinary = AgentActivityDetail(kind="reasoning") - self._manual_wait = False + self._reset_detail() task, self._task = self._task, None if task is not None: task.cancel() diff --git a/tests/adapters/activity/kit.py b/tests/adapters/activity/kit.py index d686447fc..d9aa6bea0 100644 --- a/tests/adapters/activity/kit.py +++ b/tests/adapters/activity/kit.py @@ -43,7 +43,7 @@ async def __aiter__(self) -> AsyncIterator[bytes]: await self.release.wait() result = make_tool_return_message("shell", "private-result") yield f"data: {result.model_dump_json()}\n\n".encode() - yield b'data: {"message_type":"usage_statistics","prompt_tokens":17,"completion_tokens":5}\n\n' + yield b'data: {"message_type":"usage_statistics","prompt_tokens":17,"completion_tokens":5,"cached_input_tokens":3,"cache_write_tokens":2}\n\n' yield b"data: [DONE]\n\n" async def aclose(self) -> None: diff --git a/tests/adapters/activity/test_letta_stream.py b/tests/adapters/activity/test_letta_stream.py index e5891ac0d..b0893b2c1 100644 --- a/tests/adapters/activity/test_letta_stream.py +++ b/tests/adapters/activity/test_letta_stream.py @@ -63,7 +63,12 @@ async def respond(request: httpx.Request) -> httpx.Response: "tool_call_message", "tool_return_message", ] - assert (usage.input_tokens, usage.output_tokens) == (17, 5) + assert ( + usage.input_tokens, + usage.output_tokens, + usage.cache_read_tokens, + usage.cache_write_tokens, + ) == (17, 5, 3, 2) assert recorder.events[-1] == {"kind": "reasoning"} finally: stream.release.set() diff --git a/tests/adapters/test_usage_mapping.py b/tests/adapters/test_usage_mapping.py deleted file mode 100644 index 9d837c76c..000000000 --- a/tests/adapters/test_usage_mapping.py +++ /dev/null @@ -1,270 +0,0 @@ -"""Unit tests for the Emit.USAGE per-adapter usage-mapping helpers. - -Each adapter maps its framework's response/usage object onto the shared -``TurnUsage``. These are pure static methods, so they're tested directly with -lightweight stand-ins (SimpleNamespace / dict) — no live LLM, no full adapter -construction. The emission path itself (gating on Emit.USAGE, empty-skip, -best-effort) lives on ``SimpleAdapter.emit_usage`` and is covered in -``test_anthropic_adapter.py``. -""" - -from __future__ import annotations - -from types import SimpleNamespace - -import pytest -from band_rest.types.chat_message_metadata import ChatMessageMetadata - -from band.adapters.agno import AgnoAdapter -from band.adapters.claude_sdk import ClaudeSDKAdapter -from band.adapters.codex import CodexAdapter -from band.adapters.gemini import GeminiAdapter -from band.adapters.google_adk import GoogleADKAdapter -from band.adapters.letta import LettaAdapter -from band.core.types import USAGE_METADATA_KEY, TurnUsage, is_usage_event -from band.integrations.opencode import OpencodeMessageInfo - - -class TestTurnUsageConstructors: - """The shared from_object / from_mapping constructors the adapters build on.""" - - def test_from_object_reads_named_attrs(self): - src = SimpleNamespace(inp=100, out=20, cr=5, cw=3) - assert TurnUsage.from_object( - src, input="inp", output="out", cache_read="cr", cache_write="cw" - ) == TurnUsage(100, 20, 5, 3) - - def test_from_object_none_source_is_empty(self): - assert TurnUsage.from_object(None, input="i", output="o") == TurnUsage() - - def test_from_object_omitted_cache_keys_stay_zero(self): - src = SimpleNamespace(i=100, o=20, cache_read_tokens=999) - # cache_read/cache_write not passed → not read even if present on src - assert TurnUsage.from_object(src, input="i", output="o") == TurnUsage(100, 20) - - def test_from_mapping_reads_named_keys(self): - data = {"i": 100, "o": 20, "cr": 5, "cw": 3} - assert TurnUsage.from_mapping( - data, input="i", output="o", cache_read="cr", cache_write="cw" - ) == TurnUsage(100, 20, 5, 3) - - @pytest.mark.parametrize("bad", [None, "not a mapping", 42, ["list"]]) - def test_from_mapping_non_mapping_is_empty(self, bad): - assert TurnUsage.from_mapping(bad, input="i", output="o") == TurnUsage() - - def test_missing_and_non_int_fields_default_to_zero(self): - data = {"i": "oops", "o": None} # non-int / missing - assert TurnUsage.from_mapping(data, input="i", output="o") == TurnUsage() - - def test_fields_are_raw_no_cache_folding(self): - """Per the convention, values pass through raw — cache is never folded - into input regardless of provider.""" - data = {"i": 100, "o": 20, "cr": 5, "cw": 3} - assert TurnUsage.from_mapping( - data, input="i", output="o", cache_read="cr", cache_write="cw" - ) == TurnUsage( - input_tokens=100, - output_tokens=20, - cache_read_tokens=5, - cache_write_tokens=3, - ) - - -class TestIsUsageEvent: - """The shared discriminator task-event consumers use to skip usage records.""" - - def test_true_when_band_usage_present(self): - assert is_usage_event({USAGE_METADATA_KEY: {"input_tokens": 1}}) is True - - def test_false_for_lifecycle_task_or_non_mapping(self): - assert is_usage_event({"codex_thread_id": "x"}) is False - assert is_usage_event(None) is False - assert is_usage_event("nope") is False - - def test_true_and_false_for_chat_message_metadata_model(self): - """band-client-rest 0.0.38 types ``ChatMessage.metadata`` as a model, - not a dict — the discriminator must dump it, not require ``Mapping``.""" - assert ( - is_usage_event(ChatMessageMetadata(band_usage={"input_tokens": 1})) is True - ) - assert is_usage_event(ChatMessageMetadata(claude_sdk_session_id="s")) is False - - -class TestClaudeSDKUsageMapping: - def test_maps_usage_dict(self): - """ResultMessage.usage (raw API dict) maps onto TurnUsage raw — Claude's - input_tokens excludes cache, reported separately in the cache fields.""" - result = SimpleNamespace( - usage={ - "input_tokens": 100, - "output_tokens": 20, - "cache_read_input_tokens": 5, - "cache_creation_input_tokens": 3, - } - ) - assert ClaudeSDKAdapter._usage_from_result(result) == TurnUsage( - input_tokens=100, - output_tokens=20, - cache_read_tokens=5, - cache_write_tokens=3, - ) - - def test_missing_usage_is_empty(self): - """A ResultMessage without a usage dict yields empty usage.""" - assert ( - ClaudeSDKAdapter._usage_from_result(SimpleNamespace(usage=None)) - == TurnUsage() - ) - - -class TestAgnoUsageMapping: - def test_maps_run_metrics(self): - """RunOutput.metrics (aggregated) maps onto TurnUsage; names line up.""" - metrics = SimpleNamespace( - input_tokens=100, - output_tokens=20, - cache_read_tokens=5, - cache_write_tokens=3, - ) - response = SimpleNamespace(metrics=metrics) - assert AgnoAdapter._usage_from_response(response) == TurnUsage( - input_tokens=100, - output_tokens=20, - cache_read_tokens=5, - cache_write_tokens=3, - ) - - def test_missing_metrics_is_empty(self): - """A RunOutput without metrics yields empty usage.""" - assert ( - AgnoAdapter._usage_from_response(SimpleNamespace(metrics=None)) - == TurnUsage() - ) - - -class TestGoogleADKUsageMapping: - def test_maps_event_usage_metadata(self): - """ADK event.usage_metadata maps onto TurnUsage (no cache-write dim).""" - event = SimpleNamespace( - usage_metadata=SimpleNamespace( - prompt_token_count=100, - candidates_token_count=20, - cached_content_token_count=5, - ) - ) - assert GoogleADKAdapter._usage_from_event(event) == TurnUsage( - input_tokens=100, - output_tokens=20, - cache_read_tokens=5, - cache_write_tokens=0, - ) - - def test_event_without_usage_is_empty(self): - """A non-model event (no usage_metadata) contributes empty usage.""" - assert ( - GoogleADKAdapter._usage_from_event(SimpleNamespace(usage_metadata=None)) - == TurnUsage() - ) - - -class TestGeminiUsageMapping: - def test_maps_response_usage_metadata(self): - """GenerateContentResponse.usage_metadata maps onto TurnUsage.""" - response = SimpleNamespace( - usage_metadata=SimpleNamespace( - prompt_token_count=100, - candidates_token_count=20, - cached_content_token_count=5, - ) - ) - assert GeminiAdapter._usage_from_response(response) == TurnUsage( - input_tokens=100, - output_tokens=20, - cache_read_tokens=5, - cache_write_tokens=0, - ) - - def test_response_without_usage_is_empty(self): - """A response without usage_metadata yields empty usage.""" - assert ( - GeminiAdapter._usage_from_response(SimpleNamespace(usage_metadata=None)) - == TurnUsage() - ) - - -class TestLettaUsageMapping: - def test_maps_response_usage(self): - """LettaResponse.usage (per_room mode) maps onto TurnUsage.""" - response = SimpleNamespace( - usage=SimpleNamespace( - prompt_tokens=100, - completion_tokens=20, - cached_input_tokens=5, - cache_write_tokens=3, - ) - ) - assert LettaAdapter._usage_from_response(response) == TurnUsage( - input_tokens=100, - output_tokens=20, - cache_read_tokens=5, - cache_write_tokens=3, - ) - - def test_response_without_usage_is_empty(self): - """A response with no usage (e.g. shared-mode stream) yields empty usage.""" - assert ( - LettaAdapter._usage_from_response(SimpleNamespace(usage=None)) - == TurnUsage() - ) - - -class TestOpencodeUsageMapping: - def test_maps_info_tokens_with_nested_cache(self): - """OpenCode assistant info.tokens (nested cache) maps onto TurnUsage. - - OpenCode reports reasoning disjointly from output (its own total is - input + output + reasoning + cache), so reasoning folds into - output_tokens: 20 + 7 = 27. - """ - info = OpencodeMessageInfo.model_validate( - { - "tokens": { - "input": 100, - "output": 20, - "reasoning": 7, - "cache": {"read": 5, "write": 3}, - } - } - ) - assert info.tokens is not None - assert info.tokens.to_turn_usage() == TurnUsage( - input_tokens=100, - output_tokens=27, - cache_read_tokens=5, - cache_write_tokens=3, - ) - - def test_missing_or_malformed_tokens_is_empty(self): - """No tokens (or a non-dict tokens) parses as absent usage.""" - assert OpencodeMessageInfo.model_validate({}).tokens is None - assert OpencodeMessageInfo.model_validate({"tokens": "nope"}).tokens is None - - -class TestCodexUsageMapping: - def test_maps_per_turn_deltas(self): - """Codex's per-turn token deltas (not thread cumulatives) map to TurnUsage. - - Codex reports reasoning disjointly from output (its own total is - input + output + reasoning), so reasoning folds into output_tokens: - 42 + 8 = 50. - """ - usage = SimpleNamespace( - turn_input_tokens=130, turn_output_tokens=42, turn_reasoning_tokens=8 - ) - assert CodexAdapter._turn_usage(usage) == TurnUsage( - input_tokens=130, output_tokens=50 - ) - - def test_none_usage_is_empty(self): - """No usage object for the thread yields empty usage (no emit).""" - assert CodexAdapter._turn_usage(None) == TurnUsage() diff --git a/tests/adapters/usagemapping/__init__.py b/tests/adapters/usagemapping/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/adapters/usagemapping/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/usagemapping/test_agnousagemapping.py b/tests/adapters/usagemapping/test_agnousagemapping.py new file mode 100644 index 000000000..cdeefe9df --- /dev/null +++ b/tests/adapters/usagemapping/test_agnousagemapping.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from band.adapters.agno import AgnoAdapter +from band.core.types import TurnUsage + + +class TestAgnoUsageMapping: + def test_maps_run_metrics(self): + """RunOutput.metrics (aggregated) maps onto TurnUsage; names line up.""" + metrics = SimpleNamespace( + input_tokens=100, + output_tokens=20, + cache_read_tokens=5, + cache_write_tokens=3, + ) + response = SimpleNamespace(metrics=metrics) + assert AgnoAdapter._usage_from_response(response) == TurnUsage( + input_tokens=100, + output_tokens=20, + cache_read_tokens=5, + cache_write_tokens=3, + ) + + def test_missing_metrics_is_empty(self): + """A RunOutput without metrics yields empty usage.""" + assert ( + AgnoAdapter._usage_from_response(SimpleNamespace(metrics=None)) + == TurnUsage() + ) diff --git a/tests/adapters/usagemapping/test_claudesdkusagemapping.py b/tests/adapters/usagemapping/test_claudesdkusagemapping.py new file mode 100644 index 000000000..48ffc4466 --- /dev/null +++ b/tests/adapters/usagemapping/test_claudesdkusagemapping.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from band.adapters.claude_sdk import ClaudeSDKAdapter +from band.core.types import TurnUsage + + +class TestClaudeSDKUsageMapping: + def test_maps_usage_dict(self): + """ResultMessage.usage (raw API dict) maps onto TurnUsage raw — Claude's + input_tokens excludes cache, reported separately in the cache fields.""" + result = SimpleNamespace( + usage={ + "input_tokens": 100, + "output_tokens": 20, + "cache_read_input_tokens": 5, + "cache_creation_input_tokens": 3, + } + ) + assert ClaudeSDKAdapter._usage_from_result(result) == TurnUsage( + input_tokens=100, + output_tokens=20, + cache_read_tokens=5, + cache_write_tokens=3, + ) + + def test_missing_usage_is_empty(self): + """A ResultMessage without a usage dict yields empty usage.""" + assert ( + ClaudeSDKAdapter._usage_from_result(SimpleNamespace(usage=None)) + == TurnUsage() + ) diff --git a/tests/adapters/usagemapping/test_codexusagemapping.py b/tests/adapters/usagemapping/test_codexusagemapping.py new file mode 100644 index 000000000..1d0d163e7 --- /dev/null +++ b/tests/adapters/usagemapping/test_codexusagemapping.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from band.adapters.codex import CodexAdapter +from band.core.types import TurnUsage + + +class TestCodexUsageMapping: + def test_maps_per_turn_deltas(self): + """Codex's per-turn token deltas (not thread cumulatives) map to TurnUsage. + + Codex reports reasoning disjointly from output (its own total is + input + output + reasoning), so reasoning folds into output_tokens: + 42 + 8 = 50. + """ + usage = SimpleNamespace( + turn_input_tokens=130, turn_output_tokens=42, turn_reasoning_tokens=8 + ) + assert CodexAdapter._turn_usage(usage) == TurnUsage( + input_tokens=130, output_tokens=50 + ) + + def test_none_usage_is_empty(self): + """No usage object for the thread yields empty usage (no emit).""" + assert CodexAdapter._turn_usage(None) == TurnUsage() diff --git a/tests/adapters/usagemapping/test_geminiusagemapping.py b/tests/adapters/usagemapping/test_geminiusagemapping.py new file mode 100644 index 000000000..ad6daf1ed --- /dev/null +++ b/tests/adapters/usagemapping/test_geminiusagemapping.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from band.adapters.gemini import GeminiAdapter +from band.core.types import TurnUsage + + +class TestGeminiUsageMapping: + def test_maps_response_usage_metadata(self): + """GenerateContentResponse.usage_metadata maps onto TurnUsage.""" + response = SimpleNamespace( + usage_metadata=SimpleNamespace( + prompt_token_count=100, + candidates_token_count=20, + cached_content_token_count=5, + ) + ) + assert GeminiAdapter._usage_from_response(response) == TurnUsage( + input_tokens=100, + output_tokens=20, + cache_read_tokens=5, + cache_write_tokens=0, + ) + + def test_response_without_usage_is_empty(self): + """A response without usage_metadata yields empty usage.""" + assert ( + GeminiAdapter._usage_from_response(SimpleNamespace(usage_metadata=None)) + == TurnUsage() + ) diff --git a/tests/adapters/usagemapping/test_googleadkusagemapping.py b/tests/adapters/usagemapping/test_googleadkusagemapping.py new file mode 100644 index 000000000..76e8367a2 --- /dev/null +++ b/tests/adapters/usagemapping/test_googleadkusagemapping.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from types import SimpleNamespace + +from band.adapters.google_adk import GoogleADKAdapter +from band.core.types import TurnUsage + + +class TestGoogleADKUsageMapping: + def test_maps_event_usage_metadata(self): + """ADK event.usage_metadata maps onto TurnUsage (no cache-write dim).""" + event = SimpleNamespace( + usage_metadata=SimpleNamespace( + prompt_token_count=100, + candidates_token_count=20, + cached_content_token_count=5, + ) + ) + assert GoogleADKAdapter._usage_from_event(event) == TurnUsage( + input_tokens=100, + output_tokens=20, + cache_read_tokens=5, + cache_write_tokens=0, + ) + + def test_event_without_usage_is_empty(self): + """A non-model event (no usage_metadata) contributes empty usage.""" + assert ( + GoogleADKAdapter._usage_from_event(SimpleNamespace(usage_metadata=None)) + == TurnUsage() + ) diff --git a/tests/adapters/usagemapping/test_isusageevent.py b/tests/adapters/usagemapping/test_isusageevent.py new file mode 100644 index 000000000..4f67775bd --- /dev/null +++ b/tests/adapters/usagemapping/test_isusageevent.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from band_rest.types.chat_message_metadata import ChatMessageMetadata + +from band.core.types import USAGE_METADATA_KEY, is_usage_event + + +class TestIsUsageEvent: + """The shared discriminator task-event consumers use to skip usage records.""" + + def test_true_when_band_usage_present(self): + assert is_usage_event({USAGE_METADATA_KEY: {"input_tokens": 1}}) is True + + def test_false_for_lifecycle_task_or_non_mapping(self): + assert is_usage_event({"codex_thread_id": "x"}) is False + assert is_usage_event(None) is False + assert is_usage_event("nope") is False + + def test_true_and_false_for_chat_message_metadata_model(self): + """band-client-rest 0.0.38 types ``ChatMessage.metadata`` as a model, + not a dict — the discriminator must dump it, not require ``Mapping``.""" + assert ( + is_usage_event(ChatMessageMetadata(band_usage={"input_tokens": 1})) is True + ) + assert is_usage_event(ChatMessageMetadata(claude_sdk_session_id="s")) is False diff --git a/tests/adapters/usagemapping/test_opencodeusagemapping.py b/tests/adapters/usagemapping/test_opencodeusagemapping.py new file mode 100644 index 000000000..99b8849b3 --- /dev/null +++ b/tests/adapters/usagemapping/test_opencodeusagemapping.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from band.core.types import TurnUsage +from band.integrations.opencode import OpencodeMessageInfo + + +class TestOpencodeUsageMapping: + def test_maps_info_tokens_with_nested_cache(self): + """OpenCode assistant info.tokens (nested cache) maps onto TurnUsage. + + OpenCode reports reasoning disjointly from output (its own total is + input + output + reasoning + cache), so reasoning folds into + output_tokens: 20 + 7 = 27. + """ + info = OpencodeMessageInfo.model_validate( + { + "tokens": { + "input": 100, + "output": 20, + "reasoning": 7, + "cache": {"read": 5, "write": 3}, + } + } + ) + assert info.tokens is not None + assert info.tokens.to_turn_usage() == TurnUsage( + input_tokens=100, + output_tokens=27, + cache_read_tokens=5, + cache_write_tokens=3, + ) + + def test_missing_or_malformed_tokens_is_empty(self): + """No tokens (or a non-dict tokens) parses as absent usage.""" + assert OpencodeMessageInfo.model_validate({}).tokens is None + assert OpencodeMessageInfo.model_validate({"tokens": "nope"}).tokens is None diff --git a/tests/adapters/usagemapping/test_turnusageconstructors.py b/tests/adapters/usagemapping/test_turnusageconstructors.py new file mode 100644 index 000000000..c47b5a22a --- /dev/null +++ b/tests/adapters/usagemapping/test_turnusageconstructors.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from band.core.types import TurnUsage + + +class TestTurnUsageConstructors: + """The shared from_object / from_mapping constructors the adapters build on.""" + + def test_from_object_reads_named_attrs(self): + src = SimpleNamespace(inp=100, out=20, cr=5, cw=3) + assert TurnUsage.from_object( + src, input="inp", output="out", cache_read="cr", cache_write="cw" + ) == TurnUsage(100, 20, 5, 3) + + def test_from_object_none_source_is_empty(self): + assert TurnUsage.from_object(None, input="i", output="o") == TurnUsage() + + def test_from_object_omitted_cache_keys_stay_zero(self): + src = SimpleNamespace(i=100, o=20, cache_read_tokens=999) + # cache_read/cache_write not passed → not read even if present on src + assert TurnUsage.from_object(src, input="i", output="o") == TurnUsage(100, 20) + + def test_from_mapping_reads_named_keys(self): + data = {"i": 100, "o": 20, "cr": 5, "cw": 3} + assert TurnUsage.from_mapping( + data, input="i", output="o", cache_read="cr", cache_write="cw" + ) == TurnUsage(100, 20, 5, 3) + + @pytest.mark.parametrize("bad", [None, "not a mapping", 42, ["list"]]) + def test_from_mapping_non_mapping_is_empty(self, bad): + assert TurnUsage.from_mapping(bad, input="i", output="o") == TurnUsage() + + def test_missing_and_non_int_fields_default_to_zero(self): + data = {"i": "oops", "o": None} # non-int / missing + assert TurnUsage.from_mapping(data, input="i", output="o") == TurnUsage() + + def test_fields_are_raw_no_cache_folding(self): + """Per the convention, values pass through raw — cache is never folded + into input regardless of provider.""" + data = {"i": 100, "o": 20, "cr": 5, "cw": 3} + assert TurnUsage.from_mapping( + data, input="i", output="o", cache_read="cr", cache_write="cw" + ) == TurnUsage( + input_tokens=100, + output_tokens=20, + cache_read_tokens=5, + cache_write_tokens=3, + ) From 9d7fc6418579341f269c5c0f6e0e725a5b51bf0c Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 18 Oct 2026 11:11:34 +0300 Subject: [PATCH 13/47] test: Register live activity cells and disambiguate runtime fixtures --- tests/e2e/baseline/expected-cells.json | 18 +++++++++++++ .../activityownership/test_band_tools.py | 4 ++- .../activityownership/test_claude_native.py | 18 ++++++++----- .../test_detached_detail_feature.py | 26 +++++++++++-------- 4 files changed, 48 insertions(+), 18 deletions(-) diff --git a/tests/e2e/baseline/expected-cells.json b/tests/e2e/baseline/expected-cells.json index 217501e6d..945033157 100644 --- a/tests/e2e/baseline/expected-cells.json +++ b/tests/e2e/baseline/expected-cells.json @@ -697,6 +697,24 @@ "status": "pass", "reason": null }, + { + "test": "tests/e2e/baseline/smoke/activity/test_native_reporting.py::TestNativeActivity::test_native_shell_reports_privately_without_narration", + "adapter": "claude_sdk", + "lanes": [ + "core" + ], + "status": "pass", + "reason": null + }, + { + "test": "tests/e2e/baseline/smoke/activity/test_sdk_reporting.py::TestSDKActivity::test_held_sdk_tool_reports_privately_without_narration", + "adapter": "anthropic", + "lanes": [ + "core" + ], + "status": "pass", + "reason": null + }, { "test": "tests/e2e/baseline/smoke/adapters/copilotsdk/test_handler.py::TestAskUserHandler::test_copilot_ask_user_handler_round_trips_to_room_reply", "adapter": "suite", diff --git a/tests/runtime/activityownership/test_band_tools.py b/tests/runtime/activityownership/test_band_tools.py index 439532e0d..7dc23486d 100644 --- a/tests/runtime/activityownership/test_band_tools.py +++ b/tests/runtime/activityownership/test_band_tools.py @@ -13,7 +13,9 @@ class TestBandToolActivity: async def test_direct_registered_method_and_model_event_are_observed_but_telemetry_is_quiet( - self, execution: Callable[..., ExecutionContext], activity_platform: ActivityPlatform + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, ) -> None: async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: tools = AgentTools.from_context(ctx) diff --git a/tests/runtime/activityownership/test_claude_native.py b/tests/runtime/activityownership/test_claude_native.py index b37a1cbaa..016c216a0 100644 --- a/tests/runtime/activityownership/test_claude_native.py +++ b/tests/runtime/activityownership/test_claude_native.py @@ -22,7 +22,9 @@ class TestClaudeNativeWireActivity: @pytest.mark.httpx_mock(should_mock=lambda request: request.url.host != "127.0.0.1") async def test_real_context_observes_native_tool_without_tool_narration( - self, execution: Callable[..., ExecutionContext], peer: ActivityPlatform + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, ) -> None: with scripted_claude() as cli: cli.script( @@ -57,13 +59,17 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: assert { "working": True, "activity": {"kind": "tool_call", "name": "Bash"}, - } in peer.activity_bodies - assert peer.activity_bodies[-1] == {"working": False} + } in activity_platform.activity_bodies + assert activity_platform.activity_bodies[-1] == {"working": False} assert not any( body.get("message_type") in {"tool_call", "tool_result"} - for body in peer.event_bodies + for body in activity_platform.event_bodies + ) + assert "private-argument" not in json.dumps( + activity_platform.activity_bodies + ) + assert "private-result" not in json.dumps( + activity_platform.activity_bodies ) - assert "private-argument" not in json.dumps(peer.activity_bodies) - assert "private-result" not in json.dumps(peer.activity_bodies) finally: await adapter.cleanup_all() diff --git a/tests/runtime/activityownership/test_detached_detail_feature.py b/tests/runtime/activityownership/test_detached_detail_feature.py index 33a02435f..ee6146aed 100644 --- a/tests/runtime/activityownership/test_detached_detail_feature.py +++ b/tests/runtime/activityownership/test_detached_detail_feature.py @@ -36,11 +36,11 @@ class TestDetachedActivityFeature: async def test_parked_approval_owns_detail_until_native_completion( self, execution: Callable[..., ExecutionContext], - peer: ActivityPlatform, + activity_platform: ActivityPlatform, httpx_mock: HTTPXMock, variant: str, ) -> None: - peer.reject_detail = variant == "downgrade" + activity_platform.reject_detail = variant == "downgrade" resumed, release = asyncio.Event(), asyncio.Event() captured: list[AgentTools] = [] @@ -107,7 +107,7 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: workers = tuple(owner.workers) await ctx._process_event(make_message_event(msg_id="busy")) await asyncio.sleep(9.1) - parked = list(peer.activity_bodies) + parked = list(activity_platform.activity_bodies) match variant: case "disabled" | "hub": assert not parked @@ -128,7 +128,7 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: == [{"working": True, "activity": {"kind": "permission"}}] * 3 ) - assert peer.marked("processed") == ["original"] + assert activity_platform.marked("processed") == ["original"] assert await adapter.try_handle_control( make_platform_message( room_id=ROOM_ID, content="approve ask", sender_id="user-456" @@ -137,25 +137,29 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: ) await resumed.wait() if variant in {"normal", "silent"}: - assert peer.activity_bodies[-1] == { + assert activity_platform.activity_bodies[-1] == { "working": True, "activity": {"kind": "tool_call", "name": "bash"}, } if variant == "cap": - assert peer.activity_bodies == parked + assert activity_platform.activity_bodies == parked release.set() await asyncio.gather(*(worker.task for worker in workers)) await ctx._join_activity_completion() assert not client.aborted_sessions - assert "private-argument" not in json.dumps(peer.activity_bodies) - assert "private-result" not in json.dumps(peer.activity_bodies) - before = list(peer.activity_bodies) + assert "private-argument" not in json.dumps( + activity_platform.activity_bodies + ) + assert "private-result" not in json.dumps( + activity_platform.activity_bodies + ) + before = list(activity_platform.activity_bodies) await report_tool_activity(captured[0], "late-old-callback") - assert peer.activity_bodies == before + assert activity_platform.activity_bodies == before if variant == "silent": assert not any( event.get("message_type") in {"tool_call", "tool_result"} - for event in peer.event_bodies + for event in activity_platform.event_bodies ) finally: release.set() From eb7326eadcd07e3c52300af6deeb22d0b5f2d924 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 18 Oct 2026 11:41:21 +0300 Subject: [PATCH 14/47] fix: Fence housekeeping activity and clean up cancelled tool starts --- src/band/runtime/cycle.py | 1 + src/band/runtime/execution.py | 17 +- src/band/runtime/turn.py | 8 +- tests/agent/test_startup_race_condition.py | 34 +- tests/runtime/activityownership/kit.py | 7 +- .../activityownership/test_cancelled_tool.py | 97 + .../test_detached_detail_feature.py | 8 + .../activityownership/test_housekeeping.py | 44 + tests/runtime/test_tools.py | 2624 ----------------- tests/runtime/toolbehavior/__init__.py | 1 + tests/runtime/toolbehavior/conftest.py | 12 + tests/runtime/toolbehavior/kit.py | 97 + .../test_agenttoolsaddparticipant.py | 105 + .../test_agenttoolsconstruction.py | 33 + .../test_agenttoolscontextsyncback.py | 97 + .../test_agenttoolscreatechatroom.py | 38 + .../test_agenttoolsexecutetoolcall.py | 81 + .../test_agenttoolsfromcontext.py | 26 + .../test_agenttoolsgetparticipants.py | 127 + .../test_agenttoolslookuppeers.py | 29 + .../test_agenttoolsremoveparticipant.py | 45 + .../toolbehavior/test_agenttoolsschemas.py | 124 + .../toolbehavior/test_agenttoolssendevent.py | 123 + .../test_agenttoolssendfailure.py | 52 + .../test_agenttoolssendmessage.py | 106 + .../test_canonicalizemcptoolname.py | 35 + .../test_emptymentionsvalidation.py | 93 + .../toolbehavior/test_fetchroomcontext.py | 33 + tests/runtime/toolbehavior/test_filetools.py | 755 +++++ .../test_formattoolvalidationerror.py | 37 + .../test_handlementionresolution.py | 104 + .../toolbehavior/test_ismcpcontentresult.py | 39 + .../toolbehavior/test_matchesidentifier.py | 56 + .../runtime/toolbehavior/test_memorytools.py | 281 ++ .../test_memoryvalidationpreventshttp.py | 28 + .../toolbehavior/test_mentionresolution.py | 44 + .../toolbehavior/test_replyeffectspellings.py | 35 + .../toolbehavior/test_toolinputmodels.py | 91 + ...toppedrefusalescapestoolerrorconversion.py | 38 + .../test_validmemorydispatchaddressesitem.py | 34 + 40 files changed, 2987 insertions(+), 2652 deletions(-) create mode 100644 tests/runtime/activityownership/test_cancelled_tool.py create mode 100644 tests/runtime/activityownership/test_housekeeping.py delete mode 100644 tests/runtime/test_tools.py create mode 100644 tests/runtime/toolbehavior/__init__.py create mode 100644 tests/runtime/toolbehavior/conftest.py create mode 100644 tests/runtime/toolbehavior/kit.py create mode 100644 tests/runtime/toolbehavior/test_agenttoolsaddparticipant.py create mode 100644 tests/runtime/toolbehavior/test_agenttoolsconstruction.py create mode 100644 tests/runtime/toolbehavior/test_agenttoolscontextsyncback.py create mode 100644 tests/runtime/toolbehavior/test_agenttoolscreatechatroom.py create mode 100644 tests/runtime/toolbehavior/test_agenttoolsexecutetoolcall.py create mode 100644 tests/runtime/toolbehavior/test_agenttoolsfromcontext.py create mode 100644 tests/runtime/toolbehavior/test_agenttoolsgetparticipants.py create mode 100644 tests/runtime/toolbehavior/test_agenttoolslookuppeers.py create mode 100644 tests/runtime/toolbehavior/test_agenttoolsremoveparticipant.py create mode 100644 tests/runtime/toolbehavior/test_agenttoolsschemas.py create mode 100644 tests/runtime/toolbehavior/test_agenttoolssendevent.py create mode 100644 tests/runtime/toolbehavior/test_agenttoolssendfailure.py create mode 100644 tests/runtime/toolbehavior/test_agenttoolssendmessage.py create mode 100644 tests/runtime/toolbehavior/test_canonicalizemcptoolname.py create mode 100644 tests/runtime/toolbehavior/test_emptymentionsvalidation.py create mode 100644 tests/runtime/toolbehavior/test_fetchroomcontext.py create mode 100644 tests/runtime/toolbehavior/test_filetools.py create mode 100644 tests/runtime/toolbehavior/test_formattoolvalidationerror.py create mode 100644 tests/runtime/toolbehavior/test_handlementionresolution.py create mode 100644 tests/runtime/toolbehavior/test_ismcpcontentresult.py create mode 100644 tests/runtime/toolbehavior/test_matchesidentifier.py create mode 100644 tests/runtime/toolbehavior/test_memorytools.py create mode 100644 tests/runtime/toolbehavior/test_memoryvalidationpreventshttp.py create mode 100644 tests/runtime/toolbehavior/test_mentionresolution.py create mode 100644 tests/runtime/toolbehavior/test_replyeffectspellings.py create mode 100644 tests/runtime/toolbehavior/test_toolinputmodels.py create mode 100644 tests/runtime/toolbehavior/test_unboundstoppedrefusalescapestoolerrorconversion.py create mode 100644 tests/runtime/toolbehavior/test_validmemorydispatchaddressesitem.py diff --git a/src/band/runtime/cycle.py b/src/band/runtime/cycle.py index 6cc42de9f..ef48c9672 100644 --- a/src/band/runtime/cycle.py +++ b/src/band/runtime/cycle.py @@ -18,6 +18,7 @@ class TurnScope: observer_task: asyncio.Task[None] | None = None handler: TurnTask | None = None admitted: bool = False + activity_allowed: bool = False settled: bool = False kind: ControlMode | None = None deadline: float | None = None diff --git a/src/band/runtime/execution.py b/src/band/runtime/execution.py index 89df6db79..3bb4b435c 100644 --- a/src/band/runtime/execution.py +++ b/src/band/runtime/execution.py @@ -1901,6 +1901,7 @@ async def _execute_message_cycle( ) -> None: scope = self.current_scope if scope is not None: + scope.activity_allowed = True await self._acquire_activity(scope) if scope is not None and scope.terminating: raise asyncio.CancelledError @@ -1921,6 +1922,8 @@ async def _acquire_activity(self, scope: TurnScope) -> None: await self._start_activity(scope) async def _start_activity(self, scope: TurnScope) -> None: + if not scope.activity_allowed: + return previous = self._activity_owner if previous is not None and previous is not scope: if not previous.aborted and any( @@ -1937,7 +1940,11 @@ async def _start_activity(self, scope: TurnScope) -> None: await self._working_reporter.start(owner=scope) def activity_observer(self, scope: TurnScope | None) -> ActivityObserver | None: - return ScopeActivityObserver(self, scope) if scope is not None else None + return ( + ScopeActivityObserver(self, scope) + if scope is not None and scope.activity_allowed + else None + ) def owns_activity(self, scope: TurnScope) -> bool: return self._activity_owner is scope and not scope.aborted and not self._closing @@ -1949,6 +1956,8 @@ async def observe_activity( activity: AgentActivityDetail | None = None, waiting: bool | None = None, ) -> None: + if not scope.activity_allowed: + return if ( self._activity_owner is None and scope is self.current_scope @@ -1972,7 +1981,11 @@ def register_turn_task(self, scope: TurnScope, owner: TurnTask) -> None: """Bind completion to the same owner that delivers worker cancellation.""" scope.bind(owner) owner.task.add_done_callback(lambda _: self._finish_activity(scope)) - if scope is self.current_scope and self._activity_owner is None: + if ( + scope.activity_allowed + and scope is self.current_scope + and self._activity_owner is None + ): previous = self._activity_completion async def acquire() -> None: diff --git a/src/band/runtime/turn.py b/src/band/runtime/turn.py index 5620ca6be..8898eff86 100644 --- a/src/band/runtime/turn.py +++ b/src/band/runtime/turn.py @@ -65,11 +65,11 @@ def _observing( ) -> Callable[..., Awaitable[Any]]: @functools.wraps(method) async def observe(self: Any, *args: Any, **kwargs: Any) -> Any: - if tool_name in MEMORY_SAVE_TOOLS: - await report_memory_activity(self) - else: - await report_tool_activity(self, tool_name) try: + if tool_name in MEMORY_SAVE_TOOLS: + await report_memory_activity(self) + else: + await report_tool_activity(self, tool_name) return await method(self, *args, **kwargs) finally: await end_tool_activity(self) diff --git a/tests/agent/test_startup_race_condition.py b/tests/agent/test_startup_race_condition.py index 36e01ff91..a493ea408 100644 --- a/tests/agent/test_startup_race_condition.py +++ b/tests/agent/test_startup_race_condition.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import UTC, datetime -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock import pytest @@ -14,7 +14,8 @@ ) from band.core.simple_adapter import SimpleAdapter from band.platform.event import MessageEvent -from band.runtime.context import ConversationContext +from band.runtime.execution import ExecutionContext +from band.runtime.sessionconfig import SessionConfig from band.testing.platform import platform_connection_stub @@ -26,7 +27,9 @@ class TestStartupRaceCondition: """ @pytest.mark.asyncio - async def test_adapter_on_started_before_first_message(self): + async def test_adapter_on_started_before_first_message( + self, mock_rest_client: MagicMock + ) -> None: """System prompt must be set before any message processing.""" # Track the order of calls @@ -93,20 +96,13 @@ async def start( self._on_execute = on_execute # Create a proper MessageEvent that passes through preprocessor - mock_ctx = MagicMock() - mock_ctx.is_llm_initialized = False - mock_ctx.mark_llm_initialized = MagicMock() - mock_ctx.get_context = AsyncMock( - return_value=ConversationContext( - room_id="room-123", - messages=[], - participants=[], - hydrated_at=datetime.now(UTC), - ) + mock_ctx = ExecutionContext( + room_id="room-123", + link=MagicMock(rest=mock_rest_client), + on_execute=on_execute, + agent_id="agent-123", + config=SessionConfig(enable_context_hydration=False), ) - mock_ctx.participants = [] - mock_ctx.participants_changed = MagicMock(return_value=False) - mock_ctx.room_id = "room-123" # Create a proper MessageEvent (not a MagicMock) message_event = MessageEvent( @@ -139,8 +135,4 @@ async def run_forever(self, *, install_signal_handlers: bool = True): # This will fail if on_message is called before on_started await agent.start() - # Verify on_started was called first - assert len(call_order) >= 1, "No calls were made" - assert call_order[0] == "on_started", ( - f"Expected on_started to be called first, but got: {call_order}" - ) + assert call_order == ["on_started", "on_message"] diff --git a/tests/runtime/activityownership/kit.py b/tests/runtime/activityownership/kit.py index 702be2119..e97933906 100644 --- a/tests/runtime/activityownership/kit.py +++ b/tests/runtime/activityownership/kit.py @@ -17,6 +17,7 @@ def __init__(self) -> None: self.hold_next_activity = False self.activity_entered = asyncio.Event() self.activity_release = asyncio.Event() + self.activity_cancelled = asyncio.Event() self.reject_detail = False self.event_bodies: list[dict[str, Any]] = [] @@ -32,7 +33,11 @@ async def respond(self, request: httpx.Request) -> httpx.Response: if self.hold_next_activity and request.url.path.endswith("/activity"): self.hold_next_activity = False self.activity_entered.set() - await self.activity_release.wait() + try: + await self.activity_release.wait() + except asyncio.CancelledError: + self.activity_cancelled.set() + raise return response def answer(self, request: httpx.Request) -> httpx.Response: diff --git a/tests/runtime/activityownership/test_cancelled_tool.py b/tests/runtime/activityownership/test_cancelled_tool.py new file mode 100644 index 000000000..b45cb393e --- /dev/null +++ b/tests/runtime/activityownership/test_cancelled_tool.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Callable + +from mcp import types +from mcp.shared.memory import create_connected_server_and_client_session + +from band.core.turn_lifecycle import TurnTask, bind_turn_task +from band.integrations.mcp.engine import ( + EmbeddedResolver, + EngineSpec, + build_engine, + build_tool_registration, + extend_with_chat_id, +) +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from band.runtime.tools.registry import TOOL_DEFINITIONS +from band.runtime.tools.types import BandTool +from tests.conftest import make_message_event +from tests.runtime.activityownership.kit import ActivityPlatform + + +class TestCancelledToolActivity: + async def test_mcp_request_cancellation_during_initial_report_restores_reasoning_for_live_provider( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + ready, release = asyncio.Event(), asyncio.Event() + captured: list[AgentTools] = [] + workers: list[TurnTask] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + captured.append(tools) + workers.append(bind_turn_task(tools, asyncio.create_task(release.wait()))) + ready.set() + + ctx = execution(handler) + await ctx.start() + await ctx.on_event(make_message_event()) + await ready.wait() + await ctx._wait_for_idle(timeout=2) + request_ids: list[int | str] = [] + + def resolve(room_id: str) -> AgentTools: + request_ids.append(server.get_context().request_context.request_id) + return captured[0] + + definition = TOOL_DEFINITIONS[BandTool.GET_PARTICIPANTS] + registration = build_tool_registration( + definition, + extend_with_chat_id(definition.input_model, None), + resolver=EmbeddedResolver(get_tools=resolve), + strip_chat_id=True, + ) + server = build_engine( + EngineSpec(name="activity-cancellation", tools=(registration,)) + ) + try: + async with create_connected_server_and_client_session(server) as session: + activity_platform.hold_next_activity = True + request = asyncio.create_task( + session.call_tool( + BandTool.GET_PARTICIPANTS, {"chat_id": ctx.room_id} + ) + ) + try: + await activity_platform.activity_entered.wait() + await session.send_notification( + types.ClientNotification( + types.CancelledNotification( + params=types.CancelledNotificationParams( + requestId=request_ids[-1] + ) + ) + ) + ) + await activity_platform.activity_cancelled.wait() + finally: + request.cancel() + await asyncio.gather(request, return_exceptions=True) + activity_platform.activity_release.set() + mark = len(activity_platform.activity_bodies) + await asyncio.sleep(ctx.config.working_keep_alive_seconds + 0.1) + assert not workers[0].task.done() + assert len(activity_platform.activity_bodies) > mark + assert activity_platform.activity_bodies[-1] == { + "working": True, + "activity": {"kind": "reasoning"}, + } + finally: + release.set() + await workers[0].task diff --git a/tests/runtime/activityownership/test_detached_detail_feature.py b/tests/runtime/activityownership/test_detached_detail_feature.py index ee6146aed..455268469 100644 --- a/tests/runtime/activityownership/test_detached_detail_feature.py +++ b/tests/runtime/activityownership/test_detached_detail_feature.py @@ -146,6 +146,14 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: release.set() await asyncio.gather(*(worker.task for worker in workers)) await ctx._join_activity_completion() + if variant in {"disabled", "hub"}: + assert not activity_platform.activity_bodies + assert not httpx_mock.get_requests() + else: + assert activity_platform.activity_bodies[-1] == {"working": False} + assert ( + activity_platform.activity_bodies.count({"working": False}) == 1 + ) assert not client.aborted_sessions assert "private-argument" not in json.dumps( activity_platform.activity_bodies diff --git a/tests/runtime/activityownership/test_housekeeping.py b/tests/runtime/activityownership/test_housekeeping.py new file mode 100644 index 000000000..c0a36a247 --- /dev/null +++ b/tests/runtime/activityownership/test_housekeeping.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Callable + +import pytest +from pytest_httpx import HTTPXMock + +from band.core.turn_lifecycle import bind_turn_task +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from tests.conftest import make_participant_added_event +from tests.runtime.activityownership.kit import ActivityPlatform + + +class TestHousekeepingActivity: + @pytest.mark.parametrize("detached", [False, True]) + async def test_registered_tools_and_workers_cannot_start_housekeeping_reports( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + httpx_mock: HTTPXMock, + detached: bool, + ) -> None: + handled, release = asyncio.Event(), asyncio.Event() + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + await tools.get_participants() + if detached: + bind_turn_task(tools, asyncio.create_task(release.wait())) + handled.set() + + ctx = execution(handler) + await ctx.start() + try: + await ctx.on_event(make_participant_added_event()) + await handled.wait() + await ctx._wait_for_idle(timeout=2) + assert not activity_platform.activity_bodies + assert not httpx_mock.get_requests() + finally: + release.set() diff --git a/tests/runtime/test_tools.py b/tests/runtime/test_tools.py deleted file mode 100644 index 19bdca380..000000000 --- a/tests/runtime/test_tools.py +++ /dev/null @@ -1,2624 +0,0 @@ -"""Tests for AgentTools.""" - -from __future__ import annotations - -import asyncio -import base64 -from datetime import UTC, datetime, timedelta -from types import SimpleNamespace -from typing import Any, ClassVar -from unittest.mock import AsyncMock, MagicMock, Mock - -import pytest -from band_rest import ( - ChatMessage, - GetAgentChatContextResponse, - GetAgentChatContextResponseMetadata, -) -from band_rest.errors import ForbiddenError -from band_rest.types import Error, ErrorError -from band_sdk_core import AgentFailure -from pydantic import BaseModel, ValidationError - -from band.client.rest import ( - DEFAULT_REQUEST_OPTIONS, - Attachment, - ChatMessageRequest, - NotFoundError, - UnprocessableEntityError, -) -from band.config.settings import RuntimeSettings -from band.core.exceptions import BandToolError, RoomExecutionStoppedError -from band.core.memory_types import ORGANIZATION_SCOPE_REJECTED_CODE -from band.core.types import Capability -from band.runtime.execution import ExecutionContext -from band.runtime.tools import ( - DEFAULT_FILE_CAPTION, - FILE_UNAVAILABLE_MESSAGE, - MAX_INLINE_IMAGE_BYTES, - MAX_INLINE_TEXT_BYTES, - MAX_SEND_CONTENT_BYTES, - TOOL_MODELS, - AddParticipantInput, - AgentTools, - CreateChatroomInput, - GetParticipantsInput, - LookupPeersInput, - RemoveParticipantInput, - SendEventInput, - SendMessageInput, - SendRoomFileInput, - StoreMemoryInput, - TurnEffect, - append_mention_handles_hint, - available_mention_handles, - canonicalize_mcp_tool_name, - format_tool_validation_error, - is_mcp_content_result, - matches_identifier, - turn_effect, -) -from tests.conftest import make_participant_mock -from tests.content import BLANK_CONTENT_CASES -from tests.identifiers import INVALID_IDS, UUID_ID -from tests.runtime.helpers import memory_client - - -class TestIsMcpContentResult: - """``is_mcp_content_result`` recognizes ``read_room_file``'s image-branch - shape so a consumer that supports real MCP content (claude_sdk, the MCP - engine) can pass it through instead of json.dumps-ing it into text.""" - - _IMAGE_RESULT: ClassVar[dict[str, Any]] = { - "content": [{"type": "image", "data": "YmFzZTY0", "mimeType": "image/png"}] - } - _TEXT_RESULT: ClassVar[dict[str, Any]] = { - "name": "notes.txt", - "content_type": "text/plain", - "text": "hi", - } - - def test_true_for_image_content_block(self) -> None: - assert is_mcp_content_result(self._IMAGE_RESULT) - - def test_false_for_plain_dict(self) -> None: - assert not is_mcp_content_result(self._TEXT_RESULT) - - def test_false_for_non_dict(self) -> None: - assert not is_mcp_content_result("just a string") - - def test_false_for_content_list_without_type_keys(self) -> None: - assert not is_mcp_content_result({"content": [{"no_type": 1}]}) - - def test_false_for_empty_content_list(self) -> None: - """An empty list is vacuously all()-true; every consumer indexes - content[0] right after this check, so an empty list must not pass.""" - assert not is_mcp_content_result({"content": []}) - - -@pytest.fixture -def participants(): - """Sample participants list.""" - return [ - {"id": "user-1", "name": "User One", "type": "User", "handle": "@user-one"}, - {"id": "user-2", "name": "User Two", "type": "User", "handle": "@user-two"}, - ] - - -class TestMemoryTools: - @pytest.mark.asyncio - async def test_list_memories_omits_none_filters(self, mock_rest_client) -> None: - response = MagicMock() - response.data = [] - mock_rest_client.agent_api_memories.list_agent_memories = AsyncMock( - return_value=response - ) - tools = AgentTools("room-123", mock_rest_client) - - await tools.list_memories(page_size=25) - - mock_rest_client.agent_api_memories.list_agent_memories.assert_awaited_once_with( - page_size=25, - request_options=DEFAULT_REQUEST_OPTIONS, - ) - - @pytest.mark.asyncio - async def test_store_memory_omits_none_fields(self, mock_rest_client) -> None: - response = MagicMock() - response.data = MagicMock() - mock_rest_client.agent_api_memories.create_agent_memory = AsyncMock( - return_value=response - ) - tools = AgentTools("room-123", mock_rest_client) - - await tools.store_memory( - content="remember this", - system="working", - type="semantic", - segment="user", - thought="useful later", - scope="organization", - ) - - call_kwargs = ( - mock_rest_client.agent_api_memories.create_agent_memory.call_args.kwargs - ) - memory_payload = call_kwargs["memory"].model_dump(exclude_unset=True) - assert "subject_id" not in memory_payload - assert "metadata" not in memory_payload - assert call_kwargs["request_options"] is DEFAULT_REQUEST_OPTIONS - - @pytest.mark.asyncio - async def test_store_memory_agent_scope_sent_without_subject_id( - self, mock_rest_client - ) -> None: - """Agent scope round-trips to the REST payload untouched and needs no - subject_id -- the scope value real agents fall back to when their owner - has no organization.""" - response = MagicMock() - response.data = MagicMock() - mock_rest_client.agent_api_memories.create_agent_memory = AsyncMock( - return_value=response - ) - tools = AgentTools("room-123", mock_rest_client) - - await tools.store_memory( - content="remember this", - system="working", - type="semantic", - segment="user", - thought="useful later", - scope="agent", - ) - - call_kwargs = ( - mock_rest_client.agent_api_memories.create_agent_memory.call_args.kwargs - ) - memory_payload = call_kwargs["memory"].model_dump(exclude_unset=True) - assert memory_payload["scope"] == "agent" - assert "subject_id" not in memory_payload - - @pytest.mark.asyncio - async def test_list_memories_agent_scope_sent_untouched( - self, mock_rest_client - ) -> None: - """`scope="agent"` passes straight through to the list filter.""" - response = MagicMock() - response.data = [] - mock_rest_client.agent_api_memories.list_agent_memories = AsyncMock( - return_value=response - ) - tools = AgentTools("room-123", mock_rest_client) - - await tools.list_memories(scope="agent") - - mock_rest_client.agent_api_memories.list_agent_memories.assert_awaited_once_with( - page_size=50, - scope="agent", - request_options=DEFAULT_REQUEST_OPTIONS, - ) - - @pytest.mark.asyncio - async def test_store_memory_organization_scope_rejection_becomes_actionable( - self, mock_rest_client - ) -> None: - """The platform's 422 for an orgless agent's `scope="organization"` - write must reach the caller as guidance to retry with `scope="agent"`, - not as the raw `headers: ..., status_code: 422, body: ...` dump the - generic exception handler would otherwise produce.""" - rejection_body = { - "error": { - "code": ORGANIZATION_SCOPE_REJECTED_CODE, - "details": {"organization_id": "must be present"}, - } - } - mock_rest_client.agent_api_memories.create_agent_memory = AsyncMock( - side_effect=UnprocessableEntityError(body=rejection_body) - ) - tools = AgentTools("room-123", mock_rest_client) - - with pytest.raises(BandToolError, match='scope="agent"'): - await tools.store_memory( - content="remember this", - system="working", - type="semantic", - segment="user", - thought="useful later", - scope="organization", - ) - - @pytest.mark.asyncio - async def test_store_memory_unrelated_422_is_not_mistaken_for_scope_rejection( - self, mock_rest_client - ) -> None: - """A 422 for any other reason must propagate as-is -- rewriting it to - the scope-retry message would mislead the caller about the actual - failure.""" - mock_rest_client.agent_api_memories.create_agent_memory = AsyncMock( - side_effect=UnprocessableEntityError( - body={"error": {"code": "some_other_validation_failure"}} - ) - ) - tools = AgentTools("room-123", mock_rest_client) - - with pytest.raises(UnprocessableEntityError): - await tools.store_memory( - content="remember this", - system="working", - type="semantic", - segment="user", - thought="useful later", - scope="organization", - ) - - @pytest.mark.asyncio - async def test_list_memories_organization_scope_rejection_becomes_actionable( - self, mock_rest_client - ) -> None: - """The read-side twin of the store-side test above.""" - rejection_body = {"error": {"code": ORGANIZATION_SCOPE_REJECTED_CODE}} - mock_rest_client.agent_api_memories.list_agent_memories = AsyncMock( - side_effect=UnprocessableEntityError(body=rejection_body) - ) - tools = AgentTools("room-123", mock_rest_client) - - with pytest.raises(BandToolError, match='scope="agent"'): - await tools.list_memories(scope="organization") - - @pytest.mark.asyncio - async def test_store_memory_rejects_subject_scope_without_subject_id( - self, mock_rest_client - ) -> None: - """Reject subject-scoped writes before they reach the API.""" - mock_rest_client.agent_api_memories.create_agent_memory = AsyncMock() - tools = AgentTools("room-123", mock_rest_client) - - with pytest.raises(ValueError, match="requires a subject_id"): - await tools.store_memory( - content="remember this", - system="working", - type="semantic", - segment="user", - thought="useful later", - scope="subject", - ) - - mock_rest_client.agent_api_memories.create_agent_memory.assert_not_called() - - @pytest.mark.asyncio - async def test_store_memory_rejects_type_for_wrong_system( - self, mock_rest_client - ) -> None: - """Reject a system/type mismatch before it reaches the API -- the raw - tool method's ValueError comes straight from band_sdk_core, unlike - StoreMemoryInput's model_validate, which wraps it in a - pydantic.ValidationError.""" - mock_rest_client.agent_api_memories.create_agent_memory = AsyncMock() - tools = AgentTools("room-123", mock_rest_client) - - with pytest.raises( - ValueError, match="type `semantic` is not valid for system `sensory`" - ): - await tools.store_memory( - content="remember this", - system="sensory", - type="semantic", - segment="user", - thought="useful later", - scope="agent", - ) - - mock_rest_client.agent_api_memories.create_agent_memory.assert_not_called() - - def test_store_memory_input_rejects_subject_scope_without_subject_id(self) -> None: - """Validate tool input rejects subject scope without subject_id.""" - with pytest.raises(ValidationError, match="requires a subject_id"): - StoreMemoryInput.model_validate( - { - "content": "remember this", - "system": "working", - "type": "semantic", - "segment": "user", - "thought": "useful later", - "scope": "subject", - } - ) - - def test_store_memory_input_rejects_type_for_wrong_system(self) -> None: - """Validate memory type matches the chosen system.""" - with pytest.raises( - ValidationError, match="type `semantic` is not valid for system `sensory`" - ): - StoreMemoryInput.model_validate( - { - "content": "remember this", - "system": "sensory", - "type": "semantic", - "segment": "user", - "thought": "useful later", - "scope": "organization", - } - ) - - @pytest.mark.parametrize( - ("tool_method", "rest_method"), - [ - ("get_memory", "get_agent_memory"), - ("supersede_memory", "supersede_agent_memory"), - ("archive_memory", "archive_agent_memory"), - ], - ) - @pytest.mark.asyncio - async def test_memory_mutation_calls_use_default_request_options( - self, - mock_rest_client, - tool_method: str, - rest_method: str, - ) -> None: - response = MagicMock() - response.data = MagicMock() - rest_call = AsyncMock(return_value=response) - setattr(mock_rest_client.agent_api_memories, rest_method, rest_call) - tools = AgentTools("room-123", mock_rest_client) - - await getattr(tools, tool_method)("mem-123") - - rest_call.assert_awaited_once_with( - id="mem-123", - request_options=DEFAULT_REQUEST_OPTIONS, - ) - - -def _attachment( - file_id: str = "file-1", - *, - name: str = "notes.txt", - content_type: str = "text/plain", - size: int = 20, - expires_at: datetime | None = None, -) -> Attachment: - return Attachment( - id=file_id, - name=name, - content_type=content_type, - bytes=size, - sha256="a" * 64, - has_thumb=False, - expires_at=expires_at, - ) - - -def _context_response( - messages: list[ChatMessage], - *, - next_cursor: str | None = None, - has_more: bool = False, -) -> GetAgentChatContextResponse: - return GetAgentChatContextResponse( - data=messages, - metadata=GetAgentChatContextResponseMetadata( - has_more=has_more, limit=50, next_cursor=next_cursor - ), - ) - - -def _message_with_attachments( - msg_id: str, attachments: list[Attachment] -) -> ChatMessage: - return ChatMessage( - id=msg_id, - content="", - sender_id="user-1", - sender_type="User", - message_type="text", - attachments=attachments, - ) - - -def _mock_attachment_page(mock_rest_client: MagicMock, attachment: Attachment) -> None: - """Configure the context endpoint to return one message carrying `attachment` -- - the shared setup for every read_room_file test that only cares about one file.""" - mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( - return_value=_context_response( - [_message_with_attachments("msg-1", [attachment])] - ) - ) - - -async def _fake_download(body: bytes): - """A ``download_agent_chat_file`` double: an async generator, not a - coroutine -- assigning an ``AsyncMock`` here would make the *call* itself - awaitable, which nothing in the real client does.""" - yield body - - -async def _fake_download_not_found(): - """Same shape as ``_fake_download``, but raises before ever yielding.""" - if False: - yield b"" - raise NotFoundError(body=MagicMock()) - - -def _posted_message(mock_rest_client: MagicMock) -> ChatMessageRequest: - """The ``ChatMessageRequest`` a preceding ``send_message``/``send_room_file`` - call actually posted -- the one observable outcome a REST-backed - ``AgentTools`` test has for "what did the message body look like".""" - return ( - mock_rest_client.agent_api_messages.create_agent_chat_message.await_args.kwargs[ - "message" - ] - ) - - -class TestFileTools: - """Tests for band_list_room_files / band_read_room_file / band_send_room_file.""" - - # --- list_room_files --- - - @pytest.mark.asyncio - async def test_list_room_files_dedupes_by_attachment_id(self, mock_rest_client): - shared = _attachment("file-1") - other = _attachment("file-2") - response = _context_response( - [ - _message_with_attachments("msg-1", [shared]), - _message_with_attachments("msg-2", [shared, other]), - ] - ) - mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( - return_value=response - ) - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.list_room_files() - - assert [a["id"] for a in result["data"]] == ["file-1", "file-2"] - mock_rest_client.agent_api_context.get_agent_chat_context.assert_awaited_once_with( - chat_id="room-123", - request_options=DEFAULT_REQUEST_OPTIONS, - ) - - @pytest.mark.asyncio - async def test_list_room_files_forwards_cursor_and_returns_next( - self, mock_rest_client - ): - response = _context_response( - [_message_with_attachments("msg-1", [_attachment()])], - next_cursor="cursor-2", - has_more=True, - ) - mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( - return_value=response - ) - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.list_room_files(cursor="cursor-1") - - assert result["next_cursor"] == "cursor-2" - mock_rest_client.agent_api_context.get_agent_chat_context.assert_awaited_once_with( - chat_id="room-123", - request_options=DEFAULT_REQUEST_OPTIONS, - cursor="cursor-1", - ) - - # --- read_room_file --- - - @pytest.mark.asyncio - async def test_read_room_file_inlines_small_text(self, mock_rest_client): - attachment = _attachment(content_type="text/plain", size=5) - _mock_attachment_page(mock_rest_client, attachment) - mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( - _fake_download(b"hello") - ) - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.read_room_file("file-1") - - assert result["text"] == "hello" - assert result["content_type"] == "text/plain" - assert "description" not in result - - @pytest.mark.asyncio - async def test_read_room_file_non_utf8_text_flags_lossy_decode( - self, mock_rest_client - ): - """A Windows-1252-encoded file (no charset in content_type -- the - platform derives it from magic bytes alone) must not come back - looking like a clean decode: 0x93/0x94 are curly quotes in - Windows-1252 but invalid UTF-8 continuation bytes on their own.""" - windows_1252_bytes = "“quoted”".encode("windows-1252") - attachment = _attachment( - content_type="text/plain", size=len(windows_1252_bytes) - ) - _mock_attachment_page(mock_rest_client, attachment) - mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( - _fake_download(windows_1252_bytes) - ) - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.read_room_file("file-1") - - assert "�" in result["text"] - assert "not valid UTF-8" in result["description"] - - @pytest.mark.asyncio - async def test_read_room_file_inlines_small_image(self, mock_rest_client): - attachment = _attachment(content_type="image/png", size=100) - _mock_attachment_page(mock_rest_client, attachment) - mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( - _fake_download(b"\x89PNG-fake-bytes") - ) - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.read_room_file("file-1") - - block = result["content"][0] - assert block["type"] == "image" - assert block["mimeType"] == "image/png" - assert base64.b64decode(block["data"]) == b"\x89PNG-fake-bytes" - - @pytest.mark.asyncio - async def test_read_room_file_text_exactly_at_limit_downloads( - self, mock_rest_client - ): - attachment = _attachment(content_type="text/plain", size=MAX_INLINE_TEXT_BYTES) - _mock_attachment_page(mock_rest_client, attachment) - download = MagicMock(side_effect=lambda **_kw: _fake_download(b"ok")) - mock_rest_client.agent_api_files.download_agent_chat_file = download - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.read_room_file("file-1") - - download.assert_called_once() - assert result["text"] == "ok" - - @pytest.mark.asyncio - async def test_read_room_file_text_one_byte_over_limit_skips_download( - self, mock_rest_client - ): - attachment = _attachment( - content_type="text/plain", size=MAX_INLINE_TEXT_BYTES + 1 - ) - _mock_attachment_page(mock_rest_client, attachment) - download = MagicMock(side_effect=lambda **_kw: _fake_download(b"ok")) - mock_rest_client.agent_api_files.download_agent_chat_file = download - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.read_room_file("file-1") - - download.assert_not_called() - assert "text" not in result - assert "description" in result - - @pytest.mark.asyncio - async def test_read_room_file_image_exactly_at_limit_downloads( - self, mock_rest_client - ): - attachment = _attachment(content_type="image/png", size=MAX_INLINE_IMAGE_BYTES) - _mock_attachment_page(mock_rest_client, attachment) - download = MagicMock(side_effect=lambda **_kw: _fake_download(b"ok")) - mock_rest_client.agent_api_files.download_agent_chat_file = download - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.read_room_file("file-1") - - download.assert_called_once() - assert result["content"][0]["type"] == "image" - - @pytest.mark.asyncio - async def test_read_room_file_image_one_byte_over_limit_skips_download( - self, mock_rest_client - ): - attachment = _attachment( - content_type="image/png", size=MAX_INLINE_IMAGE_BYTES + 1 - ) - _mock_attachment_page(mock_rest_client, attachment) - download = MagicMock(side_effect=lambda **_kw: _fake_download(b"ok")) - mock_rest_client.agent_api_files.download_agent_chat_file = download - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.read_room_file("file-1") - - download.assert_not_called() - assert "content" not in result - assert "description" in result - - @pytest.mark.asyncio - async def test_read_room_file_non_previewable_type_is_described( - self, mock_rest_client - ): - attachment = _attachment(content_type="application/pdf", size=10) - _mock_attachment_page(mock_rest_client, attachment) - download = MagicMock(side_effect=lambda **_kw: _fake_download(b"ok")) - mock_rest_client.agent_api_files.download_agent_chat_file = download - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.read_room_file("file-1") - - download.assert_not_called() - assert result["description"] - - @pytest.mark.asyncio - async def test_read_room_file_unknown_id_raises_band_tool_error( - self, mock_rest_client - ): - mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( - return_value=_context_response([]) - ) - tools = AgentTools("room-123", mock_rest_client) - - with pytest.raises(BandToolError, match=FILE_UNAVAILABLE_MESSAGE): - await tools.read_room_file("nope") - - @pytest.mark.asyncio - async def test_read_room_file_download_not_found_raises_band_tool_error( - self, mock_rest_client - ): - attachment = _attachment(content_type="text/plain", size=5) - _mock_attachment_page(mock_rest_client, attachment) - mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( - _fake_download_not_found() - ) - tools = AgentTools("room-123", mock_rest_client) - - with pytest.raises(BandToolError, match=FILE_UNAVAILABLE_MESSAGE): - await tools.read_room_file("file-1") - - @pytest.mark.asyncio - async def test_read_room_file_searches_past_the_first_page(self, mock_rest_client): - """_find_attachment must walk every page -- the target file may be - older than the first page returned.""" - page_one = _context_response( - [_message_with_attachments("msg-1", [_attachment("other-file")])], - next_cursor="cursor-2", - has_more=True, - ) - page_two = _context_response( - [ - _message_with_attachments( - "msg-2", [_attachment(content_type="text/plain", size=5)] - ) - ] - ) - mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( - side_effect=[page_one, page_two] - ) - mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( - _fake_download(b"hello") - ) - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.read_room_file("file-1") - - assert result["text"] == "hello" - get_context = mock_rest_client.agent_api_context.get_agent_chat_context - assert get_context.await_count == 2 - _, second_call_kwargs = get_context.await_args_list[1] - assert second_call_kwargs["cursor"] == "cursor-2" - - @pytest.mark.asyncio - async def test_find_attachment_stops_on_repeated_cursor(self, mock_rest_client): - """A malformed response repeating a cursor must not loop forever -- - the only pagination guard, since a room's history has no realistic - depth ceiling to cap against (unlike iter_chat_pages's room count).""" - - def _looping_page(**_kwargs: Any) -> GetAgentChatContextResponse: - return _context_response( - [_message_with_attachments("msg", [_attachment("other-file")])], - next_cursor="same-cursor", - has_more=True, - ) - - mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( - side_effect=_looping_page - ) - tools = AgentTools("room-123", mock_rest_client) - - with pytest.raises(BandToolError, match=FILE_UNAVAILABLE_MESSAGE): - await tools._find_attachment("file-1") - - get_context = mock_rest_client.agent_api_context.get_agent_chat_context - assert get_context.await_count == 2 - - @pytest.mark.asyncio - async def test_find_attachment_searches_past_fifty_pages(self, mock_rest_client): - """A room with more than 50 pages of history before the target file - must still resolve it -- there is no depth cap.""" - target_page = 60 - target = _attachment(content_type="text/plain", size=5) - - def _paged_history(**kwargs: Any) -> GetAgentChatContextResponse: - cursor = kwargs.get("cursor") - page = int(cursor.removeprefix("page-")) if cursor else 1 - if page < target_page: - return _context_response( - [ - _message_with_attachments( - f"msg-{page}", [_attachment("other-file")] - ) - ], - next_cursor=f"page-{page + 1}", - has_more=True, - ) - return _context_response( - [_message_with_attachments(f"msg-{page}", [target])] - ) - - mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( - side_effect=_paged_history - ) - mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( - _fake_download(b"hello") - ) - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.read_room_file("file-1") - - assert result["text"] == "hello" - get_context = mock_rest_client.agent_api_context.get_agent_chat_context - assert get_context.await_count == target_page - - @pytest.mark.asyncio - async def test_find_attachment_skips_pagination_on_cache_hit( - self, mock_rest_client - ): - """A second lookup for the same (room, rest, file_id) must not - re-paginate -- _find_attachment delegates to a shared @alru_cache - keyed on those three, not on the AgentTools instance calling it.""" - _mock_attachment_page(mock_rest_client, _attachment("file-1")) - first = AgentTools("room-123", mock_rest_client) - second = AgentTools("room-123", mock_rest_client) - - await first._find_attachment("file-1") - await second._find_attachment("file-1") - - mock_rest_client.agent_api_context.get_agent_chat_context.assert_awaited_once() - - @pytest.mark.asyncio - async def test_find_attachment_does_not_share_cache_across_rooms( - self, mock_rest_client - ): - """The same file_id looked up in two different rooms must not - cache-hit across them -- room_id is part of the cache key, not - just file_id.""" - _mock_attachment_page(mock_rest_client, _attachment("file-1")) - room_a = AgentTools("room-a", mock_rest_client) - room_b = AgentTools("room-b", mock_rest_client) - - await room_a._find_attachment("file-1") - await room_b._find_attachment("file-1") - - assert ( - mock_rest_client.agent_api_context.get_agent_chat_context.await_count == 2 - ) - - @pytest.mark.asyncio - async def test_find_attachment_does_not_share_cache_across_rest_clients( - self, mock_rest_client - ): - """The same (room_id, file_id) looked up through two different REST - clients must not cache-hit across them -- the rest client identity is - part of the cache key, so two agents (or a pool of clients) never - leak each other's attachment metadata.""" - _mock_attachment_page(mock_rest_client, _attachment("file-1")) - other_rest_client = MagicMock() - _mock_attachment_page(other_rest_client, _attachment("file-1")) - tools_a = AgentTools("room-123", mock_rest_client) - tools_b = AgentTools("room-123", other_rest_client) - - await tools_a._find_attachment("file-1") - await tools_b._find_attachment("file-1") - - mock_rest_client.agent_api_context.get_agent_chat_context.assert_awaited_once() - other_rest_client.agent_api_context.get_agent_chat_context.assert_awaited_once() - - @pytest.mark.asyncio - async def test_find_attachment_retries_pagination_after_a_miss( - self, mock_rest_client - ): - """A miss must not be cached: a file posted after a failed lookup is - still findable on the next attempt, not permanently "not found".""" - mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( - return_value=_context_response([]) - ) - tools = AgentTools("room-123", mock_rest_client) - - with pytest.raises(BandToolError, match=FILE_UNAVAILABLE_MESSAGE): - await tools._find_attachment("file-1") - - _mock_attachment_page(mock_rest_client, _attachment("file-1")) - result = await tools._find_attachment("file-1") - - assert result.id == "file-1" - - @pytest.mark.asyncio - async def test_find_attachment_rejects_expired_cached_entry(self, mock_rest_client): - """An attachment past its own expires_at must not be handed back - from cache -- it's evicted and re-fetched once (the cached copy may - just predate the platform extending the deadline), and only raised - as not-found if the fresh read is still expired too -- which also - evicts it, so the cache never keeps serving metadata it already - gave up on.""" - expired = _attachment( - "file-1", expires_at=datetime.now(UTC) - timedelta(seconds=1) - ) - _mock_attachment_page(mock_rest_client, expired) - tools = AgentTools("room-123", mock_rest_client) - - with pytest.raises(BandToolError, match=FILE_UNAVAILABLE_MESSAGE): - await tools._find_attachment("file-1") - - get_context = mock_rest_client.agent_api_context.get_agent_chat_context - assert get_context.await_count == 2 - assert not AgentTools._attachment_cache().cache_contains( - "room-123", mock_rest_client, "file-1" - ) - - @pytest.mark.asyncio - async def test_find_attachment_refreshes_an_expired_entry(self, mock_rest_client): - """A cache entry whose expires_at has passed gets exactly one - real re-fetch -- if that comes back not-expired (the platform - extended it), it's returned, not rejected on the stale cached - timestamp.""" - not_expired = _attachment( - "file-1", - content_type="text/plain", - size=5, - expires_at=datetime.now(UTC) + timedelta(days=1), - ) - expired_page = _context_response( - [ - _message_with_attachments( - "msg-1", - [ - _attachment( - "file-1", - expires_at=datetime.now(UTC) - timedelta(seconds=1), - ) - ], - ) - ] - ) - refreshed_page = _context_response( - [_message_with_attachments("msg-1", [not_expired])] - ) - mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( - side_effect=[expired_page, refreshed_page] - ) - mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( - _fake_download(b"hello") - ) - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.read_room_file("file-1") - - assert result["text"] == "hello" - - @pytest.mark.asyncio - async def test_find_attachment_rejects_expired_naive_timestamp( - self, mock_rest_client - ): - """A naive (offset-less) expires_at -- the Fern model doesn't enforce - one -- must be treated as UTC, not raise on comparison to aware - now().""" - expired = _attachment( - "file-1", - expires_at=datetime(2020, 1, 1), - ) - _mock_attachment_page(mock_rest_client, expired) - tools = AgentTools("room-123", mock_rest_client) - - with pytest.raises(BandToolError, match=FILE_UNAVAILABLE_MESSAGE): - await tools._find_attachment("file-1") - - def test_attachment_cache_maxsize_is_wired_to_settings(self) -> None: - """The @alru_cache's maxsize must actually come from RuntimeSettings - -- the eviction algorithm itself is async-lru's to test, not ours.""" - assert AgentTools._attachment_cache().cache_parameters()["maxsize"] == ( - RuntimeSettings().BAND_ATTACHMENT_CACHE_MAXSIZE - ) - - def test_attachment_cache_rereads_settings_lazily( - self, monkeypatch: pytest.MonkeyPatch - ) -> None: - """maxsize must come from RuntimeSettings at first real use, not at - module import -- every example does `from band import Agent` (which - imports this module) before its own load_dotenv() call, so baking - RuntimeSettings() into a decorator at class-body time would silently - ignore a .env-set override.""" - AgentTools._attachment_cache.cache_clear() - monkeypatch.setenv("BAND_ATTACHMENT_CACHE_MAXSIZE", "42") - try: - assert AgentTools._attachment_cache().cache_parameters()["maxsize"] == 42 - finally: - AgentTools._attachment_cache.cache_clear() - - @pytest.mark.asyncio - async def test_read_room_file_caches_attachments_across_recreated_tools( - self, mock_rest_client - ): - """Real ExecutionContext + from_context(): a file found by one turn's - AgentTools must skip re-pagination on the next turn's recreated - instance, since AgentTools.from_context() rebuilds fresh every turn. - """ - _mock_attachment_page( - mock_rest_client, _attachment(content_type="text/plain", size=5) - ) - mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( - _fake_download(b"hello") - ) - ctx = ExecutionContext( - room_id="room-789", - link=MagicMock(rest=mock_rest_client), - on_execute=AsyncMock(), - ) - - tools1 = AgentTools.from_context(ctx) - await tools1.read_room_file("file-1") - - tools2 = AgentTools.from_context(ctx) - await tools2.read_room_file("file-1") - - mock_rest_client.agent_api_context.get_agent_chat_context.assert_awaited_once() - - # --- send_room_file --- - - @pytest.mark.asyncio - async def test_send_room_file_uploads_and_posts_message( - self, mock_rest_client, participants - ): - uploaded = _attachment("file-9", name="report.txt") - upload_response = MagicMock() - upload_response.data = uploaded - mock_rest_client.agent_api_files.upload_agent_chat_file = AsyncMock( - return_value=upload_response - ) - - tools = AgentTools("room-123", mock_rest_client, participants) - - result = await tools.send_room_file( - "hello world", "report.txt", caption="here you go", mentions=["User One"] - ) - - assert result["attachment"]["id"] == "file-9" - upload_call = mock_rest_client.agent_api_files.upload_agent_chat_file - upload_call.assert_awaited_once() - _, kwargs = upload_call.await_args - assert kwargs["chat_id"] == "room-123" - assert kwargs["request"] == b"hello world" - headers = kwargs["request_options"]["additional_headers"] - assert headers["x-file-name"] == "report.txt" - mock_rest_client.agent_api_messages.create_agent_chat_message.assert_awaited_once() - posted = _posted_message(mock_rest_client) - assert posted.attachment_ids == ["file-9"] - assert posted.content == "here you go" - - @pytest.mark.asyncio - async def test_send_room_file_exactly_at_limit_succeeds( - self, mock_rest_client, participants - ): - uploaded = _attachment("file-9") - upload_response = MagicMock() - upload_response.data = uploaded - mock_rest_client.agent_api_files.upload_agent_chat_file = AsyncMock( - return_value=upload_response - ) - tools = AgentTools("room-123", mock_rest_client, participants) - body = "a" * MAX_SEND_CONTENT_BYTES - - await tools.send_room_file(body, "big.txt", mentions=["User One"]) - - mock_rest_client.agent_api_files.upload_agent_chat_file.assert_awaited_once() - - @pytest.mark.asyncio - async def test_send_room_file_one_byte_over_limit_raises_before_upload( - self, mock_rest_client, participants - ): - tools = AgentTools("room-123", mock_rest_client, participants) - body = "a" * (MAX_SEND_CONTENT_BYTES + 1) - - with pytest.raises(BandToolError, match="exceeds"): - await tools.send_room_file(body, "big.txt", mentions=["User One"]) - - mock_rest_client.agent_api_files.upload_agent_chat_file.assert_not_called() - - @pytest.mark.asyncio - async def test_send_room_file_non_ascii_filename_raises_before_upload( - self, mock_rest_client, participants - ): - """x-file-name travels as a raw HTTP header value -- httpx refuses to - encode a non-ASCII header and raises UnicodeEncodeError deep inside - the upload call. Catch it before uploading with a clear message - instead of letting that raw codec error surface to the LLM.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - with pytest.raises(BandToolError, match="ASCII characters only"): - await tools.send_room_file("hi", "报告.txt", mentions=["User One"]) - - mock_rest_client.agent_api_files.upload_agent_chat_file.assert_not_called() - - @pytest.mark.asyncio - async def test_send_room_file_crlf_in_filename_raises_before_upload( - self, mock_rest_client, participants - ): - """A bare '\\r'/'\\n' encodes to ASCII fine, so an "is it ASCII" - check alone would wave a header-injection payload straight through - to the upload call -- only rejected there, deep inside h11, as an - "Illegal header value" once the request is already being built.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - with pytest.raises(BandToolError, match="ASCII characters only"): - await tools.send_room_file( - "hi", "evil\r\nX-Injected: yes", mentions=["User One"] - ) - - mock_rest_client.agent_api_files.upload_agent_chat_file.assert_not_called() - - @pytest.mark.asyncio - async def test_send_room_file_upload_not_found_raises_band_tool_error( - self, mock_rest_client, participants - ): - mock_rest_client.agent_api_files.upload_agent_chat_file = AsyncMock( - side_effect=NotFoundError(body=MagicMock()) - ) - tools = AgentTools("room-123", mock_rest_client, participants) - - with pytest.raises(BandToolError, match=FILE_UNAVAILABLE_MESSAGE): - await tools.send_room_file("hi", "f.txt", mentions=["User One"]) - - @pytest.mark.asyncio - async def test_send_room_file_empty_mentions_raises_before_upload( - self, mock_rest_client, participants - ): - """Sharing a file is a send_message call under the hood, so a missing - mention must fail before the upload -- not after, which would leave - an orphaned attachment nothing points at.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - with pytest.raises(BandToolError, match="At least one mention is required"): - await tools.send_room_file("hi", "f.txt", mentions=[]) - - mock_rest_client.agent_api_files.upload_agent_chat_file.assert_not_called() - - @pytest.mark.asyncio - async def test_send_room_file_empty_caption_uses_default( - self, mock_rest_client, participants - ): - """The platform rejects blank message content outright - (``validate_length(:content, min: 1)``, unconditional even with - ``attachment_ids`` set) -- an omitted caption must never reach - ``send_message`` as ``""``, or the post 422s after the file has - already been uploaded, leaving an orphaned attachment.""" - uploaded = _attachment("file-9", name="report.txt") - upload_response = MagicMock() - upload_response.data = uploaded - mock_rest_client.agent_api_files.upload_agent_chat_file = AsyncMock( - return_value=upload_response - ) - tools = AgentTools("room-123", mock_rest_client, participants) - - result = await tools.send_room_file( - "hello world", "report.txt", mentions=["User One"] - ) - - assert result["attachment"]["id"] == "file-9" - assert _posted_message(mock_rest_client).content == ( - DEFAULT_FILE_CAPTION.format(filename="report.txt") - ) - - @pytest.mark.asyncio - @pytest.mark.parametrize("caption", BLANK_CONTENT_CASES) - async def test_send_room_file_blank_caption_uses_default( - self, mock_rest_client, participants, caption - ): - """A caption that only *looks* non-empty falls back to the default - too: the send refuses content with no visible characters and returns - ``None``, so anything narrower than the platform's own rule here - drops the post after the file is already uploaded, leaving an - orphaned attachment.""" - uploaded = _attachment("file-9", name="report.txt") - upload_response = MagicMock() - upload_response.data = uploaded - mock_rest_client.agent_api_files.upload_agent_chat_file = AsyncMock( - return_value=upload_response - ) - tools = AgentTools("room-123", mock_rest_client, participants) - - result = await tools.send_room_file( - "hello world", "report.txt", caption=caption, mentions=["User One"] - ) - - assert result["attachment"]["id"] == "file-9" - assert _posted_message(mock_rest_client).content == ( - DEFAULT_FILE_CAPTION.format(filename="report.txt") - ) - - @pytest.mark.asyncio - async def test_send_room_file_discoverable_from_a_new_agenttools_instance( - self, mock_rest_client, participants - ): - """AgentTools is recreated per execution (a fresh instance per turn), - so a file must be discoverable through the REST layer alone, not - through any in-process state the sending instance happened to hold. - Regression guard for the agent messages endpoint's direct_only - exclusion of self-authored messages: file discovery must go through - the context endpoint instead, which includes them.""" - uploaded = _attachment("file-9", name="report.txt", size=5) - upload_response = MagicMock() - upload_response.data = uploaded - mock_rest_client.agent_api_files.upload_agent_chat_file = AsyncMock( - return_value=upload_response - ) - sender = AgentTools("room-123", mock_rest_client, participants) - sent = await sender.send_room_file("hello", "report.txt", mentions=["User One"]) - - # A later turn: brand new instance, no shared state with `sender`. - # Mirrors what the real platform now returns -- the context endpoint - # includes messages this agent sent, unlike the plain messages one. - _mock_attachment_page(mock_rest_client, uploaded) - mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( - _fake_download(b"hello") - ) - reader = AgentTools("room-123", mock_rest_client, participants) - - listed = await reader.list_room_files() - read = await reader.read_room_file(sent["attachment"]["id"]) - - assert [a["id"] for a in listed["data"]] == ["file-9"] - assert read["text"] == "hello" - - -class TestAgentToolsConstruction: - """Test AgentTools initialization.""" - - def test_init_stores_room_id(self, mock_rest_client): - """Should store room_id.""" - tools = AgentTools("room-123", mock_rest_client) - - assert tools.room_id == "room-123" - - def test_init_stores_rest_client(self, mock_rest_client): - """Should store REST client.""" - tools = AgentTools("room-123", mock_rest_client) - - assert tools.rest is mock_rest_client - - def test_init_empty_participants_by_default(self, mock_rest_client): - """Should have empty participants by default.""" - tools = AgentTools("room-123", mock_rest_client) - - assert tools._participants == [] - - def test_init_with_participants(self, mock_rest_client, participants): - """Should accept participants.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - assert tools._participants == participants - - -class TestAgentToolsFromContext: - """Test AgentTools.from_context() factory method.""" - - def test_from_context_creates_tools(self, mock_rest_client, participants): - """from_context() should create AgentTools from ExecutionContext.""" - # Mock ExecutionContext - mock_ctx = MagicMock() - mock_ctx.room_id = "room-456" - mock_ctx.link = MagicMock() - mock_ctx.link.rest = mock_rest_client - mock_ctx.participants = participants - - tools = AgentTools.from_context(mock_ctx) - - assert tools.room_id == "room-456" - assert tools.rest is mock_rest_client - assert tools._participants == participants - - -class TestAgentToolsContextSyncBack: - """Regression tests for AgentTools._ctx sync-back to ExecutionContext. - - Before the fix, AgentTools.from_context() copied ctx.participants (which - returns a shallow copy via property). Mutations to the tools instance - (add/remove participant) never propagated back to the ExecutionContext. - On the next turn, from_context() would copy the stale list again. - """ - - @pytest.mark.asyncio - async def test_add_participant_syncs_back_to_ctx(self, mock_rest_client): - """add_participant() must call ctx.add_participant() with full dict.""" - mock_ctx = MagicMock() - mock_ctx.room_id = "room-456" - mock_ctx.link = MagicMock() - mock_ctx.link.rest = mock_rest_client - mock_ctx.participants = [] - mock_ctx.hub_room_id = None - mock_ctx.add_participant = MagicMock() - - # Override list_agent_chat_participants to return empty (avoid "already_in_room") - mock_rest_client.agent_api_participants.list_agent_chat_participants = ( - AsyncMock(return_value=MagicMock(data=[])) - ) - - tools = AgentTools.from_context(mock_ctx) - assert tools._ctx is mock_ctx - - await tools.add_participant("Agent Two") - - mock_ctx.add_participant.assert_called_once() - added = mock_ctx.add_participant.call_args.args[0] - assert added["id"] == "agent-2" - assert added["name"] == "Agent Two" - assert added["type"] == "Agent" - assert added["handle"] == "agent-two" - assert added["description"] == "Another agent" - - @pytest.mark.asyncio - async def test_remove_participant_syncs_back_to_ctx(self, mock_rest_client): - """remove_participant() must call ctx.remove_participant() with correct ID.""" - participant = { - "id": "user-1", - "name": "User One", - "type": "User", - "handle": "user-one", - } - - mock_ctx = MagicMock() - mock_ctx.room_id = "room-456" - mock_ctx.link = MagicMock() - mock_ctx.link.rest = mock_rest_client - mock_ctx.participants = [participant] - mock_ctx.hub_room_id = None - mock_ctx.remove_participant = MagicMock() - - # Return same participant from REST so snapshot matches - p_mock = make_participant_mock("user-1", "User One", "User", handle="user-one") - mock_rest_client.agent_api_participants.list_agent_chat_participants = ( - AsyncMock(return_value=MagicMock(data=[p_mock])) - ) - - tools = AgentTools.from_context(mock_ctx) - await tools.remove_participant("User One") - - mock_ctx.remove_participant.assert_called_once_with("user-1") - - @pytest.mark.asyncio - async def test_add_participant_persists_across_recreated_tools( - self, mock_rest_client - ): - """Added participant must survive tools recreation via from_context(). - - Uses real ExecutionContext — its ``participants`` property returns a - copy, so without the _ctx backref the mutation would be lost. - """ - - ctx = ExecutionContext( - room_id="room-789", - link=MagicMock(rest=mock_rest_client), - on_execute=AsyncMock(), - ) - - # Empty room - mock_rest_client.agent_api_participants.list_agent_chat_participants = ( - AsyncMock(return_value=MagicMock(data=[])) - ) - - # Turn 1: add participant - tools1 = AgentTools.from_context(ctx) - await tools1.add_participant("Agent Two") - - assert len(ctx.participants) == 1 - assert ctx.participants[0]["id"] == "agent-2" - - # Turn 2: recreate tools — participant must still be there - tools2 = AgentTools.from_context(ctx) - assert len(tools2._participants) == 1 - assert tools2._participants[0]["id"] == "agent-2" - - @pytest.mark.asyncio - async def test_remove_participant_persists_across_recreated_tools( - self, mock_rest_client - ): - """Removed participant must stay removed after tools recreation. - - Uses real ExecutionContext — its ``participants`` property returns a - copy, so without the _ctx backref the removal would be lost. - """ - - participant = { - "id": "user-1", - "name": "User One", - "type": "User", - "handle": "user-one", - } - - ctx = ExecutionContext( - room_id="room-789", - link=MagicMock(rest=mock_rest_client), - on_execute=AsyncMock(), - ) - ctx.set_participants([participant]) - - # REST snapshot must match ctx.participants - p_mock = make_participant_mock("user-1", "User One", "User", handle="user-one") - mock_rest_client.agent_api_participants.list_agent_chat_participants = ( - AsyncMock(return_value=MagicMock(data=[p_mock])) - ) - - # Turn 1: remove participant - tools1 = AgentTools.from_context(ctx) - await tools1.remove_participant("User One") - - assert len(ctx.participants) == 0 - - # Turn 2: recreate tools — participant must stay removed - tools2 = AgentTools.from_context(ctx) - assert len(tools2._participants) == 0 - - -class TestAgentToolsSendMessage: - """Test send_message tool.""" - - async def test_send_message_success(self, mock_rest_client, participants): - """send_message() should send via REST and return the Fern model.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - result = await tools.send_message("Hello!", mentions=["User One"]) - - # Now returns Fern model (mock), not dict - assert result.model_dump()["id"] == "msg-123" - mock_rest_client.agent_api_messages.create_agent_chat_message.assert_called_once() - - async def test_send_message_resolves_mentions(self, mock_rest_client, participants): - """send_message() should resolve mention names to IDs and handles.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - await tools.send_message("Hello @User One!", mentions=["User One"]) - - call_args = ( - mock_rest_client.agent_api_messages.create_agent_chat_message.call_args - ) - message = call_args.kwargs["message"] - assert len(message.mentions) == 1 - assert message.mentions[0].id == "user-1" - assert message.mentions[0].handle == "@user-one" - - async def test_send_message_accepts_participant_id( - self, mock_rest_client, participants - ): - """An exact ID routes to its participant without knowing the handle.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - await tools.send_message("Hello!", mentions=["user-1"]) - - call = mock_rest_client.agent_api_messages.create_agent_chat_message.call_args - assert call.kwargs["message"].mentions[0].id == "user-1" - - async def test_send_message_omits_attachment_ids_when_not_given( - self, mock_rest_client, participants - ): - """Without attachment_ids, the field must be unset, not explicitly None. - - ChatMessageRequest serializes with exclude_unset=True; passing - attachment_ids=None would still mark the field "set" and send a - literal JSON null, which the platform rejects (expects an array or - an absent key) -- reproduced live against a real deployment. - """ - tools = AgentTools("room-123", mock_rest_client, participants) - - await tools.send_message("Hello!", mentions=["User One"]) - - call_args = ( - mock_rest_client.agent_api_messages.create_agent_chat_message.call_args - ) - message = call_args.kwargs["message"] - assert "attachment_ids" not in message.model_fields_set - - async def test_send_message_empty_mentions_excludes_self( - self, mock_rest_client, participants - ): - """The empty-mentions error lists other participants but not the agent - itself — an agent can't @mention itself.""" - - tools = AgentTools( - "room-123", mock_rest_client, participants, agent_id="user-2" - ) - - with pytest.raises(BandToolError) as exc_info: - await tools.send_message("Hello!", mentions=[]) - - message = str(exc_info.value) - assert "@user-one" in message - assert "@user-two" not in message - - async def test_send_message_unknown_mention_raises( - self, mock_rest_client, participants - ): - """send_message() should raise for unknown mention.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - with pytest.raises(ValueError, match="Unknown participant 'Unknown'"): - await tools.send_message("Hello!", mentions=["Unknown"]) - - @pytest.mark.parametrize("content", BLANK_CONTENT_CASES) - async def test_send_message_refuses_content_with_no_visible_characters( - self, mock_rest_client, participants, content - ): - """send_message() should never call the API with blank content.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - result = await tools.send_message(content, mentions=["User One"]) - - assert result is None - mock_rest_client.agent_api_messages.create_agent_chat_message.assert_not_called() - - -class TestAgentToolsSendEvent: - """Test send_event tool.""" - - async def test_send_event_success(self, mock_rest_client): - """send_event() should send via REST and return the Fern model.""" - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.send_event("Thinking...", "thought") - - # Now returns Fern model (mock), not dict - assert result.model_dump()["message_type"] == "thought" - mock_rest_client.agent_api_events.create_agent_chat_event.assert_called_once() - - async def test_send_event_with_metadata(self, mock_rest_client): - """send_event() should pass metadata.""" - tools = AgentTools("room-123", mock_rest_client) - - await tools.send_event("Error!", "error", metadata={"code": 500}) - - call_args = mock_rest_client.agent_api_events.create_agent_chat_event.call_args - event = call_args.kwargs["event"] - assert event.metadata == {"code": 500} - - async def test_send_event_within_limit_untouched(self, mock_rest_client): - """send_event() should pass short content through unchanged.""" - tools = AgentTools("room-123", mock_rest_client) - content = "x" * 16384 - - await tools.send_event(content, "tool_result") - - call_args = mock_rest_client.agent_api_events.create_agent_chat_event.call_args - assert call_args.kwargs["event"].content == content - - async def test_send_event_truncates_oversized_content(self, mock_rest_client): - """send_event() caps oversized content, keeping both its head and tail.""" - tools = AgentTools("room-123", mock_rest_client) - content = "HEAD" * 10000 + "TAIL" * 10000 - - await tools.send_event(content, "tool_result") - - call_args = mock_rest_client.agent_api_events.create_agent_chat_event.call_args - sent_content = call_args.kwargs["event"].content - assert len(sent_content) == 16384 - assert sent_content.startswith("HEAD") # head is retained - assert sent_content.endswith("TAIL") # and the tail isn't dropped - assert "[truncated]" in sent_content # with a marker between them - - @pytest.mark.parametrize("content", BLANK_CONTENT_CASES) - async def test_send_event_refuses_content_with_no_visible_characters( - self, mock_rest_client, content - ): - """send_event() should never call the API with blank content — the platform 422s on it.""" - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.send_event(content, "tool_result") - - assert result is None - mock_rest_client.agent_api_events.create_agent_chat_event.assert_not_called() - - async def test_stopped_scope_cancels_posts_without_latching_room( - self, mock_rest_client - ): - ctx = ExecutionContext( - "room-123", MagicMock(rest=mock_rest_client), AsyncMock() - ) - await ctx._begin_scope() - tools = AgentTools.from_context(ctx) - mock_rest_client.agent_api_events.create_agent_chat_event.side_effect = ( - ForbiddenError( - body=Error( - error=ErrorError( - code="forbidden", - message="Agent execution is stopped; cannot post events", - request_id="req-1", - ) - ) - ) - ) - for content in ("first", "later"): - with pytest.raises(asyncio.CancelledError): - await tools.send_event(content, "thought") - assert not ctx.is_stopped - assert not tools.turn.complete - mock_rest_client.agent_api_events.create_agent_chat_event.assert_awaited_once() - - async def test_send_event_without_an_execution_still_raises_the_typed_error( - self, mock_rest_client - ): - """Tools built without an execution context have no room state to - adopt: the typed error propagates so the caller sees the platform's - rejection.""" - mock_rest_client.agent_api_events.create_agent_chat_event.side_effect = ( - ForbiddenError( - body=Error( - error=ErrorError( - code="forbidden", - message="Agent execution is stopped; cannot post events", - request_id="req-1", - ) - ) - ) - ) - tools = AgentTools("room-123", mock_rest_client) - - with pytest.raises(RoomExecutionStoppedError): - await tools.send_event("thinking", "thought") - - -class TestAgentToolsSendFailure: - """Test send_failure's best-effort delegation over the real REST boundary.""" - - async def test_send_failure_posts_an_error_event(self, mock_rest_client): - tools = AgentTools("room-123", mock_rest_client) - - await tools.send_failure(AgentFailure("codex", "boom", "timeout")) - - call_args = mock_rest_client.agent_api_events.create_agent_chat_event.call_args - event = call_args.kwargs["event"] - assert event.message_type == "error" - assert event.metadata["failure"] == { - "provider": "codex", - "code": "timeout", - "message": "boom", - "detail": None, - } - - async def test_send_failure_swallows_a_rest_rejection(self, mock_rest_client): - """A failed report must resolve, not raise -- it runs inside a - caller's except block reporting a real provider failure already.""" - mock_rest_client.agent_api_events.create_agent_chat_event.side_effect = ( - RuntimeError("REST rejected the event") - ) - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.send_failure(AgentFailure("codex", "boom")) - - assert result == {"ok": False, "error": "REST rejected the event"} - - async def test_send_event_itself_still_raises_on_the_same_rejection( - self, mock_rest_client - ): - """The other half of the best-effort contract: send_event's own - raising behavior is unchanged by send_failure wrapping it.""" - mock_rest_client.agent_api_events.create_agent_chat_event.side_effect = ( - RuntimeError("REST rejected the event") - ) - tools = AgentTools("room-123", mock_rest_client) - - with pytest.raises(RuntimeError, match="REST rejected the event"): - await tools.send_event("task update", "task") - - -class TestMatchesIdentifier: - """Tests for the matches_identifier helper.""" - - def test_match_by_handle(self): - entity = {"handle": "alice", "name": "Alice Smith", "id": "u-1"} - assert matches_identifier(entity, "alice") is True - - def test_match_by_name(self): - entity = {"handle": "alice", "name": "Alice Smith", "id": "u-1"} - assert matches_identifier(entity, "Alice Smith") is True - - def test_match_by_id(self): - entity = {"handle": "alice", "name": "Alice Smith", "id": "u-1"} - assert matches_identifier(entity, "u-1") is True - - def test_case_insensitive(self): - entity = {"handle": "Alice", "name": "ALICE SMITH", "id": "U-1"} - assert matches_identifier(entity, "alice") is True - assert matches_identifier(entity, "alice smith") is True - assert matches_identifier(entity, "u-1") is True - - def test_no_match(self): - entity = {"handle": "alice", "name": "Alice Smith", "id": "u-1"} - assert matches_identifier(entity, "bob") is False - - def test_missing_fields(self): - """Should handle entities with missing or None fields.""" - assert matches_identifier({"name": "Alice"}, "Alice") is True - assert matches_identifier({"handle": None, "name": "Alice"}, "Alice") is True - assert matches_identifier({}, "anything") is False - - def test_at_prefix_normalization(self): - """@alice and alice should match regardless of which side has the prefix.""" - entity_with_at = {"handle": "@alice", "name": "Alice Smith", "id": "u-1"} - entity_without_at = {"handle": "alice", "name": "Alice Smith", "id": "u-1"} - - # identifier has @, entity doesn't - assert matches_identifier(entity_without_at, "@alice") is True - # entity has @, identifier doesn't - assert matches_identifier(entity_with_at, "alice") is True - # both have @ - assert matches_identifier(entity_with_at, "@alice") is True - # neither has @ - assert matches_identifier(entity_without_at, "alice") is True - - def test_empty_identifier(self): - """Empty string should only match empty field values.""" - entity = {"handle": "alice", "name": "Alice", "id": "u-1"} - assert matches_identifier(entity, "") is False - - -class TestAgentToolsAddParticipant: - """Test add_participant tool.""" - - async def test_add_participant_by_name(self, mock_rest_client): - """add_participant() should match by name and add via REST.""" - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.add_participant("Agent Two", role="member") - - assert result["id"] == "agent-2" - assert result["name"] == "Agent Two" - assert result["role"] == "member" - assert result["status"] == "added" - mock_rest_client.agent_api_participants.add_agent_chat_participant.assert_called_once() - - async def test_add_participant_by_handle(self, mock_rest_client): - """add_participant() should match by handle.""" - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.add_participant("agent-two", role="member") - - assert result["id"] == "agent-2" - assert result["name"] == "Agent Two" - assert result["status"] == "added" - - async def test_add_participant_by_id(self, mock_rest_client): - """add_participant() should match by ID.""" - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.add_participant("agent-2", role="member") - - assert result["id"] == "agent-2" - assert result["name"] == "Agent Two" - # Plain text, not the enum member: adapters render the result with str(). - assert "'status': 'added'" in str(result) - - async def test_add_participant_already_in_room_by_handle(self, mock_rest_client): - """add_participant() should detect already-in-room by handle.""" - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.add_participant("user-one", role="member") - - assert result["id"] == "user-1" - assert "'status': 'already_in_room'" in str(result) - mock_rest_client.agent_api_participants.add_agent_chat_participant.assert_not_called() - - async def test_add_participant_ambiguous_name_resolved_by_handle( - self, mock_rest_client - ): - """Two peers with the same display name — handle disambiguates (INT-287).""" - peer_a = MagicMock() - peer_a.id = "agent-a" - peer_a.name = "Weather Agent" - peer_a.type = "Agent" - peer_a.handle = "@alice/weather" - peer_a.description = "Alice's weather agent" - - peer_b = MagicMock() - peer_b.id = "agent-b" - peer_b.name = "Weather Agent" - peer_b.type = "Agent" - peer_b.handle = "@bob/weather" - peer_b.description = "Bob's weather agent" - - peers_response = MagicMock() - peers_response.data = [peer_a, peer_b] - peers_response.metadata = MagicMock() - peers_response.metadata.page = 1 - peers_response.metadata.page_size = 100 - peers_response.metadata.total_count = 2 - peers_response.metadata.total_pages = 1 - mock_rest_client.agent_api_peers.list_agent_peers = AsyncMock( - return_value=peers_response - ) - - tools = AgentTools("room-123", mock_rest_client) - - # Using handle should pick the correct one - result = await tools.add_participant("@bob/weather", role="member") - - assert result["id"] == "agent-b" - assert result["name"] == "Weather Agent" - assert result["status"] == "added" - - async def test_add_participant_not_found_raises(self, mock_rest_client): - """add_participant() should raise if peer not found.""" - # Return empty peers - mock_rest_client.agent_api_peers.list_agent_peers.return_value = MagicMock( - data=[], metadata=MagicMock(total_pages=1) - ) - tools = AgentTools("room-123", mock_rest_client) - - with pytest.raises(ValueError, match="Participant 'Unknown' not found"): - await tools.add_participant("Unknown") - - -class TestAgentToolsRemoveParticipant: - """Test remove_participant tool.""" - - async def test_remove_participant_by_name(self, mock_rest_client): - """remove_participant() should match by name and remove via REST.""" - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.remove_participant("User One") - - assert result["id"] == "user-1" - assert result["name"] == "User One" - assert result["status"] == "removed" - mock_rest_client.agent_api_participants.remove_agent_chat_participant.assert_called_once() - - async def test_remove_participant_by_handle(self, mock_rest_client): - """remove_participant() should match by handle.""" - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.remove_participant("user-one") - - assert result["id"] == "user-1" - assert result["name"] == "User One" - assert result["status"] == "removed" - - async def test_remove_participant_not_found_raises(self, mock_rest_client): - """remove_participant() should raise if not in room.""" - # Return empty participants - mock_rest_client.agent_api_participants.list_agent_chat_participants.return_value = MagicMock( - data=[] - ) - tools = AgentTools("room-123", mock_rest_client) - - with pytest.raises(ValueError, match="not found in this room"): - await tools.remove_participant("Unknown") - - -class TestAgentToolsLookupPeers: - """Test lookup_peers tool.""" - - async def test_lookup_peers_success(self, mock_rest_client): - """lookup_peers() should return the Fern response directly.""" - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.lookup_peers(page=1, page_size=50) - - # Now returns full Fern response with .data and .metadata - assert len(result.data) == 1 - assert result.data[0].name == "Agent Two" - assert result.metadata.page == 1 - - async def test_lookup_peers_filters_by_room(self, mock_rest_client): - """lookup_peers() should filter by not_in_chat.""" - tools = AgentTools("room-123", mock_rest_client) - - await tools.lookup_peers() - - call_args = mock_rest_client.agent_api_peers.list_agent_peers.call_args - assert call_args.kwargs["not_in_chat"] == "room-123" - - -class TestAgentToolsGetParticipants: - """Test get_participants tool.""" - - async def test_get_participants_success(self, mock_rest_client): - """get_participants() should return Fern participant models.""" - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.get_participants() - - assert len(result) == 1 - assert result[0].name == "User One" - - async def test_get_participants_empty(self, mock_rest_client): - """get_participants() should return empty list if none.""" - mock_rest_client.agent_api_participants.list_agent_chat_participants.return_value = MagicMock( - data=None - ) - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.get_participants() - - assert result == [] - - async def test_get_participants_updates_cache(self, mock_rest_client): - """get_participants() should refresh self._participants for mention resolution.""" - tools = AgentTools("room-123", mock_rest_client) - assert tools._participants == [] - - await tools.get_participants() - - assert tools._participants == [ - { - "id": "user-1", - "name": "User One", - "type": "User", - "handle": "user-one", - "description": None, - } - ] - - async def test_get_participants_preserves_cache_when_data_none( - self, mock_rest_client - ): - """``data is None`` indicates a transient/unexpected response — the - cache should be preserved rather than wiped (an agent should always - be a participant in its own room).""" - mock_rest_client.agent_api_participants.list_agent_chat_participants.return_value = MagicMock( - data=None - ) - cached = [ - {"id": "user-1", "name": "User One", "type": "User", "handle": "user-one"} - ] - tools = AgentTools("room-123", mock_rest_client, participants=cached) - - result = await tools.get_participants() - - assert result == [] - assert tools._participants == cached - - async def test_get_participants_clears_cache_on_empty_list(self, mock_rest_client): - """An explicit empty list from the server is authoritative — cache should clear.""" - mock_rest_client.agent_api_participants.list_agent_chat_participants.return_value = MagicMock( - data=[] - ) - stale = [{"id": "ghost", "name": "Ghost", "type": "User", "handle": "ghost"}] - tools = AgentTools("room-123", mock_rest_client, participants=stale) - - await tools.get_participants() - - assert tools._participants == [] - - async def test_get_participants_syncs_roster_to_ctx(self, mock_rest_client): - """get_participants() must make the ctx roster follow the REST list — - stale entries drop (even ctx-only ones this AgentTools never saw) and - new ones appear — so the refresh survives turn boundaries.""" - - ctx = ExecutionContext( - room_id="room-123", - link=MagicMock(rest=mock_rest_client), - on_execute=AsyncMock(), - ) - ctx.add_participant( - {"id": "user-1", "name": "User One", "type": "User", "handle": "user-one"} - ) - tools = AgentTools.from_context(ctx) - # A ctx-only participant that joined after tools were built, then left - # before the refresh — it must not survive as a ghost. - ctx.add_participant({"id": "user-3", "name": "User Three", "type": "User"}) - - # Server returns only a *new* participant — user-1 and user-3 are gone. - new_p = make_participant_mock("user-2", "User Two", "User", handle="user-two") - mock_rest_client.agent_api_participants.list_agent_chat_participants = ( - AsyncMock(return_value=MagicMock(data=[new_p])) - ) - - await tools.get_participants() - - assert [p["id"] for p in ctx.participants] == ["user-2"] - assert ctx.participants[0]["handle"] == "user-two" - - async def test_send_message_mentions_newly_discovered_participant( - self, mock_rest_client - ): - """Mentioning a participant first seen via get_participants() should not raise.""" - tools = AgentTools("room-123", mock_rest_client) - - await tools.get_participants() - await tools.send_message("Hi @user-one!", mentions=["user-one"]) - - call_args = ( - mock_rest_client.agent_api_messages.create_agent_chat_message.call_args - ) - message = call_args.kwargs["message"] - assert len(message.mentions) == 1 - assert message.mentions[0].id == "user-1" - assert message.mentions[0].handle == "user-one" - - -class TestAgentToolsCreateChatroom: - """Test create_chatroom tool.""" - - async def test_create_chatroom_success(self, mock_rest_client): - """create_chatroom() should call REST API and return room ID.""" - mock_response = Mock() - mock_response.data.id = "room-123" - mock_rest_client.agent_api_chats.create_agent_chat = AsyncMock( - return_value=mock_response - ) - - tools = AgentTools("room-456", mock_rest_client) - result = await tools.create_chatroom(task_id="task-789") - - assert result == "room-123" - mock_rest_client.agent_api_chats.create_agent_chat.assert_called_once() - - async def test_create_chatroom_without_task_id(self, mock_rest_client): - """create_chatroom() should work without task_id.""" - mock_response = Mock() - mock_response.data.id = "room-abc" - mock_rest_client.agent_api_chats.create_agent_chat = AsyncMock( - return_value=mock_response - ) - - tools = AgentTools("room-456", mock_rest_client) - result = await tools.create_chatroom() - - assert result == "room-abc" - - -class TestAgentToolsSchemas: - """Test tool schema generation.""" - - def test_tool_models_registry(self): - """TOOL_MODELS should contain all tool input models.""" - assert "band_send_message" in TOOL_MODELS - assert "band_send_event" in TOOL_MODELS - assert "band_add_participant" in TOOL_MODELS - assert "band_remove_participant" in TOOL_MODELS - assert "band_lookup_peers" in TOOL_MODELS - assert "band_get_participants" in TOOL_MODELS - assert "band_create_chatroom" in TOOL_MODELS - - def test_tool_models_property(self, mock_rest_client): - """tool_models property should return registry.""" - tools = AgentTools("room-123", mock_rest_client) - - assert tools.tool_models is TOOL_MODELS - - def test_get_tool_schemas_openai(self, mock_rest_client): - """get_tool_schemas('openai') should return OpenAI format (memory tools excluded by default).""" - tools = AgentTools("room-123", mock_rest_client) - - schemas = tools.get_tool_schemas("openai") - - tool_names = [s["function"]["name"] for s in schemas] - # Base platform tools - assert "band_send_message" in tool_names - assert "band_send_event" in tool_names - assert "band_add_participant" in tool_names - assert "band_remove_participant" in tool_names - assert "band_get_participants" in tool_names - assert "band_lookup_peers" in tool_names - assert "band_create_chatroom" in tool_names - # Contact tools included by default - assert "band_list_contacts" in tool_names - assert "band_add_contact" in tool_names - # Memory tools excluded by default - assert "band_list_memories" not in tool_names - assert "band_store_memory" not in tool_names - - send_msg = next( - s for s in schemas if s["function"]["name"] == "band_send_message" - ) - assert send_msg["type"] == "function" - assert "parameters" in send_msg["function"] - assert "description" in send_msg["function"] - - def test_get_tool_schemas_openai_with_memory(self, mock_rest_client): - """get_tool_schemas('openai', include_memory=True) should include memory tools.""" - tools = AgentTools("room-123", mock_rest_client) - - schemas = tools.get_tool_schemas( - "openai", capabilities=frozenset({Capability.MEMORY, Capability.CONTACTS}) - ) - - tool_names = [s["function"]["name"] for s in schemas] - # Memory tools present - assert "band_list_memories" in tool_names - assert "band_store_memory" in tool_names - assert "band_get_memory" in tool_names - assert "band_supersede_memory" in tool_names - assert "band_archive_memory" in tool_names - # Base and contact tools still present - assert "band_send_message" in tool_names - assert "band_list_contacts" in tool_names - - def test_get_tool_schemas_anthropic(self, mock_rest_client): - """get_tool_schemas('anthropic') should return Anthropic format (memory tools excluded by default).""" - tools = AgentTools("room-123", mock_rest_client) - - schemas = tools.get_tool_schemas("anthropic") - - tool_names = [s["name"] for s in schemas] - assert "band_send_message" in tool_names - assert "band_list_contacts" in tool_names - assert "band_list_memories" not in tool_names - - send_msg = next(s for s in schemas if s["name"] == "band_send_message") - assert "input_schema" in send_msg - assert "description" in send_msg - - def test_get_tool_schemas_anthropic_with_memory(self, mock_rest_client): - """get_tool_schemas('anthropic', include_memory=True) should include memory tools.""" - tools = AgentTools("room-123", mock_rest_client) - - schemas = tools.get_tool_schemas( - "anthropic", - capabilities=frozenset({Capability.MEMORY, Capability.CONTACTS}), - ) - - tool_names = [s["name"] for s in schemas] - assert "band_list_memories" in tool_names - assert "band_store_memory" in tool_names - assert "band_send_message" in tool_names - assert "band_list_contacts" in tool_names - - def test_schemas_drop_numeric_bounds(self, mock_rest_client): - """Pydantic Field(ge=.., le=..) renders minimum/maximum, which some - providers reject on integer params; the schemas must omit them while the - models still enforce the bounds at execution.""" - tools = AgentTools("room-123", mock_rest_client) - - schemas = tools.get_tool_schemas( - "openai", capabilities=frozenset({Capability.MEMORY, Capability.CONTACTS}) - ) - - page_size = next( - s["function"]["parameters"]["properties"]["page_size"] - for s in schemas - if s["function"]["name"] == "band_lookup_peers" - ) - assert "minimum" not in page_size - assert "maximum" not in page_size - assert page_size["type"] == "integer" - - -class TestAgentToolsExecuteToolCall: - """Test execute_tool_call dispatch.""" - - async def test_execute_send_message(self, mock_rest_client, participants): - """execute_tool_call() should dispatch band_send_message.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - result = await tools.execute_tool_call( - "band_send_message", {"content": "Hello!", "mentions": ["User One"]} - ) - - assert result["id"] == "msg-123" - - async def test_execute_send_event(self, mock_rest_client): - """execute_tool_call() should dispatch band_send_event.""" - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.execute_tool_call( - "band_send_event", {"content": "Thinking...", "message_type": "thought"} - ) - - assert result["message_type"] == "thought" - - async def test_execute_lookup_peers(self, mock_rest_client): - """execute_tool_call() should dispatch band_lookup_peers.""" - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.execute_tool_call("band_lookup_peers", {"page": 1}) - - # execute_tool_call calls .model_dump() on the Fern response - assert isinstance(result, dict) - - async def test_execute_get_participants(self, mock_rest_client): - """execute_tool_call() should dispatch band_get_participants.""" - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.execute_tool_call("band_get_participants", {}) - - assert isinstance(result, list) - assert result[0]["id"] == "user-1" - assert result[0]["name"] == "User One" - - async def test_execute_unknown_tool(self, mock_rest_client): - """execute_tool_call() should return error for unknown tool.""" - tools = AgentTools("room-123", mock_rest_client) - - result = await tools.execute_tool_call("unknown_tool", {}) - - assert "Unknown tool" in result - - async def test_execute_validation_error(self, mock_rest_client): - """execute_tool_call() should return validation error in LLM-friendly format.""" - tools = AgentTools("room-123", mock_rest_client) - - # Missing required field - result = await tools.execute_tool_call( - "band_send_message", {"content": "Hello"} - ) - - assert "Invalid arguments for band_send_message" in result - assert "mentions" in result # Should mention the missing field - - async def test_execute_runtime_error(self, mock_rest_client, participants): - """execute_tool_call() should return execution error.""" - mock_rest_client.agent_api_messages.create_agent_chat_message.side_effect = ( - Exception("Network error") - ) - tools = AgentTools("room-123", mock_rest_client, participants) - - result = await tools.execute_tool_call( - "band_send_message", {"content": "Hello!", "mentions": ["User One"]} - ) - - assert "Error executing" in result - - -class TestFormatToolValidationError: - """Pins the exact wire-message shape `format_tool_validation_error` produces. - - The published band-mcp CLI surfaces this string verbatim to its callers - (via ``StandaloneResolver``), so the separator between multiple field - errors and the dotted-path format for a nested field are part of the - wire contract, not just an internal formatting detail free to drift. - """ - - def test_multiple_and_nested_field_errors_joined_with_comma(self): - class Nested(BaseModel): - name: str - - class Model(BaseModel): - items: list[Nested] - extra: str - - with pytest.raises(ValidationError) as exc_info: - Model(items=[{}], extra=None) - - message = format_tool_validation_error("some_tool", exc_info.value) - - assert message == ( - "Invalid arguments for some_tool: " - "items.0.name: Field required, " - "extra: Input should be a valid string" - ) - - -class TestEmptyMentionsValidation: - """Test that empty mentions return a helpful error with participant names.""" - - async def test_raises_error_with_participant_names( - self, mock_rest_client, participants - ): - """Should raise BandToolError listing available participants when mentions empty.""" - - tools = AgentTools("room-123", mock_rest_client, participants) - - with pytest.raises( - BandToolError, match="At least one mention is required" - ) as exc_info: - await tools.send_message("Hello!", mentions=[]) - - assert "@user-one" in str(exc_info.value) - assert "@user-two" in str(exc_info.value) - # Should NOT have called the API - mock_rest_client.agent_api_messages.create_agent_chat_message.assert_not_called() - - async def test_raises_error_when_mentions_none( - self, mock_rest_client, participants - ): - """Should raise BandToolError when mentions is None.""" - - tools = AgentTools("room-123", mock_rest_client, participants) - - with pytest.raises(BandToolError, match="At least one mention is required"): - await tools.send_message("Hello!", mentions=None) - - async def test_uses_handle_when_available(self, mock_rest_client): - """Should prefer handle over name in error message.""" - - participants = [ - {"id": "user-1", "name": "User One", "type": "User", "handle": "@user-one"}, - ] - tools = AgentTools("room-123", mock_rest_client, participants) - - with pytest.raises(BandToolError, match="@user-one"): - await tools.send_message("Hello!", mentions=[]) - - async def test_omits_participant_without_handle(self, mock_rest_client): - """Should omit handle-less participants — they can't be @mentioned.""" - - participants = [ - {"id": "user-1", "name": "User One", "type": "User"}, - ] - tools = AgentTools("room-123", mock_rest_client, participants) - - with pytest.raises( - BandToolError, match="At least one mention is required" - ) as exc_info: - await tools.send_message("Hello!", mentions=[]) - - # Name must not be offered as a mention target — only real handles are. - assert "User One" not in str(exc_info.value) - # With no mentionable handles there is nothing to suggest, so the error - # carries no handle list rather than an empty one. - assert "Available handles:" not in str(exc_info.value) - - async def test_no_error_when_mentions_provided( - self, mock_rest_client, participants - ): - """Should proceed normally when mentions are provided.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - result = await tools.send_message("Hello!", mentions=["User One"]) - - # Now returns Fern model; verify it has the expected attribute - assert result.model_dump()["id"] == "msg-123" - mock_rest_client.agent_api_messages.create_agent_chat_message.assert_called_once() - - async def test_execute_tool_call_raises_band_tool_error( - self, mock_rest_client, participants - ): - """execute_tool_call lets BandToolError propagate for wrapper translation.""" - - tools = AgentTools("room-123", mock_rest_client, participants) - - with pytest.raises(BandToolError, match="At least one mention is required"): - await tools.execute_tool_call( - "band_send_message", {"content": "Hello!", "mentions": []} - ) - - -class TestMentionResolution: - """Test mention resolution logic.""" - - def test_resolve_string_mentions(self, mock_rest_client, participants): - """Should resolve string mentions to dicts with id and handle.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - resolved = tools._resolve_mentions(["User One", "User Two"]) - - assert len(resolved) == 2 - assert resolved[0] == {"id": "user-1", "handle": "@user-one"} - assert resolved[1] == {"id": "user-2", "handle": "@user-two"} - - def test_resolve_dict_mentions_with_id(self, mock_rest_client, participants): - """Should pass through dict mentions with ID and handle.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - resolved = tools._resolve_mentions([{"id": "custom-id", "handle": "@custom"}]) - - assert resolved[0] == {"id": "custom-id", "handle": "@custom"} - - def test_resolve_dict_mentions_without_id(self, mock_rest_client, participants): - """Should resolve dict mentions without ID by name lookup.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - resolved = tools._resolve_mentions([{"name": "User One"}]) - - assert resolved[0] == {"id": "user-1", "handle": "@user-one"} - - def test_resolve_unknown_raises(self, mock_rest_client, participants): - """Should raise for unknown mention.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - with pytest.raises(ValueError, match="Unknown participant"): - tools._resolve_mentions(["Unknown Person"]) - - -class TestHandleMentionResolution: - """Test handle-based mention resolution.""" - - def test_available_mention_handles_excludes_self_and_missing_handles(self): - """Available handle hints should include only mentionable room handles.""" - participants = [ - {"id": "user-1", "name": "User One", "handle": "@user-one"}, - {"id": "self", "name": "Self", "handle": "@self"}, - {"id": "user-3", "name": "No Handle", "handle": None}, - ] - - assert available_mention_handles(participants, agent_id="self") == ["@user-one"] - - def test_append_mention_handles_hint_is_idempotent(self): - """An error already carrying the hint is returned unchanged, so the same - error can pass through multiple adapter enrichers without doubling.""" - enriched = append_mention_handles_hint( - "At least one mention is required", ["@alice"] - ) - assert enriched.count("Available handles:") == 1 - - twice = append_mention_handles_hint(enriched, ["@alice"]) - assert twice == enriched - - def test_append_mention_handles_hint_no_handles_is_noop(self): - """With no mentionable handles there is nothing to suggest.""" - error = "At least one mention is required" - assert append_mention_handles_hint(error, []) == error - - def test_resolve_by_handle(self, mock_rest_client, participants): - """Should resolve mentions by handle.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - resolved = tools._resolve_mentions(["@user-one"]) - - assert len(resolved) == 1 - assert resolved[0] == {"id": "user-1", "handle": "@user-one"} - - def test_resolve_handle_takes_priority(self, mock_rest_client): - """Should try handle lookup before name lookup.""" - # Participant with handle different from name - participants = [ - { - "id": "agent-1", - "name": "Weather Agent", - "type": "Agent", - "handle": "@john/weather", - }, - ] - tools = AgentTools("room-123", mock_rest_client, participants) - - # Resolve by handle - resolved = tools._resolve_mentions(["@john/weather"]) - assert resolved[0] == {"id": "agent-1", "handle": "@john/weather"} - - # Resolve by name still works - resolved = tools._resolve_mentions(["Weather Agent"]) - assert resolved[0] == {"id": "agent-1", "handle": "@john/weather"} - - def test_resolve_mixed_handles_and_names(self, mock_rest_client, participants): - """Should resolve a mix of handles and names.""" - tools = AgentTools("room-123", mock_rest_client, participants) - - resolved = tools._resolve_mentions(["@user-one", "User Two"]) - - assert len(resolved) == 2 - assert resolved[0] == {"id": "user-1", "handle": "@user-one"} - assert resolved[1] == {"id": "user-2", "handle": "@user-two"} - - def test_resolve_unknown_handle_raises(self, mock_rest_client, participants): - """Should raise for unknown handle.""" - tools = AgentTools( - "room-123", mock_rest_client, participants, agent_id="user-2" - ) - - # @ prefix is stripped during normalization - with pytest.raises(ValueError, match="Unknown participant 'unknown'") as exc: - tools._resolve_mentions(["@unknown"]) - - message = str(exc.value) - assert "@user-one" in message - assert "@user-two" not in message - - def test_resolve_participant_without_handle(self, mock_rest_client): - """Should resolve by name when participant has no handle.""" - participants = [ - {"id": "user-1", "name": "User One", "type": "User", "handle": None}, - ] - tools = AgentTools("room-123", mock_rest_client, participants) - - resolved = tools._resolve_mentions(["User One"]) - - assert resolved[0] == {"id": "user-1", "handle": None} - - -class TestToolInputModels: - """Test Pydantic tool input models.""" - - def test_send_message_input_validation(self): - """SendMessageInput should validate fields.""" - model = SendMessageInput(content="Hello", mentions=["User"]) - assert model.content == "Hello" - assert model.mentions == ["User"] - - def test_send_message_input_accepts_empty_mentions(self): - """SendMessageInput allows empty mentions (runtime validates instead).""" - model = SendMessageInput(content="Hello", mentions=[]) - assert model.mentions == [] - - def test_send_room_file_input_requires_mentions(self): - """band_send_room_file posts a message to attach the file, so its - schema must require mentions just like SendMessageInput -- an - optional-looking field here would let the LLM omit it, upload the - file, and only then discover the platform's mention requirement.""" - with pytest.raises(ValidationError): - SendRoomFileInput(content="hi", filename="f.txt") - - def test_send_room_file_input_accepts_empty_mentions(self): - """Present but empty is still valid at the schema level (runtime - validates the "at least one" rule instead, same as SendMessageInput).""" - model = SendRoomFileInput(content="hi", filename="f.txt", mentions=[]) - assert model.mentions == [] - - def test_send_event_input_validation(self): - """SendEventInput should validate fields.""" - model = SendEventInput(content="Thinking", message_type="thought") - assert model.message_type == "thought" - - def test_send_event_input_validates_type(self): - """SendEventInput should validate message_type.""" - with pytest.raises(ValidationError, match="literal_error"): - SendEventInput(content="Test", message_type="invalid") - - def test_add_participant_input_defaults(self): - """AddParticipantInput should have default role.""" - model = AddParticipantInput(identifier="User") - assert model.role == "member" - - def test_add_participant_input_accepts_legacy_name_field(self): - """AddParticipantInput should accept 'name' as alias for backward compat.""" - model = AddParticipantInput.model_validate({"name": "Agent Two"}) - assert model.identifier == "Agent Two" - - def test_remove_participant_input_accepts_legacy_name_field(self): - """RemoveParticipantInput should accept 'name' as alias for backward compat.""" - - model = RemoveParticipantInput.model_validate({"name": "User One"}) - assert model.identifier == "User One" - - def test_lookup_peers_input_defaults(self): - """LookupPeersInput should have defaults.""" - model = LookupPeersInput() - assert model.page == 1 - assert model.page_size == 50 - - def test_get_participants_input_no_fields(self): - """GetParticipantsInput should have no required fields.""" - model = GetParticipantsInput() - assert model is not None - - def test_create_chatroom_input_validation(self): - """CreateChatroomInput should allow optional task_id.""" - model = CreateChatroomInput(task_id="task-123") - assert model.task_id == "task-123" - - def test_create_chatroom_input_no_task_id(self): - """CreateChatroomInput should work without task_id.""" - model = CreateChatroomInput() - assert model.task_id is None - - -class TestReplyEffectSpellings: - """Which tool names resolve to Band's reply effect.""" - - def test_sdk_injected_tool(self): - assert turn_effect("band_send_message") is TurnEffect.REPLY - - def test_standalone_band_mcp_tool(self): - assert turn_effect("create_agent_chat_message") is TurnEffect.REPLY - - def test_mcp_server_prefixed_names(self): - """MCP clients may prefix the server name onto the tool name.""" - assert turn_effect("band-band_send_message") is TurnEffect.REPLY - assert turn_effect("band-create_agent_chat_message") is TurnEffect.REPLY - - def test_non_settling_tools(self): - assert turn_effect("band_send_event") is TurnEffect.OBSERVE - assert turn_effect("band_lookup_peers") is TurnEffect.OBSERVE - assert turn_effect("get_weather") is TurnEffect.OBSERVE - - def test_no_substring_false_positive(self): - """Only an exact or server-prefixed match counts, not any substring.""" - assert turn_effect("band_send_message_draft") is TurnEffect.OBSERVE - - def test_non_band_server_prefix_does_not_resolve(self): - """An unrelated MCP server's own tool must never be treated as a Band - reply just because it ends in ``-band_send_message``.""" - assert turn_effect("other-band_send_message") is TurnEffect.OBSERVE - - -class TestCanonicalizeMcpToolName: - """Recovering the canonical band name from an MCP ``-`` spelling.""" - - OWN = frozenset({"band_send_message", "echo"}) - - def test_strips_server_prefix_off_own_tool(self): - name = canonicalize_mcp_tool_name("band-band_send_message", self.OWN) - assert name == "band_send_message" - - def test_custom_tool_prefix_stripped(self): - assert canonicalize_mcp_tool_name("band-echo", self.OWN) == "echo" - - def test_non_band_server_prefix_passes_through(self): - """Only the Band MCP server's own ``band-`` prefix resolves -- an - unrelated server's tool that happens to end in one of our names must - not be misattributed to us (e.g. narrated/suppressed as our own).""" - name = canonicalize_mcp_tool_name("platform-band_send_message", self.OWN) - assert name == "platform-band_send_message" - - def test_foreign_tool_passes_through(self): - """A prefixed name that reveals none of ours stays as reported.""" - assert canonicalize_mcp_tool_name("band-grep", self.OWN) == "band-grep" - - def test_unprefixed_name_passes_through(self): - assert ( - canonicalize_mcp_tool_name("band_send_message", self.OWN) - == "band_send_message" - ) - - -class TestFetchRoomContext: - """Paging the agent context endpoint.""" - - @pytest.mark.asyncio - async def test_prefers_the_required_pagination_field(self, mock_rest_client): - """`metadata` is required; `meta` is optional and often absent. - - Reading only `meta` collapses every room to one synthesized page, so a - caller paging through history silently stops at the first page. - """ - pagination = MagicMock() - pagination.model_dump.return_value = {"total_pages": 3} - mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( - return_value=SimpleNamespace(data=[], metadata=pagination) - ) - - context = await AgentTools("room-123", mock_rest_client).fetch_room_context( - room_id="room-123" - ) - - assert context["meta"]["total_pages"] == 3 - - -@pytest.mark.parametrize( - "name", ["band_get_memory", "band_supersede_memory", "band_archive_memory"] -) -@pytest.mark.parametrize("memory_id", INVALID_IDS) -@pytest.mark.asyncio -async def test_memory_validation_prevents_http(name: str, memory_id: str) -> None: - async with memory_client() as (rest, requests): - outcome = await AgentTools("room-1", rest).execute_tool_call_structured( - name, {"memory_id": memory_id} - ) - assert not outcome.ok - assert f"Invalid arguments for {name}" in outcome.error_message - assert "memory_id:" in outcome.error_message - assert requests == [] - - -@pytest.mark.parametrize( - "name,method,suffix", - [ - ("band_get_memory", "GET", ""), - ("band_supersede_memory", "POST", "/supersede"), - ("band_archive_memory", "POST", "/archive"), - ], -) -@pytest.mark.asyncio -async def test_valid_memory_dispatch_addresses_item( - name: str, method: str, suffix: str -) -> None: - async with memory_client() as (rest, requests): - outcome = await AgentTools("room-1", rest).execute_tool_call_structured( - name, {"memory_id": UUID_ID} - ) - assert outcome.ok, outcome.error_message - assert outcome.value["id"] == UUID_ID - assert outcome.value["content"] == "remember this" - assert [(q.method, q.url.path) for q in requests] == [ - (method, f"/api/v1/agent/memories/{UUID_ID}{suffix}") - ] - - -@pytest.mark.parametrize("boundary", ["dispatcher", "failure"]) -async def test_unbound_stopped_refusal_escapes_tool_error_conversion( - mock_rest_client, boundary: str -) -> None: - tools = AgentTools("room-123", mock_rest_client) - mock_rest_client.agent_api_events.create_agent_chat_event.side_effect = ( - ForbiddenError( - body=Error( - error=ErrorError( - code="forbidden", - message="Agent execution is stopped; cannot post events", - request_id="req-1", - ) - ) - ) - ) - with pytest.raises(RoomExecutionStoppedError): - if boundary == "dispatcher": - await tools.execute_tool_call_structured( - "band_send_event", {"content": "thought", "message_type": "thought"} - ) - else: - await tools.send_failure(AgentFailure("provider", "failed")) - assert not tools.turn.complete diff --git a/tests/runtime/toolbehavior/__init__.py b/tests/runtime/toolbehavior/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/runtime/toolbehavior/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/runtime/toolbehavior/conftest.py b/tests/runtime/toolbehavior/conftest.py new file mode 100644 index 000000000..4fa9abc4d --- /dev/null +++ b/tests/runtime/toolbehavior/conftest.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +import pytest + + +@pytest.fixture +def participants(): + """Sample participants list.""" + return [ + {"id": "user-1", "name": "User One", "type": "User", "handle": "@user-one"}, + {"id": "user-2", "name": "User Two", "type": "User", "handle": "@user-two"}, + ] diff --git a/tests/runtime/toolbehavior/kit.py b/tests/runtime/toolbehavior/kit.py new file mode 100644 index 000000000..816f96115 --- /dev/null +++ b/tests/runtime/toolbehavior/kit.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from datetime import datetime +from unittest.mock import AsyncMock, MagicMock + +from band_rest import ( + ChatMessage, + GetAgentChatContextResponse, + GetAgentChatContextResponseMetadata, +) + +from band.client.rest import ( + Attachment, + ChatMessageRequest, + NotFoundError, +) + + +def _attachment( + file_id: str = "file-1", + *, + name: str = "notes.txt", + content_type: str = "text/plain", + size: int = 20, + expires_at: datetime | None = None, +) -> Attachment: + return Attachment( + id=file_id, + name=name, + content_type=content_type, + bytes=size, + sha256="a" * 64, + has_thumb=False, + expires_at=expires_at, + ) + + +def _context_response( + messages: list[ChatMessage], + *, + next_cursor: str | None = None, + has_more: bool = False, +) -> GetAgentChatContextResponse: + return GetAgentChatContextResponse( + data=messages, + metadata=GetAgentChatContextResponseMetadata( + has_more=has_more, limit=50, next_cursor=next_cursor + ), + ) + + +def _message_with_attachments( + msg_id: str, attachments: list[Attachment] +) -> ChatMessage: + return ChatMessage( + id=msg_id, + content="", + sender_id="user-1", + sender_type="User", + message_type="text", + attachments=attachments, + ) + + +def _mock_attachment_page(mock_rest_client: MagicMock, attachment: Attachment) -> None: + """Configure the context endpoint to return one message carrying `attachment` -- + the shared setup for every read_room_file test that only cares about one file.""" + mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( + return_value=_context_response( + [_message_with_attachments("msg-1", [attachment])] + ) + ) + + +async def _fake_download(body: bytes): + """A ``download_agent_chat_file`` double: an async generator, not a + coroutine -- assigning an ``AsyncMock`` here would make the *call* itself + awaitable, which nothing in the real client does.""" + yield body + + +async def _fake_download_not_found(): + """Same shape as ``_fake_download``, but raises before ever yielding.""" + if False: + yield b"" + raise NotFoundError(body=MagicMock()) + + +def _posted_message(mock_rest_client: MagicMock) -> ChatMessageRequest: + """The ``ChatMessageRequest`` a preceding ``send_message``/``send_room_file`` + call actually posted -- the one observable outcome a REST-backed + ``AgentTools`` test has for "what did the message body look like".""" + return ( + mock_rest_client.agent_api_messages.create_agent_chat_message.await_args.kwargs[ + "message" + ] + ) diff --git a/tests/runtime/toolbehavior/test_agenttoolsaddparticipant.py b/tests/runtime/toolbehavior/test_agenttoolsaddparticipant.py new file mode 100644 index 000000000..a718bbf7b --- /dev/null +++ b/tests/runtime/toolbehavior/test_agenttoolsaddparticipant.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from band.runtime.tools import ( + AgentTools, +) + + +class TestAgentToolsAddParticipant: + """Test add_participant tool.""" + + async def test_add_participant_by_name(self, mock_rest_client): + """add_participant() should match by name and add via REST.""" + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.add_participant("Agent Two", role="member") + + assert result["id"] == "agent-2" + assert result["name"] == "Agent Two" + assert result["role"] == "member" + assert result["status"] == "added" + mock_rest_client.agent_api_participants.add_agent_chat_participant.assert_called_once() + + async def test_add_participant_by_handle(self, mock_rest_client): + """add_participant() should match by handle.""" + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.add_participant("agent-two", role="member") + + assert result["id"] == "agent-2" + assert result["name"] == "Agent Two" + assert result["status"] == "added" + + async def test_add_participant_by_id(self, mock_rest_client): + """add_participant() should match by ID.""" + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.add_participant("agent-2", role="member") + + assert result["id"] == "agent-2" + assert result["name"] == "Agent Two" + # Plain text, not the enum member: adapters render the result with str(). + assert "'status': 'added'" in str(result) + + async def test_add_participant_already_in_room_by_handle(self, mock_rest_client): + """add_participant() should detect already-in-room by handle.""" + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.add_participant("user-one", role="member") + + assert result["id"] == "user-1" + assert "'status': 'already_in_room'" in str(result) + mock_rest_client.agent_api_participants.add_agent_chat_participant.assert_not_called() + + async def test_add_participant_ambiguous_name_resolved_by_handle( + self, mock_rest_client + ): + """Two peers with the same display name — handle disambiguates (INT-287).""" + peer_a = MagicMock() + peer_a.id = "agent-a" + peer_a.name = "Weather Agent" + peer_a.type = "Agent" + peer_a.handle = "@alice/weather" + peer_a.description = "Alice's weather agent" + + peer_b = MagicMock() + peer_b.id = "agent-b" + peer_b.name = "Weather Agent" + peer_b.type = "Agent" + peer_b.handle = "@bob/weather" + peer_b.description = "Bob's weather agent" + + peers_response = MagicMock() + peers_response.data = [peer_a, peer_b] + peers_response.metadata = MagicMock() + peers_response.metadata.page = 1 + peers_response.metadata.page_size = 100 + peers_response.metadata.total_count = 2 + peers_response.metadata.total_pages = 1 + mock_rest_client.agent_api_peers.list_agent_peers = AsyncMock( + return_value=peers_response + ) + + tools = AgentTools("room-123", mock_rest_client) + + # Using handle should pick the correct one + result = await tools.add_participant("@bob/weather", role="member") + + assert result["id"] == "agent-b" + assert result["name"] == "Weather Agent" + assert result["status"] == "added" + + async def test_add_participant_not_found_raises(self, mock_rest_client): + """add_participant() should raise if peer not found.""" + # Return empty peers + mock_rest_client.agent_api_peers.list_agent_peers.return_value = MagicMock( + data=[], metadata=MagicMock(total_pages=1) + ) + tools = AgentTools("room-123", mock_rest_client) + + with pytest.raises(ValueError, match="Participant 'Unknown' not found"): + await tools.add_participant("Unknown") diff --git a/tests/runtime/toolbehavior/test_agenttoolsconstruction.py b/tests/runtime/toolbehavior/test_agenttoolsconstruction.py new file mode 100644 index 000000000..081338370 --- /dev/null +++ b/tests/runtime/toolbehavior/test_agenttoolsconstruction.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from band.runtime.tools import ( + AgentTools, +) + + +class TestAgentToolsConstruction: + """Test AgentTools initialization.""" + + def test_init_stores_room_id(self, mock_rest_client): + """Should store room_id.""" + tools = AgentTools("room-123", mock_rest_client) + + assert tools.room_id == "room-123" + + def test_init_stores_rest_client(self, mock_rest_client): + """Should store REST client.""" + tools = AgentTools("room-123", mock_rest_client) + + assert tools.rest is mock_rest_client + + def test_init_empty_participants_by_default(self, mock_rest_client): + """Should have empty participants by default.""" + tools = AgentTools("room-123", mock_rest_client) + + assert tools._participants == [] + + def test_init_with_participants(self, mock_rest_client, participants): + """Should accept participants.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + assert tools._participants == participants diff --git a/tests/runtime/toolbehavior/test_agenttoolscontextsyncback.py b/tests/runtime/toolbehavior/test_agenttoolscontextsyncback.py new file mode 100644 index 000000000..39b0f003c --- /dev/null +++ b/tests/runtime/toolbehavior/test_agenttoolscontextsyncback.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from band.runtime.execution import ExecutionContext +from band.runtime.tools import ( + AgentTools, +) +from tests.conftest import make_participant_mock + + +class TestAgentToolsContextSyncBack: + """Participant changes persist across tools built from the same context.""" + + @pytest.mark.asyncio + async def test_add_participant_persists_across_recreated_tools( + self, mock_rest_client + ): + """Added participant must survive tools recreation via from_context(). + + Uses real ExecutionContext — its ``participants`` property returns a + copy, so without the _ctx backref the mutation would be lost. + """ + + ctx = ExecutionContext( + room_id="room-789", + link=MagicMock(rest=mock_rest_client), + on_execute=AsyncMock(), + ) + + # Empty room + mock_rest_client.agent_api_participants.list_agent_chat_participants = ( + AsyncMock(return_value=MagicMock(data=[])) + ) + + # Turn 1: add participant + tools1 = AgentTools.from_context(ctx) + await tools1.add_participant("Agent Two") + + added = ctx.participants[0] + assert len(ctx.participants) == 1 + assert { + key: added[key] for key in ("id", "name", "type", "handle", "description") + } == { + "id": "agent-2", + "name": "Agent Two", + "type": "Agent", + "handle": "agent-two", + "description": "Another agent", + } + + # Turn 2: recreate tools — participant must still be there + tools2 = AgentTools.from_context(ctx) + assert len(tools2._participants) == 1 + assert tools2._participants[0]["id"] == "agent-2" + + @pytest.mark.asyncio + async def test_remove_participant_persists_across_recreated_tools( + self, mock_rest_client + ): + """Removed participant must stay removed after tools recreation. + + Uses real ExecutionContext — its ``participants`` property returns a + copy, so without the _ctx backref the removal would be lost. + """ + + participant = { + "id": "user-1", + "name": "User One", + "type": "User", + "handle": "user-one", + } + + ctx = ExecutionContext( + room_id="room-789", + link=MagicMock(rest=mock_rest_client), + on_execute=AsyncMock(), + ) + ctx.set_participants([participant]) + + # REST snapshot must match ctx.participants + p_mock = make_participant_mock("user-1", "User One", "User", handle="user-one") + mock_rest_client.agent_api_participants.list_agent_chat_participants = ( + AsyncMock(return_value=MagicMock(data=[p_mock])) + ) + + # Turn 1: remove participant + tools1 = AgentTools.from_context(ctx) + await tools1.remove_participant("User One") + + assert len(ctx.participants) == 0 + + # Turn 2: recreate tools — participant must stay removed + tools2 = AgentTools.from_context(ctx) + assert len(tools2._participants) == 0 diff --git a/tests/runtime/toolbehavior/test_agenttoolscreatechatroom.py b/tests/runtime/toolbehavior/test_agenttoolscreatechatroom.py new file mode 100644 index 000000000..1ca7a43b9 --- /dev/null +++ b/tests/runtime/toolbehavior/test_agenttoolscreatechatroom.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock, Mock + +from band.runtime.tools import ( + AgentTools, +) + + +class TestAgentToolsCreateChatroom: + """Test create_chatroom tool.""" + + async def test_create_chatroom_success(self, mock_rest_client): + """create_chatroom() should call REST API and return room ID.""" + mock_response = Mock() + mock_response.data.id = "room-123" + mock_rest_client.agent_api_chats.create_agent_chat = AsyncMock( + return_value=mock_response + ) + + tools = AgentTools("room-456", mock_rest_client) + result = await tools.create_chatroom(task_id="task-789") + + assert result == "room-123" + mock_rest_client.agent_api_chats.create_agent_chat.assert_called_once() + + async def test_create_chatroom_without_task_id(self, mock_rest_client): + """create_chatroom() should work without task_id.""" + mock_response = Mock() + mock_response.data.id = "room-abc" + mock_rest_client.agent_api_chats.create_agent_chat = AsyncMock( + return_value=mock_response + ) + + tools = AgentTools("room-456", mock_rest_client) + result = await tools.create_chatroom() + + assert result == "room-abc" diff --git a/tests/runtime/toolbehavior/test_agenttoolsexecutetoolcall.py b/tests/runtime/toolbehavior/test_agenttoolsexecutetoolcall.py new file mode 100644 index 000000000..e9346d3e1 --- /dev/null +++ b/tests/runtime/toolbehavior/test_agenttoolsexecutetoolcall.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from band.runtime.tools import ( + AgentTools, +) + + +class TestAgentToolsExecuteToolCall: + """Test execute_tool_call dispatch.""" + + async def test_execute_send_message(self, mock_rest_client, participants): + """execute_tool_call() should dispatch band_send_message.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + result = await tools.execute_tool_call( + "band_send_message", {"content": "Hello!", "mentions": ["User One"]} + ) + + assert result["id"] == "msg-123" + + async def test_execute_send_event(self, mock_rest_client): + """execute_tool_call() should dispatch band_send_event.""" + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.execute_tool_call( + "band_send_event", {"content": "Thinking...", "message_type": "thought"} + ) + + assert result["message_type"] == "thought" + + async def test_execute_lookup_peers(self, mock_rest_client): + """execute_tool_call() should dispatch band_lookup_peers.""" + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.execute_tool_call("band_lookup_peers", {"page": 1}) + + # execute_tool_call calls .model_dump() on the Fern response + assert isinstance(result, dict) + + async def test_execute_get_participants(self, mock_rest_client): + """execute_tool_call() should dispatch band_get_participants.""" + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.execute_tool_call("band_get_participants", {}) + + assert isinstance(result, list) + assert result[0]["id"] == "user-1" + assert result[0]["name"] == "User One" + + async def test_execute_unknown_tool(self, mock_rest_client): + """execute_tool_call() should return error for unknown tool.""" + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.execute_tool_call("unknown_tool", {}) + + assert "Unknown tool" in result + + async def test_execute_validation_error(self, mock_rest_client): + """execute_tool_call() should return validation error in LLM-friendly format.""" + tools = AgentTools("room-123", mock_rest_client) + + # Missing required field + result = await tools.execute_tool_call( + "band_send_message", {"content": "Hello"} + ) + + assert "Invalid arguments for band_send_message" in result + assert "mentions" in result # Should mention the missing field + + async def test_execute_runtime_error(self, mock_rest_client, participants): + """execute_tool_call() should return execution error.""" + mock_rest_client.agent_api_messages.create_agent_chat_message.side_effect = ( + Exception("Network error") + ) + tools = AgentTools("room-123", mock_rest_client, participants) + + result = await tools.execute_tool_call( + "band_send_message", {"content": "Hello!", "mentions": ["User One"]} + ) + + assert "Error executing" in result diff --git a/tests/runtime/toolbehavior/test_agenttoolsfromcontext.py b/tests/runtime/toolbehavior/test_agenttoolsfromcontext.py new file mode 100644 index 000000000..505caa10f --- /dev/null +++ b/tests/runtime/toolbehavior/test_agenttoolsfromcontext.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from unittest.mock import MagicMock + +from band.runtime.tools import ( + AgentTools, +) + + +class TestAgentToolsFromContext: + """Test AgentTools.from_context() factory method.""" + + def test_from_context_creates_tools(self, mock_rest_client, participants): + """from_context() should create AgentTools from ExecutionContext.""" + # Mock ExecutionContext + mock_ctx = MagicMock() + mock_ctx.room_id = "room-456" + mock_ctx.link = MagicMock() + mock_ctx.link.rest = mock_rest_client + mock_ctx.participants = participants + + tools = AgentTools.from_context(mock_ctx) + + assert tools.room_id == "room-456" + assert tools.rest is mock_rest_client + assert tools._participants == participants diff --git a/tests/runtime/toolbehavior/test_agenttoolsgetparticipants.py b/tests/runtime/toolbehavior/test_agenttoolsgetparticipants.py new file mode 100644 index 000000000..a99760ed0 --- /dev/null +++ b/tests/runtime/toolbehavior/test_agenttoolsgetparticipants.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +from band.runtime.execution import ExecutionContext +from band.runtime.tools import ( + AgentTools, +) +from tests.conftest import make_participant_mock + + +class TestAgentToolsGetParticipants: + """Test get_participants tool.""" + + async def test_get_participants_success(self, mock_rest_client): + """get_participants() should return Fern participant models.""" + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.get_participants() + + assert len(result) == 1 + assert result[0].name == "User One" + + async def test_get_participants_empty(self, mock_rest_client): + """get_participants() should return empty list if none.""" + mock_rest_client.agent_api_participants.list_agent_chat_participants.return_value = MagicMock( + data=None + ) + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.get_participants() + + assert result == [] + + async def test_get_participants_updates_cache(self, mock_rest_client): + """get_participants() should refresh self._participants for mention resolution.""" + tools = AgentTools("room-123", mock_rest_client) + assert tools._participants == [] + + await tools.get_participants() + + assert tools._participants == [ + { + "id": "user-1", + "name": "User One", + "type": "User", + "handle": "user-one", + "description": None, + } + ] + + async def test_get_participants_preserves_cache_when_data_none( + self, mock_rest_client + ): + """``data is None`` indicates a transient/unexpected response — the + cache should be preserved rather than wiped (an agent should always + be a participant in its own room).""" + mock_rest_client.agent_api_participants.list_agent_chat_participants.return_value = MagicMock( + data=None + ) + cached = [ + {"id": "user-1", "name": "User One", "type": "User", "handle": "user-one"} + ] + tools = AgentTools("room-123", mock_rest_client, participants=cached) + + result = await tools.get_participants() + + assert result == [] + assert tools._participants == cached + + async def test_get_participants_clears_cache_on_empty_list(self, mock_rest_client): + """An explicit empty list from the server is authoritative — cache should clear.""" + mock_rest_client.agent_api_participants.list_agent_chat_participants.return_value = MagicMock( + data=[] + ) + stale = [{"id": "ghost", "name": "Ghost", "type": "User", "handle": "ghost"}] + tools = AgentTools("room-123", mock_rest_client, participants=stale) + + await tools.get_participants() + + assert tools._participants == [] + + async def test_get_participants_syncs_roster_to_ctx(self, mock_rest_client): + """get_participants() must make the ctx roster follow the REST list — + stale entries drop (even ctx-only ones this AgentTools never saw) and + new ones appear — so the refresh survives turn boundaries.""" + + ctx = ExecutionContext( + room_id="room-123", + link=MagicMock(rest=mock_rest_client), + on_execute=AsyncMock(), + ) + ctx.add_participant( + {"id": "user-1", "name": "User One", "type": "User", "handle": "user-one"} + ) + tools = AgentTools.from_context(ctx) + # A ctx-only participant that joined after tools were built, then left + # before the refresh — it must not survive as a ghost. + ctx.add_participant({"id": "user-3", "name": "User Three", "type": "User"}) + + # Server returns only a *new* participant — user-1 and user-3 are gone. + new_p = make_participant_mock("user-2", "User Two", "User", handle="user-two") + mock_rest_client.agent_api_participants.list_agent_chat_participants = ( + AsyncMock(return_value=MagicMock(data=[new_p])) + ) + + await tools.get_participants() + + assert [p["id"] for p in ctx.participants] == ["user-2"] + assert ctx.participants[0]["handle"] == "user-two" + + async def test_send_message_mentions_newly_discovered_participant( + self, mock_rest_client + ): + """Mentioning a participant first seen via get_participants() should not raise.""" + tools = AgentTools("room-123", mock_rest_client) + + await tools.get_participants() + await tools.send_message("Hi @user-one!", mentions=["user-one"]) + + call_args = ( + mock_rest_client.agent_api_messages.create_agent_chat_message.call_args + ) + message = call_args.kwargs["message"] + assert len(message.mentions) == 1 + assert message.mentions[0].id == "user-1" + assert message.mentions[0].handle == "user-one" diff --git a/tests/runtime/toolbehavior/test_agenttoolslookuppeers.py b/tests/runtime/toolbehavior/test_agenttoolslookuppeers.py new file mode 100644 index 000000000..be52141fc --- /dev/null +++ b/tests/runtime/toolbehavior/test_agenttoolslookuppeers.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from band.runtime.tools import ( + AgentTools, +) + + +class TestAgentToolsLookupPeers: + """Test lookup_peers tool.""" + + async def test_lookup_peers_success(self, mock_rest_client): + """lookup_peers() should return the Fern response directly.""" + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.lookup_peers(page=1, page_size=50) + + # Now returns full Fern response with .data and .metadata + assert len(result.data) == 1 + assert result.data[0].name == "Agent Two" + assert result.metadata.page == 1 + + async def test_lookup_peers_filters_by_room(self, mock_rest_client): + """lookup_peers() should filter by not_in_chat.""" + tools = AgentTools("room-123", mock_rest_client) + + await tools.lookup_peers() + + call_args = mock_rest_client.agent_api_peers.list_agent_peers.call_args + assert call_args.kwargs["not_in_chat"] == "room-123" diff --git a/tests/runtime/toolbehavior/test_agenttoolsremoveparticipant.py b/tests/runtime/toolbehavior/test_agenttoolsremoveparticipant.py new file mode 100644 index 000000000..24a7401e0 --- /dev/null +++ b/tests/runtime/toolbehavior/test_agenttoolsremoveparticipant.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from band.runtime.tools import ( + AgentTools, +) + + +class TestAgentToolsRemoveParticipant: + """Test remove_participant tool.""" + + async def test_remove_participant_by_name(self, mock_rest_client): + """remove_participant() should match by name and remove via REST.""" + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.remove_participant("User One") + + assert result["id"] == "user-1" + assert result["name"] == "User One" + assert result["status"] == "removed" + mock_rest_client.agent_api_participants.remove_agent_chat_participant.assert_called_once() + + async def test_remove_participant_by_handle(self, mock_rest_client): + """remove_participant() should match by handle.""" + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.remove_participant("user-one") + + assert result["id"] == "user-1" + assert result["name"] == "User One" + assert result["status"] == "removed" + + async def test_remove_participant_not_found_raises(self, mock_rest_client): + """remove_participant() should raise if not in room.""" + # Return empty participants + mock_rest_client.agent_api_participants.list_agent_chat_participants.return_value = MagicMock( + data=[] + ) + tools = AgentTools("room-123", mock_rest_client) + + with pytest.raises(ValueError, match="not found in this room"): + await tools.remove_participant("Unknown") diff --git a/tests/runtime/toolbehavior/test_agenttoolsschemas.py b/tests/runtime/toolbehavior/test_agenttoolsschemas.py new file mode 100644 index 000000000..f5cbcf0fc --- /dev/null +++ b/tests/runtime/toolbehavior/test_agenttoolsschemas.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +from band.core.types import Capability +from band.runtime.tools import ( + TOOL_MODELS, + AgentTools, +) + + +class TestAgentToolsSchemas: + """Test tool schema generation.""" + + def test_tool_models_registry(self): + """TOOL_MODELS should contain all tool input models.""" + assert "band_send_message" in TOOL_MODELS + assert "band_send_event" in TOOL_MODELS + assert "band_add_participant" in TOOL_MODELS + assert "band_remove_participant" in TOOL_MODELS + assert "band_lookup_peers" in TOOL_MODELS + assert "band_get_participants" in TOOL_MODELS + assert "band_create_chatroom" in TOOL_MODELS + + def test_tool_models_property(self, mock_rest_client): + """tool_models property should return registry.""" + tools = AgentTools("room-123", mock_rest_client) + + assert tools.tool_models is TOOL_MODELS + + def test_get_tool_schemas_openai(self, mock_rest_client): + """get_tool_schemas('openai') should return OpenAI format (memory tools excluded by default).""" + tools = AgentTools("room-123", mock_rest_client) + + schemas = tools.get_tool_schemas("openai") + + tool_names = [s["function"]["name"] for s in schemas] + # Base platform tools + assert "band_send_message" in tool_names + assert "band_send_event" in tool_names + assert "band_add_participant" in tool_names + assert "band_remove_participant" in tool_names + assert "band_get_participants" in tool_names + assert "band_lookup_peers" in tool_names + assert "band_create_chatroom" in tool_names + # Contact tools included by default + assert "band_list_contacts" in tool_names + assert "band_add_contact" in tool_names + # Memory tools excluded by default + assert "band_list_memories" not in tool_names + assert "band_store_memory" not in tool_names + + send_msg = next( + s for s in schemas if s["function"]["name"] == "band_send_message" + ) + assert send_msg["type"] == "function" + assert "parameters" in send_msg["function"] + assert "description" in send_msg["function"] + + def test_get_tool_schemas_openai_with_memory(self, mock_rest_client): + """get_tool_schemas('openai', include_memory=True) should include memory tools.""" + tools = AgentTools("room-123", mock_rest_client) + + schemas = tools.get_tool_schemas( + "openai", capabilities=frozenset({Capability.MEMORY, Capability.CONTACTS}) + ) + + tool_names = [s["function"]["name"] for s in schemas] + # Memory tools present + assert "band_list_memories" in tool_names + assert "band_store_memory" in tool_names + assert "band_get_memory" in tool_names + assert "band_supersede_memory" in tool_names + assert "band_archive_memory" in tool_names + # Base and contact tools still present + assert "band_send_message" in tool_names + assert "band_list_contacts" in tool_names + + def test_get_tool_schemas_anthropic(self, mock_rest_client): + """get_tool_schemas('anthropic') should return Anthropic format (memory tools excluded by default).""" + tools = AgentTools("room-123", mock_rest_client) + + schemas = tools.get_tool_schemas("anthropic") + + tool_names = [s["name"] for s in schemas] + assert "band_send_message" in tool_names + assert "band_list_contacts" in tool_names + assert "band_list_memories" not in tool_names + + send_msg = next(s for s in schemas if s["name"] == "band_send_message") + assert "input_schema" in send_msg + assert "description" in send_msg + + def test_get_tool_schemas_anthropic_with_memory(self, mock_rest_client): + """get_tool_schemas('anthropic', include_memory=True) should include memory tools.""" + tools = AgentTools("room-123", mock_rest_client) + + schemas = tools.get_tool_schemas( + "anthropic", + capabilities=frozenset({Capability.MEMORY, Capability.CONTACTS}), + ) + + tool_names = [s["name"] for s in schemas] + assert "band_list_memories" in tool_names + assert "band_store_memory" in tool_names + assert "band_send_message" in tool_names + assert "band_list_contacts" in tool_names + + def test_schemas_drop_numeric_bounds(self, mock_rest_client): + """Pydantic Field(ge=.., le=..) renders minimum/maximum, which some + providers reject on integer params; the schemas must omit them while the + models still enforce the bounds at execution.""" + tools = AgentTools("room-123", mock_rest_client) + + schemas = tools.get_tool_schemas( + "openai", capabilities=frozenset({Capability.MEMORY, Capability.CONTACTS}) + ) + + page_size = next( + s["function"]["parameters"]["properties"]["page_size"] + for s in schemas + if s["function"]["name"] == "band_lookup_peers" + ) + assert "minimum" not in page_size + assert "maximum" not in page_size + assert page_size["type"] == "integer" diff --git a/tests/runtime/toolbehavior/test_agenttoolssendevent.py b/tests/runtime/toolbehavior/test_agenttoolssendevent.py new file mode 100644 index 000000000..3ba0936b3 --- /dev/null +++ b/tests/runtime/toolbehavior/test_agenttoolssendevent.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import asyncio +from unittest.mock import AsyncMock, MagicMock + +import pytest +from band_rest.errors import ForbiddenError +from band_rest.types import Error, ErrorError + +from band.core.exceptions import RoomExecutionStoppedError +from band.runtime.execution import ExecutionContext +from band.runtime.tools import ( + AgentTools, +) +from tests.content import BLANK_CONTENT_CASES + + +class TestAgentToolsSendEvent: + """Test send_event tool.""" + + async def test_send_event_success(self, mock_rest_client): + """send_event() should send via REST and return the Fern model.""" + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.send_event("Thinking...", "thought") + + # Now returns Fern model (mock), not dict + assert result.model_dump()["message_type"] == "thought" + mock_rest_client.agent_api_events.create_agent_chat_event.assert_called_once() + + async def test_send_event_with_metadata(self, mock_rest_client): + """send_event() should pass metadata.""" + tools = AgentTools("room-123", mock_rest_client) + + await tools.send_event("Error!", "error", metadata={"code": 500}) + + call_args = mock_rest_client.agent_api_events.create_agent_chat_event.call_args + event = call_args.kwargs["event"] + assert event.metadata == {"code": 500} + + async def test_send_event_within_limit_untouched(self, mock_rest_client): + """send_event() should pass short content through unchanged.""" + tools = AgentTools("room-123", mock_rest_client) + content = "x" * 16384 + + await tools.send_event(content, "tool_result") + + call_args = mock_rest_client.agent_api_events.create_agent_chat_event.call_args + assert call_args.kwargs["event"].content == content + + async def test_send_event_truncates_oversized_content(self, mock_rest_client): + """send_event() caps oversized content, keeping both its head and tail.""" + tools = AgentTools("room-123", mock_rest_client) + content = "HEAD" * 10000 + "TAIL" * 10000 + + await tools.send_event(content, "tool_result") + + call_args = mock_rest_client.agent_api_events.create_agent_chat_event.call_args + sent_content = call_args.kwargs["event"].content + assert len(sent_content) == 16384 + assert sent_content.startswith("HEAD") # head is retained + assert sent_content.endswith("TAIL") # and the tail isn't dropped + assert "[truncated]" in sent_content # with a marker between them + + @pytest.mark.parametrize("content", BLANK_CONTENT_CASES) + async def test_send_event_refuses_content_with_no_visible_characters( + self, mock_rest_client, content + ): + """send_event() should never call the API with blank content — the platform 422s on it.""" + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.send_event(content, "tool_result") + + assert result is None + mock_rest_client.agent_api_events.create_agent_chat_event.assert_not_called() + + async def test_stopped_scope_cancels_posts_without_latching_room( + self, mock_rest_client + ): + ctx = ExecutionContext( + "room-123", MagicMock(rest=mock_rest_client), AsyncMock() + ) + await ctx._begin_scope() + tools = AgentTools.from_context(ctx) + mock_rest_client.agent_api_events.create_agent_chat_event.side_effect = ( + ForbiddenError( + body=Error( + error=ErrorError( + code="forbidden", + message="Agent execution is stopped; cannot post events", + request_id="req-1", + ) + ) + ) + ) + for content in ("first", "later"): + with pytest.raises(asyncio.CancelledError): + await tools.send_event(content, "thought") + assert not ctx.is_stopped + assert not tools.turn.complete + mock_rest_client.agent_api_events.create_agent_chat_event.assert_awaited_once() + + async def test_send_event_without_an_execution_still_raises_the_typed_error( + self, mock_rest_client + ): + """Tools built without an execution context have no room state to + adopt: the typed error propagates so the caller sees the platform's + rejection.""" + mock_rest_client.agent_api_events.create_agent_chat_event.side_effect = ( + ForbiddenError( + body=Error( + error=ErrorError( + code="forbidden", + message="Agent execution is stopped; cannot post events", + request_id="req-1", + ) + ) + ) + ) + tools = AgentTools("room-123", mock_rest_client) + + with pytest.raises(RoomExecutionStoppedError): + await tools.send_event("thinking", "thought") diff --git a/tests/runtime/toolbehavior/test_agenttoolssendfailure.py b/tests/runtime/toolbehavior/test_agenttoolssendfailure.py new file mode 100644 index 000000000..79c814635 --- /dev/null +++ b/tests/runtime/toolbehavior/test_agenttoolssendfailure.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import pytest +from band_sdk_core import AgentFailure + +from band.runtime.tools import ( + AgentTools, +) + + +class TestAgentToolsSendFailure: + """Test send_failure's best-effort delegation over the real REST boundary.""" + + async def test_send_failure_posts_an_error_event(self, mock_rest_client): + tools = AgentTools("room-123", mock_rest_client) + + await tools.send_failure(AgentFailure("codex", "boom", "timeout")) + + call_args = mock_rest_client.agent_api_events.create_agent_chat_event.call_args + event = call_args.kwargs["event"] + assert event.message_type == "error" + assert event.metadata["failure"] == { + "provider": "codex", + "code": "timeout", + "message": "boom", + "detail": None, + } + + async def test_send_failure_swallows_a_rest_rejection(self, mock_rest_client): + """A failed report must resolve, not raise -- it runs inside a + caller's except block reporting a real provider failure already.""" + mock_rest_client.agent_api_events.create_agent_chat_event.side_effect = ( + RuntimeError("REST rejected the event") + ) + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.send_failure(AgentFailure("codex", "boom")) + + assert result == {"ok": False, "error": "REST rejected the event"} + + async def test_send_event_itself_still_raises_on_the_same_rejection( + self, mock_rest_client + ): + """The other half of the best-effort contract: send_event's own + raising behavior is unchanged by send_failure wrapping it.""" + mock_rest_client.agent_api_events.create_agent_chat_event.side_effect = ( + RuntimeError("REST rejected the event") + ) + tools = AgentTools("room-123", mock_rest_client) + + with pytest.raises(RuntimeError, match="REST rejected the event"): + await tools.send_event("task update", "task") diff --git a/tests/runtime/toolbehavior/test_agenttoolssendmessage.py b/tests/runtime/toolbehavior/test_agenttoolssendmessage.py new file mode 100644 index 000000000..74a52c43c --- /dev/null +++ b/tests/runtime/toolbehavior/test_agenttoolssendmessage.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import pytest + +from band.core.exceptions import BandToolError +from band.runtime.tools import ( + AgentTools, +) +from tests.content import BLANK_CONTENT_CASES + + +class TestAgentToolsSendMessage: + """Test send_message tool.""" + + async def test_send_message_success(self, mock_rest_client, participants): + """send_message() should send via REST and return the Fern model.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + result = await tools.send_message("Hello!", mentions=["User One"]) + + # Now returns Fern model (mock), not dict + assert result.model_dump()["id"] == "msg-123" + mock_rest_client.agent_api_messages.create_agent_chat_message.assert_called_once() + + async def test_send_message_resolves_mentions(self, mock_rest_client, participants): + """send_message() should resolve mention names to IDs and handles.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + await tools.send_message("Hello @User One!", mentions=["User One"]) + + call_args = ( + mock_rest_client.agent_api_messages.create_agent_chat_message.call_args + ) + message = call_args.kwargs["message"] + assert len(message.mentions) == 1 + assert message.mentions[0].id == "user-1" + assert message.mentions[0].handle == "@user-one" + + async def test_send_message_accepts_participant_id( + self, mock_rest_client, participants + ): + """An exact ID routes to its participant without knowing the handle.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + await tools.send_message("Hello!", mentions=["user-1"]) + + call = mock_rest_client.agent_api_messages.create_agent_chat_message.call_args + assert call.kwargs["message"].mentions[0].id == "user-1" + + async def test_send_message_omits_attachment_ids_when_not_given( + self, mock_rest_client, participants + ): + """Without attachment_ids, the field must be unset, not explicitly None. + + ChatMessageRequest serializes with exclude_unset=True; passing + attachment_ids=None would still mark the field "set" and send a + literal JSON null, which the platform rejects (expects an array or + an absent key) -- reproduced live against a real deployment. + """ + tools = AgentTools("room-123", mock_rest_client, participants) + + await tools.send_message("Hello!", mentions=["User One"]) + + call_args = ( + mock_rest_client.agent_api_messages.create_agent_chat_message.call_args + ) + message = call_args.kwargs["message"] + assert "attachment_ids" not in message.model_fields_set + + async def test_send_message_empty_mentions_excludes_self( + self, mock_rest_client, participants + ): + """The empty-mentions error lists other participants but not the agent + itself — an agent can't @mention itself.""" + + tools = AgentTools( + "room-123", mock_rest_client, participants, agent_id="user-2" + ) + + with pytest.raises(BandToolError) as exc_info: + await tools.send_message("Hello!", mentions=[]) + + message = str(exc_info.value) + assert "@user-one" in message + assert "@user-two" not in message + + async def test_send_message_unknown_mention_raises( + self, mock_rest_client, participants + ): + """send_message() should raise for unknown mention.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + with pytest.raises(ValueError, match="Unknown participant 'Unknown'"): + await tools.send_message("Hello!", mentions=["Unknown"]) + + @pytest.mark.parametrize("content", BLANK_CONTENT_CASES) + async def test_send_message_refuses_content_with_no_visible_characters( + self, mock_rest_client, participants, content + ): + """send_message() should never call the API with blank content.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + result = await tools.send_message(content, mentions=["User One"]) + + assert result is None + mock_rest_client.agent_api_messages.create_agent_chat_message.assert_not_called() diff --git a/tests/runtime/toolbehavior/test_canonicalizemcptoolname.py b/tests/runtime/toolbehavior/test_canonicalizemcptoolname.py new file mode 100644 index 000000000..26c0040f4 --- /dev/null +++ b/tests/runtime/toolbehavior/test_canonicalizemcptoolname.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from band.runtime.tools import ( + canonicalize_mcp_tool_name, +) + + +class TestCanonicalizeMcpToolName: + """Recovering the canonical band name from an MCP ``-`` spelling.""" + + OWN = frozenset({"band_send_message", "echo"}) + + def test_strips_server_prefix_off_own_tool(self): + name = canonicalize_mcp_tool_name("band-band_send_message", self.OWN) + assert name == "band_send_message" + + def test_custom_tool_prefix_stripped(self): + assert canonicalize_mcp_tool_name("band-echo", self.OWN) == "echo" + + def test_non_band_server_prefix_passes_through(self): + """Only the Band MCP server's own ``band-`` prefix resolves -- an + unrelated server's tool that happens to end in one of our names must + not be misattributed to us (e.g. narrated/suppressed as our own).""" + name = canonicalize_mcp_tool_name("platform-band_send_message", self.OWN) + assert name == "platform-band_send_message" + + def test_foreign_tool_passes_through(self): + """A prefixed name that reveals none of ours stays as reported.""" + assert canonicalize_mcp_tool_name("band-grep", self.OWN) == "band-grep" + + def test_unprefixed_name_passes_through(self): + assert ( + canonicalize_mcp_tool_name("band_send_message", self.OWN) + == "band_send_message" + ) diff --git a/tests/runtime/toolbehavior/test_emptymentionsvalidation.py b/tests/runtime/toolbehavior/test_emptymentionsvalidation.py new file mode 100644 index 000000000..9e6524e3a --- /dev/null +++ b/tests/runtime/toolbehavior/test_emptymentionsvalidation.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import pytest + +from band.core.exceptions import BandToolError +from band.runtime.tools import ( + AgentTools, +) + + +class TestEmptyMentionsValidation: + """Test that empty mentions return a helpful error with participant names.""" + + async def test_raises_error_with_participant_names( + self, mock_rest_client, participants + ): + """Should raise BandToolError listing available participants when mentions empty.""" + + tools = AgentTools("room-123", mock_rest_client, participants) + + with pytest.raises( + BandToolError, match="At least one mention is required" + ) as exc_info: + await tools.send_message("Hello!", mentions=[]) + + assert "@user-one" in str(exc_info.value) + assert "@user-two" in str(exc_info.value) + # Should NOT have called the API + mock_rest_client.agent_api_messages.create_agent_chat_message.assert_not_called() + + async def test_raises_error_when_mentions_none( + self, mock_rest_client, participants + ): + """Should raise BandToolError when mentions is None.""" + + tools = AgentTools("room-123", mock_rest_client, participants) + + with pytest.raises(BandToolError, match="At least one mention is required"): + await tools.send_message("Hello!", mentions=None) + + async def test_uses_handle_when_available(self, mock_rest_client): + """Should prefer handle over name in error message.""" + + participants = [ + {"id": "user-1", "name": "User One", "type": "User", "handle": "@user-one"}, + ] + tools = AgentTools("room-123", mock_rest_client, participants) + + with pytest.raises(BandToolError, match="@user-one"): + await tools.send_message("Hello!", mentions=[]) + + async def test_omits_participant_without_handle(self, mock_rest_client): + """Should omit handle-less participants — they can't be @mentioned.""" + + participants = [ + {"id": "user-1", "name": "User One", "type": "User"}, + ] + tools = AgentTools("room-123", mock_rest_client, participants) + + with pytest.raises( + BandToolError, match="At least one mention is required" + ) as exc_info: + await tools.send_message("Hello!", mentions=[]) + + # Name must not be offered as a mention target — only real handles are. + assert "User One" not in str(exc_info.value) + # With no mentionable handles there is nothing to suggest, so the error + # carries no handle list rather than an empty one. + assert "Available handles:" not in str(exc_info.value) + + async def test_no_error_when_mentions_provided( + self, mock_rest_client, participants + ): + """Should proceed normally when mentions are provided.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + result = await tools.send_message("Hello!", mentions=["User One"]) + + # Now returns Fern model; verify it has the expected attribute + assert result.model_dump()["id"] == "msg-123" + mock_rest_client.agent_api_messages.create_agent_chat_message.assert_called_once() + + async def test_execute_tool_call_raises_band_tool_error( + self, mock_rest_client, participants + ): + """execute_tool_call lets BandToolError propagate for wrapper translation.""" + + tools = AgentTools("room-123", mock_rest_client, participants) + + with pytest.raises(BandToolError, match="At least one mention is required"): + await tools.execute_tool_call( + "band_send_message", {"content": "Hello!", "mentions": []} + ) diff --git a/tests/runtime/toolbehavior/test_fetchroomcontext.py b/tests/runtime/toolbehavior/test_fetchroomcontext.py new file mode 100644 index 000000000..fafee1a00 --- /dev/null +++ b/tests/runtime/toolbehavior/test_fetchroomcontext.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from band.runtime.tools import ( + AgentTools, +) + + +class TestFetchRoomContext: + """Paging the agent context endpoint.""" + + @pytest.mark.asyncio + async def test_prefers_the_required_pagination_field(self, mock_rest_client): + """`metadata` is required; `meta` is optional and often absent. + + Reading only `meta` collapses every room to one synthesized page, so a + caller paging through history silently stops at the first page. + """ + pagination = MagicMock() + pagination.model_dump.return_value = {"total_pages": 3} + mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( + return_value=SimpleNamespace(data=[], metadata=pagination) + ) + + context = await AgentTools("room-123", mock_rest_client).fetch_room_context( + room_id="room-123" + ) + + assert context["meta"]["total_pages"] == 3 diff --git a/tests/runtime/toolbehavior/test_filetools.py b/tests/runtime/toolbehavior/test_filetools.py new file mode 100644 index 000000000..035144b93 --- /dev/null +++ b/tests/runtime/toolbehavior/test_filetools.py @@ -0,0 +1,755 @@ +from __future__ import annotations + +import base64 +from datetime import UTC, datetime, timedelta +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest +from band_rest import ( + GetAgentChatContextResponse, +) + +from band.client.rest import ( + DEFAULT_REQUEST_OPTIONS, + NotFoundError, +) +from band.config.settings import RuntimeSettings +from band.core.exceptions import BandToolError +from band.runtime.execution import ExecutionContext +from band.runtime.tools import ( + DEFAULT_FILE_CAPTION, + FILE_UNAVAILABLE_MESSAGE, + MAX_INLINE_IMAGE_BYTES, + MAX_INLINE_TEXT_BYTES, + MAX_SEND_CONTENT_BYTES, + AgentTools, +) +from tests.content import BLANK_CONTENT_CASES +from tests.runtime.toolbehavior.kit import ( + _attachment, + _context_response, + _fake_download, + _fake_download_not_found, + _message_with_attachments, + _mock_attachment_page, + _posted_message, +) + + +class TestFileTools: + """Tests for band_list_room_files / band_read_room_file / band_send_room_file.""" + + # --- list_room_files --- + + @pytest.mark.asyncio + async def test_list_room_files_dedupes_by_attachment_id(self, mock_rest_client): + shared = _attachment("file-1") + other = _attachment("file-2") + response = _context_response( + [ + _message_with_attachments("msg-1", [shared]), + _message_with_attachments("msg-2", [shared, other]), + ] + ) + mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( + return_value=response + ) + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.list_room_files() + + assert [a["id"] for a in result["data"]] == ["file-1", "file-2"] + mock_rest_client.agent_api_context.get_agent_chat_context.assert_awaited_once_with( + chat_id="room-123", + request_options=DEFAULT_REQUEST_OPTIONS, + ) + + @pytest.mark.asyncio + async def test_list_room_files_forwards_cursor_and_returns_next( + self, mock_rest_client + ): + response = _context_response( + [_message_with_attachments("msg-1", [_attachment()])], + next_cursor="cursor-2", + has_more=True, + ) + mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( + return_value=response + ) + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.list_room_files(cursor="cursor-1") + + assert result["next_cursor"] == "cursor-2" + mock_rest_client.agent_api_context.get_agent_chat_context.assert_awaited_once_with( + chat_id="room-123", + request_options=DEFAULT_REQUEST_OPTIONS, + cursor="cursor-1", + ) + + # --- read_room_file --- + + @pytest.mark.asyncio + async def test_read_room_file_inlines_small_text(self, mock_rest_client): + attachment = _attachment(content_type="text/plain", size=5) + _mock_attachment_page(mock_rest_client, attachment) + mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( + _fake_download(b"hello") + ) + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.read_room_file("file-1") + + assert result["text"] == "hello" + assert result["content_type"] == "text/plain" + assert "description" not in result + + @pytest.mark.asyncio + async def test_read_room_file_non_utf8_text_flags_lossy_decode( + self, mock_rest_client + ): + """A Windows-1252-encoded file (no charset in content_type -- the + platform derives it from magic bytes alone) must not come back + looking like a clean decode: 0x93/0x94 are curly quotes in + Windows-1252 but invalid UTF-8 continuation bytes on their own.""" + windows_1252_bytes = "“quoted”".encode("windows-1252") + attachment = _attachment( + content_type="text/plain", size=len(windows_1252_bytes) + ) + _mock_attachment_page(mock_rest_client, attachment) + mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( + _fake_download(windows_1252_bytes) + ) + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.read_room_file("file-1") + + assert "�" in result["text"] + assert "not valid UTF-8" in result["description"] + + @pytest.mark.asyncio + async def test_read_room_file_inlines_small_image(self, mock_rest_client): + attachment = _attachment(content_type="image/png", size=100) + _mock_attachment_page(mock_rest_client, attachment) + mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( + _fake_download(b"\x89PNG-fake-bytes") + ) + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.read_room_file("file-1") + + block = result["content"][0] + assert block["type"] == "image" + assert block["mimeType"] == "image/png" + assert base64.b64decode(block["data"]) == b"\x89PNG-fake-bytes" + + @pytest.mark.asyncio + async def test_read_room_file_text_exactly_at_limit_downloads( + self, mock_rest_client + ): + attachment = _attachment(content_type="text/plain", size=MAX_INLINE_TEXT_BYTES) + _mock_attachment_page(mock_rest_client, attachment) + download = MagicMock(side_effect=lambda **_kw: _fake_download(b"ok")) + mock_rest_client.agent_api_files.download_agent_chat_file = download + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.read_room_file("file-1") + + download.assert_called_once() + assert result["text"] == "ok" + + @pytest.mark.asyncio + async def test_read_room_file_text_one_byte_over_limit_skips_download( + self, mock_rest_client + ): + attachment = _attachment( + content_type="text/plain", size=MAX_INLINE_TEXT_BYTES + 1 + ) + _mock_attachment_page(mock_rest_client, attachment) + download = MagicMock(side_effect=lambda **_kw: _fake_download(b"ok")) + mock_rest_client.agent_api_files.download_agent_chat_file = download + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.read_room_file("file-1") + + download.assert_not_called() + assert "text" not in result + assert "description" in result + + @pytest.mark.asyncio + async def test_read_room_file_image_exactly_at_limit_downloads( + self, mock_rest_client + ): + attachment = _attachment(content_type="image/png", size=MAX_INLINE_IMAGE_BYTES) + _mock_attachment_page(mock_rest_client, attachment) + download = MagicMock(side_effect=lambda **_kw: _fake_download(b"ok")) + mock_rest_client.agent_api_files.download_agent_chat_file = download + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.read_room_file("file-1") + + download.assert_called_once() + assert result["content"][0]["type"] == "image" + + @pytest.mark.asyncio + async def test_read_room_file_image_one_byte_over_limit_skips_download( + self, mock_rest_client + ): + attachment = _attachment( + content_type="image/png", size=MAX_INLINE_IMAGE_BYTES + 1 + ) + _mock_attachment_page(mock_rest_client, attachment) + download = MagicMock(side_effect=lambda **_kw: _fake_download(b"ok")) + mock_rest_client.agent_api_files.download_agent_chat_file = download + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.read_room_file("file-1") + + download.assert_not_called() + assert "content" not in result + assert "description" in result + + @pytest.mark.asyncio + async def test_read_room_file_non_previewable_type_is_described( + self, mock_rest_client + ): + attachment = _attachment(content_type="application/pdf", size=10) + _mock_attachment_page(mock_rest_client, attachment) + download = MagicMock(side_effect=lambda **_kw: _fake_download(b"ok")) + mock_rest_client.agent_api_files.download_agent_chat_file = download + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.read_room_file("file-1") + + download.assert_not_called() + assert result["description"] + + @pytest.mark.asyncio + async def test_read_room_file_unknown_id_raises_band_tool_error( + self, mock_rest_client + ): + mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( + return_value=_context_response([]) + ) + tools = AgentTools("room-123", mock_rest_client) + + with pytest.raises(BandToolError, match=FILE_UNAVAILABLE_MESSAGE): + await tools.read_room_file("nope") + + @pytest.mark.asyncio + async def test_read_room_file_download_not_found_raises_band_tool_error( + self, mock_rest_client + ): + attachment = _attachment(content_type="text/plain", size=5) + _mock_attachment_page(mock_rest_client, attachment) + mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( + _fake_download_not_found() + ) + tools = AgentTools("room-123", mock_rest_client) + + with pytest.raises(BandToolError, match=FILE_UNAVAILABLE_MESSAGE): + await tools.read_room_file("file-1") + + @pytest.mark.asyncio + async def test_read_room_file_searches_past_the_first_page(self, mock_rest_client): + """_find_attachment must walk every page -- the target file may be + older than the first page returned.""" + page_one = _context_response( + [_message_with_attachments("msg-1", [_attachment("other-file")])], + next_cursor="cursor-2", + has_more=True, + ) + page_two = _context_response( + [ + _message_with_attachments( + "msg-2", [_attachment(content_type="text/plain", size=5)] + ) + ] + ) + mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( + side_effect=[page_one, page_two] + ) + mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( + _fake_download(b"hello") + ) + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.read_room_file("file-1") + + assert result["text"] == "hello" + get_context = mock_rest_client.agent_api_context.get_agent_chat_context + assert get_context.await_count == 2 + _, second_call_kwargs = get_context.await_args_list[1] + assert second_call_kwargs["cursor"] == "cursor-2" + + @pytest.mark.asyncio + async def test_find_attachment_stops_on_repeated_cursor(self, mock_rest_client): + """A malformed response repeating a cursor must not loop forever -- + the only pagination guard, since a room's history has no realistic + depth ceiling to cap against (unlike iter_chat_pages's room count).""" + + def _looping_page(**_kwargs: Any) -> GetAgentChatContextResponse: + return _context_response( + [_message_with_attachments("msg", [_attachment("other-file")])], + next_cursor="same-cursor", + has_more=True, + ) + + mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( + side_effect=_looping_page + ) + tools = AgentTools("room-123", mock_rest_client) + + with pytest.raises(BandToolError, match=FILE_UNAVAILABLE_MESSAGE): + await tools._find_attachment("file-1") + + get_context = mock_rest_client.agent_api_context.get_agent_chat_context + assert get_context.await_count == 2 + + @pytest.mark.asyncio + async def test_find_attachment_searches_past_fifty_pages(self, mock_rest_client): + """A room with more than 50 pages of history before the target file + must still resolve it -- there is no depth cap.""" + target_page = 60 + target = _attachment(content_type="text/plain", size=5) + + def _paged_history(**kwargs: Any) -> GetAgentChatContextResponse: + cursor = kwargs.get("cursor") + page = int(cursor.removeprefix("page-")) if cursor else 1 + if page < target_page: + return _context_response( + [ + _message_with_attachments( + f"msg-{page}", [_attachment("other-file")] + ) + ], + next_cursor=f"page-{page + 1}", + has_more=True, + ) + return _context_response( + [_message_with_attachments(f"msg-{page}", [target])] + ) + + mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( + side_effect=_paged_history + ) + mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( + _fake_download(b"hello") + ) + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.read_room_file("file-1") + + assert result["text"] == "hello" + get_context = mock_rest_client.agent_api_context.get_agent_chat_context + assert get_context.await_count == target_page + + @pytest.mark.asyncio + async def test_find_attachment_skips_pagination_on_cache_hit( + self, mock_rest_client + ): + """A second lookup for the same (room, rest, file_id) must not + re-paginate -- _find_attachment delegates to a shared @alru_cache + keyed on those three, not on the AgentTools instance calling it.""" + _mock_attachment_page(mock_rest_client, _attachment("file-1")) + first = AgentTools("room-123", mock_rest_client) + second = AgentTools("room-123", mock_rest_client) + + await first._find_attachment("file-1") + await second._find_attachment("file-1") + + mock_rest_client.agent_api_context.get_agent_chat_context.assert_awaited_once() + + @pytest.mark.asyncio + async def test_find_attachment_does_not_share_cache_across_rooms( + self, mock_rest_client + ): + """The same file_id looked up in two different rooms must not + cache-hit across them -- room_id is part of the cache key, not + just file_id.""" + _mock_attachment_page(mock_rest_client, _attachment("file-1")) + room_a = AgentTools("room-a", mock_rest_client) + room_b = AgentTools("room-b", mock_rest_client) + + await room_a._find_attachment("file-1") + await room_b._find_attachment("file-1") + + assert ( + mock_rest_client.agent_api_context.get_agent_chat_context.await_count == 2 + ) + + @pytest.mark.asyncio + async def test_find_attachment_does_not_share_cache_across_rest_clients( + self, mock_rest_client + ): + """The same (room_id, file_id) looked up through two different REST + clients must not cache-hit across them -- the rest client identity is + part of the cache key, so two agents (or a pool of clients) never + leak each other's attachment metadata.""" + _mock_attachment_page(mock_rest_client, _attachment("file-1")) + other_rest_client = MagicMock() + _mock_attachment_page(other_rest_client, _attachment("file-1")) + tools_a = AgentTools("room-123", mock_rest_client) + tools_b = AgentTools("room-123", other_rest_client) + + await tools_a._find_attachment("file-1") + await tools_b._find_attachment("file-1") + + mock_rest_client.agent_api_context.get_agent_chat_context.assert_awaited_once() + other_rest_client.agent_api_context.get_agent_chat_context.assert_awaited_once() + + @pytest.mark.asyncio + async def test_find_attachment_retries_pagination_after_a_miss( + self, mock_rest_client + ): + """A miss must not be cached: a file posted after a failed lookup is + still findable on the next attempt, not permanently "not found".""" + mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( + return_value=_context_response([]) + ) + tools = AgentTools("room-123", mock_rest_client) + + with pytest.raises(BandToolError, match=FILE_UNAVAILABLE_MESSAGE): + await tools._find_attachment("file-1") + + _mock_attachment_page(mock_rest_client, _attachment("file-1")) + result = await tools._find_attachment("file-1") + + assert result.id == "file-1" + + @pytest.mark.asyncio + async def test_find_attachment_rejects_expired_cached_entry(self, mock_rest_client): + """An attachment past its own expires_at must not be handed back + from cache -- it's evicted and re-fetched once (the cached copy may + just predate the platform extending the deadline), and only raised + as not-found if the fresh read is still expired too -- which also + evicts it, so the cache never keeps serving metadata it already + gave up on.""" + expired = _attachment( + "file-1", expires_at=datetime.now(UTC) - timedelta(seconds=1) + ) + _mock_attachment_page(mock_rest_client, expired) + tools = AgentTools("room-123", mock_rest_client) + + with pytest.raises(BandToolError, match=FILE_UNAVAILABLE_MESSAGE): + await tools._find_attachment("file-1") + + get_context = mock_rest_client.agent_api_context.get_agent_chat_context + assert get_context.await_count == 2 + assert not AgentTools._attachment_cache().cache_contains( + "room-123", mock_rest_client, "file-1" + ) + + @pytest.mark.asyncio + async def test_find_attachment_refreshes_an_expired_entry(self, mock_rest_client): + """A cache entry whose expires_at has passed gets exactly one + real re-fetch -- if that comes back not-expired (the platform + extended it), it's returned, not rejected on the stale cached + timestamp.""" + not_expired = _attachment( + "file-1", + content_type="text/plain", + size=5, + expires_at=datetime.now(UTC) + timedelta(days=1), + ) + expired_page = _context_response( + [ + _message_with_attachments( + "msg-1", + [ + _attachment( + "file-1", + expires_at=datetime.now(UTC) - timedelta(seconds=1), + ) + ], + ) + ] + ) + refreshed_page = _context_response( + [_message_with_attachments("msg-1", [not_expired])] + ) + mock_rest_client.agent_api_context.get_agent_chat_context = AsyncMock( + side_effect=[expired_page, refreshed_page] + ) + mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( + _fake_download(b"hello") + ) + tools = AgentTools("room-123", mock_rest_client) + + result = await tools.read_room_file("file-1") + + assert result["text"] == "hello" + + @pytest.mark.asyncio + async def test_find_attachment_rejects_expired_naive_timestamp( + self, mock_rest_client + ): + """A naive (offset-less) expires_at -- the Fern model doesn't enforce + one -- must be treated as UTC, not raise on comparison to aware + now().""" + expired = _attachment( + "file-1", + expires_at=datetime(2020, 1, 1), + ) + _mock_attachment_page(mock_rest_client, expired) + tools = AgentTools("room-123", mock_rest_client) + + with pytest.raises(BandToolError, match=FILE_UNAVAILABLE_MESSAGE): + await tools._find_attachment("file-1") + + def test_attachment_cache_maxsize_is_wired_to_settings(self) -> None: + """The @alru_cache's maxsize must actually come from RuntimeSettings + -- the eviction algorithm itself is async-lru's to test, not ours.""" + assert AgentTools._attachment_cache().cache_parameters()["maxsize"] == ( + RuntimeSettings().BAND_ATTACHMENT_CACHE_MAXSIZE + ) + + def test_attachment_cache_rereads_settings_lazily( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """maxsize must come from RuntimeSettings at first real use, not at + module import -- every example does `from band import Agent` (which + imports this module) before its own load_dotenv() call, so baking + RuntimeSettings() into a decorator at class-body time would silently + ignore a .env-set override.""" + AgentTools._attachment_cache.cache_clear() + monkeypatch.setenv("BAND_ATTACHMENT_CACHE_MAXSIZE", "42") + try: + assert AgentTools._attachment_cache().cache_parameters()["maxsize"] == 42 + finally: + AgentTools._attachment_cache.cache_clear() + + @pytest.mark.asyncio + async def test_read_room_file_caches_attachments_across_recreated_tools( + self, mock_rest_client + ): + """Real ExecutionContext + from_context(): a file found by one turn's + AgentTools must skip re-pagination on the next turn's recreated + instance, since AgentTools.from_context() rebuilds fresh every turn. + """ + _mock_attachment_page( + mock_rest_client, _attachment(content_type="text/plain", size=5) + ) + mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( + _fake_download(b"hello") + ) + ctx = ExecutionContext( + room_id="room-789", + link=MagicMock(rest=mock_rest_client), + on_execute=AsyncMock(), + ) + + tools1 = AgentTools.from_context(ctx) + await tools1.read_room_file("file-1") + + tools2 = AgentTools.from_context(ctx) + await tools2.read_room_file("file-1") + + mock_rest_client.agent_api_context.get_agent_chat_context.assert_awaited_once() + + # --- send_room_file --- + + @pytest.mark.asyncio + async def test_send_room_file_uploads_and_posts_message( + self, mock_rest_client, participants + ): + uploaded = _attachment("file-9", name="report.txt") + upload_response = MagicMock() + upload_response.data = uploaded + mock_rest_client.agent_api_files.upload_agent_chat_file = AsyncMock( + return_value=upload_response + ) + + tools = AgentTools("room-123", mock_rest_client, participants) + + result = await tools.send_room_file( + "hello world", "report.txt", caption="here you go", mentions=["User One"] + ) + + assert result["attachment"]["id"] == "file-9" + upload_call = mock_rest_client.agent_api_files.upload_agent_chat_file + upload_call.assert_awaited_once() + _, kwargs = upload_call.await_args + assert kwargs["chat_id"] == "room-123" + assert kwargs["request"] == b"hello world" + headers = kwargs["request_options"]["additional_headers"] + assert headers["x-file-name"] == "report.txt" + mock_rest_client.agent_api_messages.create_agent_chat_message.assert_awaited_once() + posted = _posted_message(mock_rest_client) + assert posted.attachment_ids == ["file-9"] + assert posted.content == "here you go" + + @pytest.mark.asyncio + async def test_send_room_file_exactly_at_limit_succeeds( + self, mock_rest_client, participants + ): + uploaded = _attachment("file-9") + upload_response = MagicMock() + upload_response.data = uploaded + mock_rest_client.agent_api_files.upload_agent_chat_file = AsyncMock( + return_value=upload_response + ) + tools = AgentTools("room-123", mock_rest_client, participants) + body = "a" * MAX_SEND_CONTENT_BYTES + + await tools.send_room_file(body, "big.txt", mentions=["User One"]) + + mock_rest_client.agent_api_files.upload_agent_chat_file.assert_awaited_once() + + @pytest.mark.asyncio + async def test_send_room_file_one_byte_over_limit_raises_before_upload( + self, mock_rest_client, participants + ): + tools = AgentTools("room-123", mock_rest_client, participants) + body = "a" * (MAX_SEND_CONTENT_BYTES + 1) + + with pytest.raises(BandToolError, match="exceeds"): + await tools.send_room_file(body, "big.txt", mentions=["User One"]) + + mock_rest_client.agent_api_files.upload_agent_chat_file.assert_not_called() + + @pytest.mark.asyncio + async def test_send_room_file_non_ascii_filename_raises_before_upload( + self, mock_rest_client, participants + ): + """x-file-name travels as a raw HTTP header value -- httpx refuses to + encode a non-ASCII header and raises UnicodeEncodeError deep inside + the upload call. Catch it before uploading with a clear message + instead of letting that raw codec error surface to the LLM.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + with pytest.raises(BandToolError, match="ASCII characters only"): + await tools.send_room_file("hi", "报告.txt", mentions=["User One"]) + + mock_rest_client.agent_api_files.upload_agent_chat_file.assert_not_called() + + @pytest.mark.asyncio + async def test_send_room_file_crlf_in_filename_raises_before_upload( + self, mock_rest_client, participants + ): + """A bare '\\r'/'\\n' encodes to ASCII fine, so an "is it ASCII" + check alone would wave a header-injection payload straight through + to the upload call -- only rejected there, deep inside h11, as an + "Illegal header value" once the request is already being built.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + with pytest.raises(BandToolError, match="ASCII characters only"): + await tools.send_room_file( + "hi", "evil\r\nX-Injected: yes", mentions=["User One"] + ) + + mock_rest_client.agent_api_files.upload_agent_chat_file.assert_not_called() + + @pytest.mark.asyncio + async def test_send_room_file_upload_not_found_raises_band_tool_error( + self, mock_rest_client, participants + ): + mock_rest_client.agent_api_files.upload_agent_chat_file = AsyncMock( + side_effect=NotFoundError(body=MagicMock()) + ) + tools = AgentTools("room-123", mock_rest_client, participants) + + with pytest.raises(BandToolError, match=FILE_UNAVAILABLE_MESSAGE): + await tools.send_room_file("hi", "f.txt", mentions=["User One"]) + + @pytest.mark.asyncio + async def test_send_room_file_empty_mentions_raises_before_upload( + self, mock_rest_client, participants + ): + """Sharing a file is a send_message call under the hood, so a missing + mention must fail before the upload -- not after, which would leave + an orphaned attachment nothing points at.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + with pytest.raises(BandToolError, match="At least one mention is required"): + await tools.send_room_file("hi", "f.txt", mentions=[]) + + mock_rest_client.agent_api_files.upload_agent_chat_file.assert_not_called() + + @pytest.mark.asyncio + async def test_send_room_file_empty_caption_uses_default( + self, mock_rest_client, participants + ): + """The platform rejects blank message content outright + (``validate_length(:content, min: 1)``, unconditional even with + ``attachment_ids`` set) -- an omitted caption must never reach + ``send_message`` as ``""``, or the post 422s after the file has + already been uploaded, leaving an orphaned attachment.""" + uploaded = _attachment("file-9", name="report.txt") + upload_response = MagicMock() + upload_response.data = uploaded + mock_rest_client.agent_api_files.upload_agent_chat_file = AsyncMock( + return_value=upload_response + ) + tools = AgentTools("room-123", mock_rest_client, participants) + + result = await tools.send_room_file( + "hello world", "report.txt", mentions=["User One"] + ) + + assert result["attachment"]["id"] == "file-9" + assert _posted_message(mock_rest_client).content == ( + DEFAULT_FILE_CAPTION.format(filename="report.txt") + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize("caption", BLANK_CONTENT_CASES) + async def test_send_room_file_blank_caption_uses_default( + self, mock_rest_client, participants, caption + ): + """A caption that only *looks* non-empty falls back to the default + too: the send refuses content with no visible characters and returns + ``None``, so anything narrower than the platform's own rule here + drops the post after the file is already uploaded, leaving an + orphaned attachment.""" + uploaded = _attachment("file-9", name="report.txt") + upload_response = MagicMock() + upload_response.data = uploaded + mock_rest_client.agent_api_files.upload_agent_chat_file = AsyncMock( + return_value=upload_response + ) + tools = AgentTools("room-123", mock_rest_client, participants) + + result = await tools.send_room_file( + "hello world", "report.txt", caption=caption, mentions=["User One"] + ) + + assert result["attachment"]["id"] == "file-9" + assert _posted_message(mock_rest_client).content == ( + DEFAULT_FILE_CAPTION.format(filename="report.txt") + ) + + @pytest.mark.asyncio + async def test_send_room_file_discoverable_from_a_new_agenttools_instance( + self, mock_rest_client, participants + ): + """AgentTools is recreated per execution (a fresh instance per turn), + so a file must be discoverable through the REST layer alone, not + through any in-process state the sending instance happened to hold. + Regression guard for the agent messages endpoint's direct_only + exclusion of self-authored messages: file discovery must go through + the context endpoint instead, which includes them.""" + uploaded = _attachment("file-9", name="report.txt", size=5) + upload_response = MagicMock() + upload_response.data = uploaded + mock_rest_client.agent_api_files.upload_agent_chat_file = AsyncMock( + return_value=upload_response + ) + sender = AgentTools("room-123", mock_rest_client, participants) + sent = await sender.send_room_file("hello", "report.txt", mentions=["User One"]) + + # A later turn: brand new instance, no shared state with `sender`. + # Mirrors what the real platform now returns -- the context endpoint + # includes messages this agent sent, unlike the plain messages one. + _mock_attachment_page(mock_rest_client, uploaded) + mock_rest_client.agent_api_files.download_agent_chat_file = lambda **_kw: ( + _fake_download(b"hello") + ) + reader = AgentTools("room-123", mock_rest_client, participants) + + listed = await reader.list_room_files() + read = await reader.read_room_file(sent["attachment"]["id"]) + + assert [a["id"] for a in listed["data"]] == ["file-9"] + assert read["text"] == "hello" diff --git a/tests/runtime/toolbehavior/test_formattoolvalidationerror.py b/tests/runtime/toolbehavior/test_formattoolvalidationerror.py new file mode 100644 index 000000000..e4c286239 --- /dev/null +++ b/tests/runtime/toolbehavior/test_formattoolvalidationerror.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import pytest +from pydantic import BaseModel, ValidationError + +from band.runtime.tools import ( + format_tool_validation_error, +) + + +class TestFormatToolValidationError: + """Pins the exact wire-message shape `format_tool_validation_error` produces. + + The published band-mcp CLI surfaces this string verbatim to its callers + (via ``StandaloneResolver``), so the separator between multiple field + errors and the dotted-path format for a nested field are part of the + wire contract, not just an internal formatting detail free to drift. + """ + + def test_multiple_and_nested_field_errors_joined_with_comma(self): + class Nested(BaseModel): + name: str + + class Model(BaseModel): + items: list[Nested] + extra: str + + with pytest.raises(ValidationError) as exc_info: + Model(items=[{}], extra=None) + + message = format_tool_validation_error("some_tool", exc_info.value) + + assert message == ( + "Invalid arguments for some_tool: " + "items.0.name: Field required, " + "extra: Input should be a valid string" + ) diff --git a/tests/runtime/toolbehavior/test_handlementionresolution.py b/tests/runtime/toolbehavior/test_handlementionresolution.py new file mode 100644 index 000000000..9980b6651 --- /dev/null +++ b/tests/runtime/toolbehavior/test_handlementionresolution.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import pytest + +from band.runtime.tools import ( + AgentTools, + append_mention_handles_hint, + available_mention_handles, +) + + +class TestHandleMentionResolution: + """Test handle-based mention resolution.""" + + def test_available_mention_handles_excludes_self_and_missing_handles(self): + """Available handle hints should include only mentionable room handles.""" + participants = [ + {"id": "user-1", "name": "User One", "handle": "@user-one"}, + {"id": "self", "name": "Self", "handle": "@self"}, + {"id": "user-3", "name": "No Handle", "handle": None}, + ] + + assert available_mention_handles(participants, agent_id="self") == ["@user-one"] + + def test_append_mention_handles_hint_is_idempotent(self): + """An error already carrying the hint is returned unchanged, so the same + error can pass through multiple adapter enrichers without doubling.""" + enriched = append_mention_handles_hint( + "At least one mention is required", ["@alice"] + ) + assert enriched.count("Available handles:") == 1 + + twice = append_mention_handles_hint(enriched, ["@alice"]) + assert twice == enriched + + def test_append_mention_handles_hint_no_handles_is_noop(self): + """With no mentionable handles there is nothing to suggest.""" + error = "At least one mention is required" + assert append_mention_handles_hint(error, []) == error + + def test_resolve_by_handle(self, mock_rest_client, participants): + """Should resolve mentions by handle.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + resolved = tools._resolve_mentions(["@user-one"]) + + assert len(resolved) == 1 + assert resolved[0] == {"id": "user-1", "handle": "@user-one"} + + def test_resolve_handle_takes_priority(self, mock_rest_client): + """Should try handle lookup before name lookup.""" + # Participant with handle different from name + participants = [ + { + "id": "agent-1", + "name": "Weather Agent", + "type": "Agent", + "handle": "@john/weather", + }, + ] + tools = AgentTools("room-123", mock_rest_client, participants) + + # Resolve by handle + resolved = tools._resolve_mentions(["@john/weather"]) + assert resolved[0] == {"id": "agent-1", "handle": "@john/weather"} + + # Resolve by name still works + resolved = tools._resolve_mentions(["Weather Agent"]) + assert resolved[0] == {"id": "agent-1", "handle": "@john/weather"} + + def test_resolve_mixed_handles_and_names(self, mock_rest_client, participants): + """Should resolve a mix of handles and names.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + resolved = tools._resolve_mentions(["@user-one", "User Two"]) + + assert len(resolved) == 2 + assert resolved[0] == {"id": "user-1", "handle": "@user-one"} + assert resolved[1] == {"id": "user-2", "handle": "@user-two"} + + def test_resolve_unknown_handle_raises(self, mock_rest_client, participants): + """Should raise for unknown handle.""" + tools = AgentTools( + "room-123", mock_rest_client, participants, agent_id="user-2" + ) + + # @ prefix is stripped during normalization + with pytest.raises(ValueError, match="Unknown participant 'unknown'") as exc: + tools._resolve_mentions(["@unknown"]) + + message = str(exc.value) + assert "@user-one" in message + assert "@user-two" not in message + + def test_resolve_participant_without_handle(self, mock_rest_client): + """Should resolve by name when participant has no handle.""" + participants = [ + {"id": "user-1", "name": "User One", "type": "User", "handle": None}, + ] + tools = AgentTools("room-123", mock_rest_client, participants) + + resolved = tools._resolve_mentions(["User One"]) + + assert resolved[0] == {"id": "user-1", "handle": None} diff --git a/tests/runtime/toolbehavior/test_ismcpcontentresult.py b/tests/runtime/toolbehavior/test_ismcpcontentresult.py new file mode 100644 index 000000000..79c575898 --- /dev/null +++ b/tests/runtime/toolbehavior/test_ismcpcontentresult.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from typing import Any, ClassVar + +from band.runtime.tools import ( + is_mcp_content_result, +) + + +class TestIsMcpContentResult: + """``is_mcp_content_result`` recognizes ``read_room_file``'s image-branch + shape so a consumer that supports real MCP content (claude_sdk, the MCP + engine) can pass it through instead of json.dumps-ing it into text.""" + + _IMAGE_RESULT: ClassVar[dict[str, Any]] = { + "content": [{"type": "image", "data": "YmFzZTY0", "mimeType": "image/png"}] + } + _TEXT_RESULT: ClassVar[dict[str, Any]] = { + "name": "notes.txt", + "content_type": "text/plain", + "text": "hi", + } + + def test_true_for_image_content_block(self) -> None: + assert is_mcp_content_result(self._IMAGE_RESULT) + + def test_false_for_plain_dict(self) -> None: + assert not is_mcp_content_result(self._TEXT_RESULT) + + def test_false_for_non_dict(self) -> None: + assert not is_mcp_content_result("just a string") + + def test_false_for_content_list_without_type_keys(self) -> None: + assert not is_mcp_content_result({"content": [{"no_type": 1}]}) + + def test_false_for_empty_content_list(self) -> None: + """An empty list is vacuously all()-true; every consumer indexes + content[0] right after this check, so an empty list must not pass.""" + assert not is_mcp_content_result({"content": []}) diff --git a/tests/runtime/toolbehavior/test_matchesidentifier.py b/tests/runtime/toolbehavior/test_matchesidentifier.py new file mode 100644 index 000000000..3c2e68d79 --- /dev/null +++ b/tests/runtime/toolbehavior/test_matchesidentifier.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from band.runtime.tools import ( + matches_identifier, +) + + +class TestMatchesIdentifier: + """Tests for the matches_identifier helper.""" + + def test_match_by_handle(self): + entity = {"handle": "alice", "name": "Alice Smith", "id": "u-1"} + assert matches_identifier(entity, "alice") is True + + def test_match_by_name(self): + entity = {"handle": "alice", "name": "Alice Smith", "id": "u-1"} + assert matches_identifier(entity, "Alice Smith") is True + + def test_match_by_id(self): + entity = {"handle": "alice", "name": "Alice Smith", "id": "u-1"} + assert matches_identifier(entity, "u-1") is True + + def test_case_insensitive(self): + entity = {"handle": "Alice", "name": "ALICE SMITH", "id": "U-1"} + assert matches_identifier(entity, "alice") is True + assert matches_identifier(entity, "alice smith") is True + assert matches_identifier(entity, "u-1") is True + + def test_no_match(self): + entity = {"handle": "alice", "name": "Alice Smith", "id": "u-1"} + assert matches_identifier(entity, "bob") is False + + def test_missing_fields(self): + """Should handle entities with missing or None fields.""" + assert matches_identifier({"name": "Alice"}, "Alice") is True + assert matches_identifier({"handle": None, "name": "Alice"}, "Alice") is True + assert matches_identifier({}, "anything") is False + + def test_at_prefix_normalization(self): + """@alice and alice should match regardless of which side has the prefix.""" + entity_with_at = {"handle": "@alice", "name": "Alice Smith", "id": "u-1"} + entity_without_at = {"handle": "alice", "name": "Alice Smith", "id": "u-1"} + + # identifier has @, entity doesn't + assert matches_identifier(entity_without_at, "@alice") is True + # entity has @, identifier doesn't + assert matches_identifier(entity_with_at, "alice") is True + # both have @ + assert matches_identifier(entity_with_at, "@alice") is True + # neither has @ + assert matches_identifier(entity_without_at, "alice") is True + + def test_empty_identifier(self): + """Empty string should only match empty field values.""" + entity = {"handle": "alice", "name": "Alice", "id": "u-1"} + assert matches_identifier(entity, "") is False diff --git a/tests/runtime/toolbehavior/test_memorytools.py b/tests/runtime/toolbehavior/test_memorytools.py new file mode 100644 index 000000000..7e97ce623 --- /dev/null +++ b/tests/runtime/toolbehavior/test_memorytools.py @@ -0,0 +1,281 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest +from pydantic import ValidationError + +from band.client.rest import ( + DEFAULT_REQUEST_OPTIONS, + UnprocessableEntityError, +) +from band.core.exceptions import BandToolError +from band.core.memory_types import ORGANIZATION_SCOPE_REJECTED_CODE +from band.runtime.tools import ( + AgentTools, + StoreMemoryInput, +) + + +class TestMemoryTools: + @pytest.mark.asyncio + async def test_list_memories_omits_none_filters(self, mock_rest_client) -> None: + response = MagicMock() + response.data = [] + mock_rest_client.agent_api_memories.list_agent_memories = AsyncMock( + return_value=response + ) + tools = AgentTools("room-123", mock_rest_client) + + await tools.list_memories(page_size=25) + + mock_rest_client.agent_api_memories.list_agent_memories.assert_awaited_once_with( + page_size=25, + request_options=DEFAULT_REQUEST_OPTIONS, + ) + + @pytest.mark.asyncio + async def test_store_memory_omits_none_fields(self, mock_rest_client) -> None: + response = MagicMock() + response.data = MagicMock() + mock_rest_client.agent_api_memories.create_agent_memory = AsyncMock( + return_value=response + ) + tools = AgentTools("room-123", mock_rest_client) + + await tools.store_memory( + content="remember this", + system="working", + type="semantic", + segment="user", + thought="useful later", + scope="organization", + ) + + call_kwargs = ( + mock_rest_client.agent_api_memories.create_agent_memory.call_args.kwargs + ) + memory_payload = call_kwargs["memory"].model_dump(exclude_unset=True) + assert "subject_id" not in memory_payload + assert "metadata" not in memory_payload + assert call_kwargs["request_options"] is DEFAULT_REQUEST_OPTIONS + + @pytest.mark.asyncio + async def test_store_memory_agent_scope_sent_without_subject_id( + self, mock_rest_client + ) -> None: + """Agent scope round-trips to the REST payload untouched and needs no + subject_id -- the scope value real agents fall back to when their owner + has no organization.""" + response = MagicMock() + response.data = MagicMock() + mock_rest_client.agent_api_memories.create_agent_memory = AsyncMock( + return_value=response + ) + tools = AgentTools("room-123", mock_rest_client) + + await tools.store_memory( + content="remember this", + system="working", + type="semantic", + segment="user", + thought="useful later", + scope="agent", + ) + + call_kwargs = ( + mock_rest_client.agent_api_memories.create_agent_memory.call_args.kwargs + ) + memory_payload = call_kwargs["memory"].model_dump(exclude_unset=True) + assert memory_payload["scope"] == "agent" + assert "subject_id" not in memory_payload + + @pytest.mark.asyncio + async def test_list_memories_agent_scope_sent_untouched( + self, mock_rest_client + ) -> None: + """`scope="agent"` passes straight through to the list filter.""" + response = MagicMock() + response.data = [] + mock_rest_client.agent_api_memories.list_agent_memories = AsyncMock( + return_value=response + ) + tools = AgentTools("room-123", mock_rest_client) + + await tools.list_memories(scope="agent") + + mock_rest_client.agent_api_memories.list_agent_memories.assert_awaited_once_with( + page_size=50, + scope="agent", + request_options=DEFAULT_REQUEST_OPTIONS, + ) + + @pytest.mark.asyncio + async def test_store_memory_organization_scope_rejection_becomes_actionable( + self, mock_rest_client + ) -> None: + """The platform's 422 for an orgless agent's `scope="organization"` + write must reach the caller as guidance to retry with `scope="agent"`, + not as the raw `headers: ..., status_code: 422, body: ...` dump the + generic exception handler would otherwise produce.""" + rejection_body = { + "error": { + "code": ORGANIZATION_SCOPE_REJECTED_CODE, + "details": {"organization_id": "must be present"}, + } + } + mock_rest_client.agent_api_memories.create_agent_memory = AsyncMock( + side_effect=UnprocessableEntityError(body=rejection_body) + ) + tools = AgentTools("room-123", mock_rest_client) + + with pytest.raises(BandToolError, match='scope="agent"'): + await tools.store_memory( + content="remember this", + system="working", + type="semantic", + segment="user", + thought="useful later", + scope="organization", + ) + + @pytest.mark.asyncio + async def test_store_memory_unrelated_422_is_not_mistaken_for_scope_rejection( + self, mock_rest_client + ) -> None: + """A 422 for any other reason must propagate as-is -- rewriting it to + the scope-retry message would mislead the caller about the actual + failure.""" + mock_rest_client.agent_api_memories.create_agent_memory = AsyncMock( + side_effect=UnprocessableEntityError( + body={"error": {"code": "some_other_validation_failure"}} + ) + ) + tools = AgentTools("room-123", mock_rest_client) + + with pytest.raises(UnprocessableEntityError): + await tools.store_memory( + content="remember this", + system="working", + type="semantic", + segment="user", + thought="useful later", + scope="organization", + ) + + @pytest.mark.asyncio + async def test_list_memories_organization_scope_rejection_becomes_actionable( + self, mock_rest_client + ) -> None: + """The read-side twin of the store-side test above.""" + rejection_body = {"error": {"code": ORGANIZATION_SCOPE_REJECTED_CODE}} + mock_rest_client.agent_api_memories.list_agent_memories = AsyncMock( + side_effect=UnprocessableEntityError(body=rejection_body) + ) + tools = AgentTools("room-123", mock_rest_client) + + with pytest.raises(BandToolError, match='scope="agent"'): + await tools.list_memories(scope="organization") + + @pytest.mark.asyncio + async def test_store_memory_rejects_subject_scope_without_subject_id( + self, mock_rest_client + ) -> None: + """Reject subject-scoped writes before they reach the API.""" + mock_rest_client.agent_api_memories.create_agent_memory = AsyncMock() + tools = AgentTools("room-123", mock_rest_client) + + with pytest.raises(ValueError, match="requires a subject_id"): + await tools.store_memory( + content="remember this", + system="working", + type="semantic", + segment="user", + thought="useful later", + scope="subject", + ) + + mock_rest_client.agent_api_memories.create_agent_memory.assert_not_called() + + @pytest.mark.asyncio + async def test_store_memory_rejects_type_for_wrong_system( + self, mock_rest_client + ) -> None: + """Reject a system/type mismatch before it reaches the API -- the raw + tool method's ValueError comes straight from band_sdk_core, unlike + StoreMemoryInput's model_validate, which wraps it in a + pydantic.ValidationError.""" + mock_rest_client.agent_api_memories.create_agent_memory = AsyncMock() + tools = AgentTools("room-123", mock_rest_client) + + with pytest.raises( + ValueError, match="type `semantic` is not valid for system `sensory`" + ): + await tools.store_memory( + content="remember this", + system="sensory", + type="semantic", + segment="user", + thought="useful later", + scope="agent", + ) + + mock_rest_client.agent_api_memories.create_agent_memory.assert_not_called() + + def test_store_memory_input_rejects_subject_scope_without_subject_id(self) -> None: + """Validate tool input rejects subject scope without subject_id.""" + with pytest.raises(ValidationError, match="requires a subject_id"): + StoreMemoryInput.model_validate( + { + "content": "remember this", + "system": "working", + "type": "semantic", + "segment": "user", + "thought": "useful later", + "scope": "subject", + } + ) + + def test_store_memory_input_rejects_type_for_wrong_system(self) -> None: + """Validate memory type matches the chosen system.""" + with pytest.raises( + ValidationError, match="type `semantic` is not valid for system `sensory`" + ): + StoreMemoryInput.model_validate( + { + "content": "remember this", + "system": "sensory", + "type": "semantic", + "segment": "user", + "thought": "useful later", + "scope": "organization", + } + ) + + @pytest.mark.parametrize( + ("tool_method", "rest_method"), + [ + ("get_memory", "get_agent_memory"), + ("supersede_memory", "supersede_agent_memory"), + ("archive_memory", "archive_agent_memory"), + ], + ) + @pytest.mark.asyncio + async def test_memory_mutation_calls_use_default_request_options( + self, + mock_rest_client, + tool_method: str, + rest_method: str, + ) -> None: + response = MagicMock() + response.data = MagicMock() + rest_call = AsyncMock(return_value=response) + setattr(mock_rest_client.agent_api_memories, rest_method, rest_call) + tools = AgentTools("room-123", mock_rest_client) + + await getattr(tools, tool_method)("mem-123") + + rest_call.assert_awaited_once_with( + id="mem-123", + request_options=DEFAULT_REQUEST_OPTIONS, + ) diff --git a/tests/runtime/toolbehavior/test_memoryvalidationpreventshttp.py b/tests/runtime/toolbehavior/test_memoryvalidationpreventshttp.py new file mode 100644 index 000000000..500568725 --- /dev/null +++ b/tests/runtime/toolbehavior/test_memoryvalidationpreventshttp.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import pytest + +from band.runtime.tools import ( + AgentTools, +) +from tests.identifiers import INVALID_IDS +from tests.runtime.helpers import memory_client + + +class TestMemoryValidationPreventsHttp: + @pytest.mark.parametrize( + "name", ["band_get_memory", "band_supersede_memory", "band_archive_memory"] + ) + @pytest.mark.parametrize("memory_id", INVALID_IDS) + @pytest.mark.asyncio + async def test_memory_validation_prevents_http( + self, name: str, memory_id: str + ) -> None: + async with memory_client() as (rest, requests): + outcome = await AgentTools("room-1", rest).execute_tool_call_structured( + name, {"memory_id": memory_id} + ) + assert not outcome.ok + assert f"Invalid arguments for {name}" in outcome.error_message + assert "memory_id:" in outcome.error_message + assert requests == [] diff --git a/tests/runtime/toolbehavior/test_mentionresolution.py b/tests/runtime/toolbehavior/test_mentionresolution.py new file mode 100644 index 000000000..29311b8d5 --- /dev/null +++ b/tests/runtime/toolbehavior/test_mentionresolution.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import pytest + +from band.runtime.tools import ( + AgentTools, +) + + +class TestMentionResolution: + """Test mention resolution logic.""" + + def test_resolve_string_mentions(self, mock_rest_client, participants): + """Should resolve string mentions to dicts with id and handle.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + resolved = tools._resolve_mentions(["User One", "User Two"]) + + assert len(resolved) == 2 + assert resolved[0] == {"id": "user-1", "handle": "@user-one"} + assert resolved[1] == {"id": "user-2", "handle": "@user-two"} + + def test_resolve_dict_mentions_with_id(self, mock_rest_client, participants): + """Should pass through dict mentions with ID and handle.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + resolved = tools._resolve_mentions([{"id": "custom-id", "handle": "@custom"}]) + + assert resolved[0] == {"id": "custom-id", "handle": "@custom"} + + def test_resolve_dict_mentions_without_id(self, mock_rest_client, participants): + """Should resolve dict mentions without ID by name lookup.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + resolved = tools._resolve_mentions([{"name": "User One"}]) + + assert resolved[0] == {"id": "user-1", "handle": "@user-one"} + + def test_resolve_unknown_raises(self, mock_rest_client, participants): + """Should raise for unknown mention.""" + tools = AgentTools("room-123", mock_rest_client, participants) + + with pytest.raises(ValueError, match="Unknown participant"): + tools._resolve_mentions(["Unknown Person"]) diff --git a/tests/runtime/toolbehavior/test_replyeffectspellings.py b/tests/runtime/toolbehavior/test_replyeffectspellings.py new file mode 100644 index 000000000..81eaa9216 --- /dev/null +++ b/tests/runtime/toolbehavior/test_replyeffectspellings.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from band.runtime.tools import ( + TurnEffect, + turn_effect, +) + + +class TestReplyEffectSpellings: + """Which tool names resolve to Band's reply effect.""" + + def test_sdk_injected_tool(self): + assert turn_effect("band_send_message") is TurnEffect.REPLY + + def test_standalone_band_mcp_tool(self): + assert turn_effect("create_agent_chat_message") is TurnEffect.REPLY + + def test_mcp_server_prefixed_names(self): + """MCP clients may prefix the server name onto the tool name.""" + assert turn_effect("band-band_send_message") is TurnEffect.REPLY + assert turn_effect("band-create_agent_chat_message") is TurnEffect.REPLY + + def test_non_settling_tools(self): + assert turn_effect("band_send_event") is TurnEffect.OBSERVE + assert turn_effect("band_lookup_peers") is TurnEffect.OBSERVE + assert turn_effect("get_weather") is TurnEffect.OBSERVE + + def test_no_substring_false_positive(self): + """Only an exact or server-prefixed match counts, not any substring.""" + assert turn_effect("band_send_message_draft") is TurnEffect.OBSERVE + + def test_non_band_server_prefix_does_not_resolve(self): + """An unrelated MCP server's own tool must never be treated as a Band + reply just because it ends in ``-band_send_message``.""" + assert turn_effect("other-band_send_message") is TurnEffect.OBSERVE diff --git a/tests/runtime/toolbehavior/test_toolinputmodels.py b/tests/runtime/toolbehavior/test_toolinputmodels.py new file mode 100644 index 000000000..7c580b680 --- /dev/null +++ b/tests/runtime/toolbehavior/test_toolinputmodels.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from band.runtime.tools import ( + AddParticipantInput, + CreateChatroomInput, + GetParticipantsInput, + LookupPeersInput, + RemoveParticipantInput, + SendEventInput, + SendMessageInput, + SendRoomFileInput, +) + + +class TestToolInputModels: + """Test Pydantic tool input models.""" + + def test_send_message_input_validation(self): + """SendMessageInput should validate fields.""" + model = SendMessageInput(content="Hello", mentions=["User"]) + assert model.content == "Hello" + assert model.mentions == ["User"] + + def test_send_message_input_accepts_empty_mentions(self): + """SendMessageInput allows empty mentions (runtime validates instead).""" + model = SendMessageInput(content="Hello", mentions=[]) + assert model.mentions == [] + + def test_send_room_file_input_requires_mentions(self): + """band_send_room_file posts a message to attach the file, so its + schema must require mentions just like SendMessageInput -- an + optional-looking field here would let the LLM omit it, upload the + file, and only then discover the platform's mention requirement.""" + with pytest.raises(ValidationError): + SendRoomFileInput(content="hi", filename="f.txt") + + def test_send_room_file_input_accepts_empty_mentions(self): + """Present but empty is still valid at the schema level (runtime + validates the "at least one" rule instead, same as SendMessageInput).""" + model = SendRoomFileInput(content="hi", filename="f.txt", mentions=[]) + assert model.mentions == [] + + def test_send_event_input_validation(self): + """SendEventInput should validate fields.""" + model = SendEventInput(content="Thinking", message_type="thought") + assert model.message_type == "thought" + + def test_send_event_input_validates_type(self): + """SendEventInput should validate message_type.""" + with pytest.raises(ValidationError, match="literal_error"): + SendEventInput(content="Test", message_type="invalid") + + def test_add_participant_input_defaults(self): + """AddParticipantInput should have default role.""" + model = AddParticipantInput(identifier="User") + assert model.role == "member" + + def test_add_participant_input_accepts_legacy_name_field(self): + """AddParticipantInput should accept 'name' as alias for backward compat.""" + model = AddParticipantInput.model_validate({"name": "Agent Two"}) + assert model.identifier == "Agent Two" + + def test_remove_participant_input_accepts_legacy_name_field(self): + """RemoveParticipantInput should accept 'name' as alias for backward compat.""" + + model = RemoveParticipantInput.model_validate({"name": "User One"}) + assert model.identifier == "User One" + + def test_lookup_peers_input_defaults(self): + """LookupPeersInput should have defaults.""" + model = LookupPeersInput() + assert model.page == 1 + assert model.page_size == 50 + + def test_get_participants_input_no_fields(self): + """GetParticipantsInput should have no required fields.""" + model = GetParticipantsInput() + assert model is not None + + def test_create_chatroom_input_validation(self): + """CreateChatroomInput should allow optional task_id.""" + model = CreateChatroomInput(task_id="task-123") + assert model.task_id == "task-123" + + def test_create_chatroom_input_no_task_id(self): + """CreateChatroomInput should work without task_id.""" + model = CreateChatroomInput() + assert model.task_id is None diff --git a/tests/runtime/toolbehavior/test_unboundstoppedrefusalescapestoolerrorconversion.py b/tests/runtime/toolbehavior/test_unboundstoppedrefusalescapestoolerrorconversion.py new file mode 100644 index 000000000..88f1ebd46 --- /dev/null +++ b/tests/runtime/toolbehavior/test_unboundstoppedrefusalescapestoolerrorconversion.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import pytest +from band_rest.errors import ForbiddenError +from band_rest.types import Error, ErrorError +from band_sdk_core import AgentFailure + +from band.core.exceptions import RoomExecutionStoppedError +from band.runtime.tools import ( + AgentTools, +) + + +class TestUnboundStoppedRefusalEscapesToolErrorConversion: + @pytest.mark.parametrize("boundary", ["dispatcher", "failure"]) + async def test_unbound_stopped_refusal_escapes_tool_error_conversion( + self, mock_rest_client, boundary: str + ) -> None: + tools = AgentTools("room-123", mock_rest_client) + mock_rest_client.agent_api_events.create_agent_chat_event.side_effect = ( + ForbiddenError( + body=Error( + error=ErrorError( + code="forbidden", + message="Agent execution is stopped; cannot post events", + request_id="req-1", + ) + ) + ) + ) + with pytest.raises(RoomExecutionStoppedError): + if boundary == "dispatcher": + await tools.execute_tool_call_structured( + "band_send_event", {"content": "thought", "message_type": "thought"} + ) + else: + await tools.send_failure(AgentFailure("provider", "failed")) + assert not tools.turn.complete diff --git a/tests/runtime/toolbehavior/test_validmemorydispatchaddressesitem.py b/tests/runtime/toolbehavior/test_validmemorydispatchaddressesitem.py new file mode 100644 index 000000000..e242a3b0c --- /dev/null +++ b/tests/runtime/toolbehavior/test_validmemorydispatchaddressesitem.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import pytest + +from band.runtime.tools import ( + AgentTools, +) +from tests.identifiers import UUID_ID +from tests.runtime.helpers import memory_client + + +class TestValidMemoryDispatchAddressesItem: + @pytest.mark.parametrize( + "name,method,suffix", + [ + ("band_get_memory", "GET", ""), + ("band_supersede_memory", "POST", "/supersede"), + ("band_archive_memory", "POST", "/archive"), + ], + ) + @pytest.mark.asyncio + async def test_valid_memory_dispatch_addresses_item( + self, name: str, method: str, suffix: str + ) -> None: + async with memory_client() as (rest, requests): + outcome = await AgentTools("room-1", rest).execute_tool_call_structured( + name, {"memory_id": UUID_ID} + ) + assert outcome.ok, outcome.error_message + assert outcome.value["id"] == UUID_ID + assert outcome.value["content"] == "remember this" + assert [(q.method, q.url.path) for q in requests] == [ + (method, f"/api/v1/agent/memories/{UUID_ID}{suffix}") + ] From a674b67f6deeea036f7f4a55bf9a34b5d7e935f2 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Sun, 18 Oct 2026 12:22:55 +0300 Subject: [PATCH 15/47] refactor: Share activity observation records and assertions Give the repeated tool-completion and privacy checks one helper, and name the ACP call record so activity identity is not a bare tuple. Co-authored-by: Cursor --- src/band/adapters/copilot_sdk.py | 26 +++++----- src/band/adapters/letta.py | 8 ++-- src/band/adapters/opencode/adapter.py | 48 +++++++++++-------- src/band/integrations/acp/collecting.py | 29 +++++++---- src/band/runtime/working_state.py | 3 +- src/band/testing/activity.py | 6 +++ tests/adapters/activity/test_acp.py | 5 +- tests/adapters/activity/test_execution.py | 5 +- tests/adapters/activity/test_model_event.py | 5 +- tests/adapters/activity/test_native.py | 5 +- .../adapters/activity/test_opencode_waits.py | 1 - tests/adapters/agno/adapter/test_activity.py | 5 +- tests/adapters/strands/activity/test_hooks.py | 5 +- .../crewai/activity/test_execution.py | 5 +- .../parlant/activity/test_execution.py | 10 +--- tests/runtime/activityownership/kit.py | 11 +++++ .../activityownership/test_claude_native.py | 13 +---- .../test_detached_detail_feature.py | 13 +---- tests/toolkit/activity/test_subscription.py | 1 - 19 files changed, 95 insertions(+), 109 deletions(-) diff --git a/src/band/adapters/copilot_sdk.py b/src/band/adapters/copilot_sdk.py index 636cb3cbe..0e8273fb0 100644 --- a/src/band/adapters/copilot_sdk.py +++ b/src/band/adapters/copilot_sdk.py @@ -630,12 +630,13 @@ def _session_kwargs( return kwargs def _activity_user_input_handler(self, room_id: str) -> UserInputHandler: - handler = ( - self._ask_user - if callable(self._ask_user) - else self._make_room_ask_user_handler(room_id) - ) - manual = callable(self._ask_user) + ask_user = self._ask_user + if callable(ask_user): + handler: UserInputHandler = ask_user + manual = True + else: + handler = self._make_room_ask_user_handler(room_id) + manual = False pending = 0 async def handle( @@ -647,16 +648,15 @@ async def handle( result = handler(request, context) if not inspect.isawaitable(result): return result - if manual: - pending += 1 + if not manual: + return await result + pending += 1 try: - if manual: - await set_permission_wait(tools, True) + await set_permission_wait(tools, True) return await result finally: - if manual: - pending -= 1 - await set_permission_wait(tools, pending > 0) + pending -= 1 + await set_permission_wait(tools, pending > 0) return handle diff --git a/src/band/adapters/letta.py b/src/band/adapters/letta.py index 33628f57f..b31d141d2 100644 --- a/src/band/adapters/letta.py +++ b/src/band/adapters/letta.py @@ -532,14 +532,16 @@ async def _call_provider( room_ctx: RoomContext, tools: AgentToolsProtocol, ) -> tuple[list[Any], TurnUsage]: - """Round-trip to the Letta API -- no response processing or delivery.""" + """Stream the provider response and return its messages and usage. + + Records tool activity as events arrive. Does not narrate or deliver. + """ # Use Conversations API in shared mode, direct agent API in per_room mode if self.config.mode == "shared" and room_ctx.conversation_id: - conversation_stream = await self._client.conversations.messages.create( + stream = await self._client.conversations.messages.create( conversation_id=room_ctx.conversation_id, messages=messages, ) - stream = conversation_stream per_room = False else: stream = await self._client.agents.messages.create( diff --git a/src/band/adapters/opencode/adapter.py b/src/band/adapters/opencode/adapter.py index d12aa0afc..01ed2ebf9 100644 --- a/src/band/adapters/opencode/adapter.py +++ b/src/band/adapters/opencode/adapter.py @@ -74,6 +74,10 @@ logger = logging.getLogger(__name__) +_ENDED_TOOL_ACTIVITY = frozenset( + {OpencodeToolStatus.COMPLETED, OpencodeToolStatus.ERROR} +) + _PROVIDER = "opencode" _OPENCODE_SYSTEM_NOTE = """\ @@ -893,33 +897,35 @@ async def _handle_part_update( case "tool": await self._report_tool_part(room_state, part) + async def _observe_tool_activity( + self, room_state: RoomState, part: OpencodePart + ) -> None: + turn = room_state.turn + state = part.state + if turn is None or part.id is None or state is None: + return + call_id = part.call_id or part.id + previous = turn.activity_tool_status.get(call_id) + status = state.status + turn.activity_tool_status[call_id] = status + if ( + status in {OpencodeToolStatus.PENDING, OpencodeToolStatus.RUNNING} + and previous is None + ): + await report_tool_activity( + turn.tools, self._canonical_tool_name(part.tool or "") + ) + return + if status in _ENDED_TOOL_ACTIVITY and previous not in _ENDED_TOOL_ACTIVITY: + await end_tool_activity(turn.tools) + async def _report_tool_part( self, room_state: RoomState, part: OpencodePart ) -> None: """Report the tool's call/result.""" if part.state is None: return - turn = room_state.turn - if turn is not None and part.id is not None: - call_id = part.call_id or part.id - previous = turn.activity_tool_status.get(call_id) - status = part.state.status - turn.activity_tool_status[call_id] = status - if ( - status in {OpencodeToolStatus.PENDING, OpencodeToolStatus.RUNNING} - and previous is None - ): - await report_tool_activity( - turn.tools, self._canonical_tool_name(part.tool or "") - ) - elif status in { - OpencodeToolStatus.COMPLETED, - OpencodeToolStatus.ERROR, - } and previous not in { - OpencodeToolStatus.COMPLETED, - OpencodeToolStatus.ERROR, - }: - await end_tool_activity(turn.tools) + await self._observe_tool_activity(room_state, part) if Emit.TOOL_CALLS not in self.features.emit: return diff --git a/src/band/integrations/acp/collecting.py b/src/band/integrations/acp/collecting.py index 9e2165c78..8287d7e8c 100644 --- a/src/band/integrations/acp/collecting.py +++ b/src/band/integrations/acp/collecting.py @@ -5,6 +5,7 @@ import asyncio import logging from collections.abc import Awaitable, Callable, Sequence +from typing import NamedTuple from acp.interfaces import Client from acp.schema import ConfigOptionUpdate, DeclineElicitationResponse @@ -35,6 +36,17 @@ logger = logging.getLogger(__name__) +class _ActivityCall(NamedTuple): + name: str + ended: bool + + +def _recorded_activity_name(call: _ActivityCall | None) -> str: + if call is None: + return UNKNOWN_ACTIVITY_TOOL_NAME + return call.name + + class ACPCollectingClient(ACPUpdateParser, Client): # type: ignore[misc] # ACP Client has optional methods treated as abstract by pyrefly """Generic ACP client that buffers session updates by session_id. @@ -56,7 +68,7 @@ def __init__( self._permission_handlers: dict[str, PermissionHandler] = {} self._elicitation_handlers: dict[str, ElicitationHandler] = {} self._activity_tools: dict[str, AgentToolsProtocol] = {} - self._activity_calls: dict[str, dict[str, tuple[str, bool]]] = {} + self._activity_calls: dict[str, dict[str, _ActivityCall]] = {} # Retain locks while a straggler can still hold one. self._session_locks: dict[str, asyncio.Lock] = {} # Each session's latest advertised config catalog; like the locks, @@ -116,21 +128,18 @@ async def _observe_activity( previous = calls.get(call_id) status = getattr(update, "status", None) if status in {ToolStatus.COMPLETED, ToolStatus.FAILED}: - if previous is None or not previous[1]: - calls[call_id] = ( - previous[0] if previous else UNKNOWN_ACTIVITY_TOOL_NAME, - True, - ) + if previous is None or not previous.ended: + calls[call_id] = _ActivityCall(_recorded_activity_name(previous), True) await end_tool_activity(tools) return - if previous is not None and previous[1]: + if previous is not None and previous.ended: return - name = previous[0] if previous else UNKNOWN_ACTIVITY_TOOL_NAME + name = _recorded_activity_name(previous) call = chunk.tool if isinstance(call, ACPToolCall) and call.mcp_server is not None: name = call.name - if previous is None or previous[0] != name: - calls[call_id] = (name, False) + if previous is None or previous.name != name: + calls[call_id] = _ActivityCall(name, False) await report_tool_activity(tools, name) async def request_permission( # type: ignore[override] # ACP Client uses specific types; we widen to object diff --git a/src/band/runtime/working_state.py b/src/band/runtime/working_state.py index e05aff083..6723e994b 100644 --- a/src/band/runtime/working_state.py +++ b/src/band/runtime/working_state.py @@ -109,8 +109,7 @@ def __init__( self._active = False self._cycle: object | None = None - self._ordinary = AgentActivityDetail(kind="reasoning") - self._manual_wait = False + self._reset_detail() self._started_at: float | None = None self._task: asyncio.Task[None] | None = None self._false_send: asyncio.Future[None] | None = None diff --git a/src/band/testing/activity.py b/src/band/testing/activity.py index d8022b986..3f8610a7b 100644 --- a/src/band/testing/activity.py +++ b/src/band/testing/activity.py @@ -23,3 +23,9 @@ async def update(self, activity: AgentActivityDetail) -> None: async def set_manual_wait(self, waiting: bool) -> None: self.events.append({"waiting": waiting}) + + def assert_completed_tool(self, name: str) -> None: + assert self.events == [ + {"kind": "tool_call", "name": name}, + {"kind": "reasoning"}, + ] diff --git a/tests/adapters/activity/test_acp.py b/tests/adapters/activity/test_acp.py index 5e4de2690..a5e0d9aab 100644 --- a/tests/adapters/activity/test_acp.py +++ b/tests/adapters/activity/test_acp.py @@ -43,10 +43,7 @@ async def test_activity_precedes_buffered_narration_and_ignores_private_titles( sessionUpdate="tool_call_update", toolCallId="call", status="completed" ), ) - assert recorder.events == [ - {"kind": "tool_call", "name": "unknown"}, - {"kind": "reasoning"}, - ] + recorder.assert_completed_tool("unknown") client.set_activity_tools("session", None) await client.session_update( "session", diff --git a/tests/adapters/activity/test_execution.py b/tests/adapters/activity/test_execution.py index 2a964e72d..8e681e992 100644 --- a/tests/adapters/activity/test_execution.py +++ b/tests/adapters/activity/test_execution.py @@ -60,8 +60,5 @@ async def lookup(args: ActivityLookup) -> str: else: release.set() await task - assert recorder.events == [ - {"kind": "tool_call", "name": name}, - {"kind": "reasoning"}, - ] + recorder.assert_completed_tool(name) assert not tools.events_sent diff --git a/tests/adapters/activity/test_model_event.py b/tests/adapters/activity/test_model_event.py index 44a3a3cd9..cc2db0cee 100644 --- a/tests/adapters/activity/test_model_event.py +++ b/tests/adapters/activity/test_model_event.py @@ -32,7 +32,4 @@ async def test_model_event_reports_while_raw_telemetry_stays_quiet( await dispatch( RunContext(deps=tools, model=TestModel(), usage=RunUsage()), **arguments ) - assert recorder.events == [ - {"kind": "tool_call", "name": BandTool.SEND_EVENT}, - {"kind": "reasoning"}, - ] + recorder.assert_completed_tool(BandTool.SEND_EVENT) diff --git a/tests/adapters/activity/test_native.py b/tests/adapters/activity/test_native.py index 10fee1729..12d3fb0dd 100644 --- a/tests/adapters/activity/test_native.py +++ b/tests/adapters/activity/test_native.py @@ -91,8 +91,5 @@ async def test_start_and_end_are_independent_of_narration( ), tools, ) - assert recorder.events == [ - {"kind": "tool_call", "name": "Bash"}, - {"kind": "reasoning"}, - ] + recorder.assert_completed_tool("Bash") assert not tools.events_sent diff --git a/tests/adapters/activity/test_opencode_waits.py b/tests/adapters/activity/test_opencode_waits.py index 4e6d834bb..558ff150e 100644 --- a/tests/adapters/activity/test_opencode_waits.py +++ b/tests/adapters/activity/test_opencode_waits.py @@ -66,5 +66,4 @@ async def held_reply( await first approvals.cancel() assert not approvals.awaiting_human() - assert not recorder.waiting assert recorder.events[-1] == {"waiting": False} diff --git a/tests/adapters/agno/adapter/test_activity.py b/tests/adapters/agno/adapter/test_activity.py index 1092a4426..dd1396649 100644 --- a/tests/adapters/agno/adapter/test_activity.py +++ b/tests/adapters/agno/adapter/test_activity.py @@ -40,10 +40,7 @@ async def test_streamed_tool_activity_preserves_final_response_with_or_without_n is_session_bootstrap=True, room_id="room-1", ) - assert recorder.events == [ - {"kind": "tool_call", "name": "shell"}, - {"kind": "reasoning"}, - ] + recorder.assert_completed_tool("shell") assert not tools.messages_sent assert adapter._message_history["room-1"][-1].content == "final reply" assert bool(tools.events_sent) is narrated diff --git a/tests/adapters/strands/activity/test_hooks.py b/tests/adapters/strands/activity/test_hooks.py index b662fd503..daeec28c1 100644 --- a/tests/adapters/strands/activity/test_hooks.py +++ b/tests/adapters/strands/activity/test_hooks.py @@ -42,8 +42,5 @@ async def test_native_hook_reports_without_narration(self, status: str) -> None: }, ) ) - assert recorder.events == [ - {"kind": "tool_call", "name": "shell"}, - {"kind": "reasoning"}, - ] + recorder.assert_completed_tool("shell") assert not tools.events_sent diff --git a/tests/integrations/crewai/activity/test_execution.py b/tests/integrations/crewai/activity/test_execution.py index 0207c60f1..6f532ac4c 100644 --- a/tests/integrations/crewai/activity/test_execution.py +++ b/tests/integrations/crewai/activity/test_execution.py @@ -40,8 +40,5 @@ async def lookup(args: LookupInput) -> str: lookup_tool = next(tool for tool in built if tool.name == "lookup") result = json.loads(lookup_tool.run(note="private-argument")) assert result["status"] == ("error" if failed else "success") - assert recorder.events == [ - {"kind": "tool_call", "name": "lookup"}, - {"kind": "reasoning"}, - ] + recorder.assert_completed_tool("lookup") assert not tools.events_sent diff --git a/tests/integrations/parlant/activity/test_execution.py b/tests/integrations/parlant/activity/test_execution.py index eb689ff79..1b2b5e1ed 100644 --- a/tests/integrations/parlant/activity/test_execution.py +++ b/tests/integrations/parlant/activity/test_execution.py @@ -57,10 +57,7 @@ async def lookup(args: LookupInput) -> str: assert result == ( "Error running lookup: private-error" if failed else "private-result" ) - assert recorder.events == [ - {"kind": "tool_call", "name": "lookup"}, - {"kind": "reasoning"}, - ] + recorder.assert_completed_tool("lookup") assert not tools.events_sent async def test_custom_handler_cancellation_ends_activity(self) -> None: @@ -88,7 +85,4 @@ async def lookup(args: LookupInput) -> str: task.cancel() with pytest.raises(asyncio.CancelledError): await task - assert recorder.events == [ - {"kind": "tool_call", "name": "lookup"}, - {"kind": "reasoning"}, - ] + recorder.assert_completed_tool("lookup") diff --git a/tests/runtime/activityownership/kit.py b/tests/runtime/activityownership/kit.py index e97933906..a5a954730 100644 --- a/tests/runtime/activityownership/kit.py +++ b/tests/runtime/activityownership/kit.py @@ -40,6 +40,17 @@ async def respond(self, request: httpx.Request) -> httpx.Response: raise return response + def assert_hides(self, *private: str) -> None: + serialized = json.dumps(self.activity_bodies) + for value in private: + assert value not in serialized + + def assert_no_tool_narration(self) -> None: + assert not any( + body.get("message_type") in {"tool_call", "tool_result"} + for body in self.event_bodies + ) + def answer(self, request: httpx.Request) -> httpx.Response: tail = request.url.path.rsplit("/", 1)[-1] if tail == "events" and request.method == "POST": diff --git a/tests/runtime/activityownership/test_claude_native.py b/tests/runtime/activityownership/test_claude_native.py index 016c216a0..4718cda0e 100644 --- a/tests/runtime/activityownership/test_claude_native.py +++ b/tests/runtime/activityownership/test_claude_native.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json from collections.abc import Callable import pytest @@ -61,15 +60,7 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: "activity": {"kind": "tool_call", "name": "Bash"}, } in activity_platform.activity_bodies assert activity_platform.activity_bodies[-1] == {"working": False} - assert not any( - body.get("message_type") in {"tool_call", "tool_result"} - for body in activity_platform.event_bodies - ) - assert "private-argument" not in json.dumps( - activity_platform.activity_bodies - ) - assert "private-result" not in json.dumps( - activity_platform.activity_bodies - ) + activity_platform.assert_no_tool_narration() + activity_platform.assert_hides("private-argument", "private-result") finally: await adapter.cleanup_all() diff --git a/tests/runtime/activityownership/test_detached_detail_feature.py b/tests/runtime/activityownership/test_detached_detail_feature.py index 455268469..4d7481aef 100644 --- a/tests/runtime/activityownership/test_detached_detail_feature.py +++ b/tests/runtime/activityownership/test_detached_detail_feature.py @@ -1,7 +1,6 @@ from __future__ import annotations import asyncio -import json from collections.abc import Callable from typing import Any @@ -155,20 +154,12 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: activity_platform.activity_bodies.count({"working": False}) == 1 ) assert not client.aborted_sessions - assert "private-argument" not in json.dumps( - activity_platform.activity_bodies - ) - assert "private-result" not in json.dumps( - activity_platform.activity_bodies - ) + activity_platform.assert_hides("private-argument", "private-result") before = list(activity_platform.activity_bodies) await report_tool_activity(captured[0], "late-old-callback") assert activity_platform.activity_bodies == before if variant == "silent": - assert not any( - event.get("message_type") in {"tool_call", "tool_result"} - for event in activity_platform.event_bodies - ) + activity_platform.assert_no_tool_narration() finally: release.set() await adapter.on_cleanup(ROOM_ID) diff --git a/tests/toolkit/activity/test_subscription.py b/tests/toolkit/activity/test_subscription.py index 8a9d2b7b5..2a1d73fa9 100644 --- a/tests/toolkit/activity/test_subscription.py +++ b/tests/toolkit/activity/test_subscription.py @@ -37,7 +37,6 @@ async def test_joins_before_trigger_dispatches_and_leaves(self) -> None: {"agent_id": "agent", "working": False}, ) await capture.wait_for_activity_stop("agent", since=mark) - capture.activities.assert_names_only("private-input", "private-output") assert topic not in ws.joined_topics() assert "chat_room:room" not in ws.joined_topics() From 2d8c903a789c8771289d5a11ea21d980ff54cb43 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Tue, 20 Oct 2026 13:29:35 +0300 Subject: [PATCH 16/47] test: give the Strands/NOOA job coverage guard a longer timeout It runs three collect-only subprocesses, which exceeded the default per-test timeout on Windows runners. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01AgnNR4uiqD787Bg3yKdZpL --- tests/framework_conformance/test_strands_nooa_job_coverage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/framework_conformance/test_strands_nooa_job_coverage.py b/tests/framework_conformance/test_strands_nooa_job_coverage.py index 715f21408..6f37efed9 100644 --- a/tests/framework_conformance/test_strands_nooa_job_coverage.py +++ b/tests/framework_conformance/test_strands_nooa_job_coverage.py @@ -32,6 +32,8 @@ def test_expected_configs_are_present(self, framework_environment: None) -> None assert expected <= {config.framework_id for config in ADAPTER_CONFIGS} assert expected <= {config.framework_id for config in CONVERTER_CONFIGS} + # Three collect-only subprocesses; each is slow to start on Windows runners. + @pytest.mark.timeout(120) def test_job_selects_every_required_node(self, framework_environment: None) -> None: needed = coverage.tests_needing_venv( coverage.needs_venv_pattern(frozenset({"strands", "nooa"})) From 3d52a139554fd463b947e07b9ce2b152c9eea7e8 Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Tue, 20 Oct 2026 14:35:44 +0300 Subject: [PATCH 17/47] feat: gate activity tool names on the tool-call emit choice Adapters that do not report tool calls must not leak tool names through the live room indicator. The decision is taken once per turn in SimpleAdapter from the negotiated emit set and enforced at the single observer every activity reporter funnels through, so no adapter can bypass it. Permission and memory detail name no content and always report. Guards: tests/runtime/activity/test_namegate.py pins that a tool_call frame reaches the wire with Emit.TOOL_CALLS and degrades to nameless reasoning without it; fails if the wrapper is removed. --- src/band/core/activity.py | 52 ++++++++++++ src/band/core/simple_adapter.py | 4 + src/band/runtime/execution.py | 24 ++++-- tests/runtime/activity/test_namegate.py | 108 ++++++++++++++++++++++++ 4 files changed, 183 insertions(+), 5 deletions(-) create mode 100644 src/band/core/activity.py create mode 100644 tests/runtime/activity/test_namegate.py diff --git a/src/band/core/activity.py b/src/band/core/activity.py new file mode 100644 index 000000000..74bf0f090 --- /dev/null +++ b/src/band/core/activity.py @@ -0,0 +1,52 @@ +"""Room-visible activity detail and the choice that governs tool names. + +A tool name reaches the room the same way its narration does: an agent that +does not report tool calls must not leak their names through the live +indicator. Every reporter funnels through the execution context's observer, +so the decision is applied once there and never per adapter. +""" + +from __future__ import annotations + +from collections.abc import Collection +from typing import TYPE_CHECKING + +from band_rest.types.agent_activity_detail import AgentActivityDetail + +from band.core.types import Emit + +if TYPE_CHECKING: + from band.runtime.working_state import ActivityObserver + + +class ToolNamesObserver: + """Hide tool names while leaving every other activity detail intact. + + Permission waits and memory activity name no content, so they report + whatever the tool-name choice is. + """ + + def __init__(self, inner: ActivityObserver) -> None: + self._inner = inner + + async def update(self, activity: AgentActivityDetail) -> None: + if activity.kind == "tool_call": + await self._inner.update(AgentActivityDetail(kind="reasoning")) + return + await self._inner.update(activity) + + async def set_manual_wait(self, waiting: bool) -> None: + await self._inner.set_manual_wait(waiting) + + +def sync_activity_tool_names(tools: object, emit: Collection[Emit]) -> None: + """Push the adapter's tool-narration choice onto the bound context. + + ``Emit.TOOL_CALLS`` absent means the room sees ``reasoning`` where a tool + runs. Tools without an execution context (fakes, unbound instances) have + no room indicator to gate. + """ + ctx = getattr(tools, "_ctx", None) + setter = getattr(ctx, "set_activity_tool_names", None) + if setter is not None: + setter(Emit.TOOL_CALLS in emit) diff --git a/src/band/core/simple_adapter.py b/src/band/core/simple_adapter.py index 39623e2a0..4d8d0c516 100644 --- a/src/band/core/simple_adapter.py +++ b/src/band/core/simple_adapter.py @@ -12,6 +12,7 @@ from band.client.rest import AsyncRestClient from band.client.streaming import ControlMode +from band.core.activity import sync_activity_tool_names from band.core.disposal import AdapterDisposal, CleanupCallback from band.core.exceptions import BandConfigError, InputNotAccepted from band.core.handoff import context_handoff @@ -441,6 +442,9 @@ async def on_event(self, inp: AgentInput) -> None: else: converted_history = inp.history + # One turn, one gate decision: the live tool-name detail follows + # the same choice as tool-call narration (see runtime/activity.py). + sync_activity_tool_names(inp.tools, self.features.emit) await self.run_judged_turn( msg=inp.msg, tools=inp.tools, diff --git a/src/band/runtime/execution.py b/src/band/runtime/execution.py index e792c3f8f..c9f98dac1 100644 --- a/src/band/runtime/execution.py +++ b/src/band/runtime/execution.py @@ -47,6 +47,7 @@ MessageCreatedPayload, MessageMetadata, ) +from band.core.activity import ToolNamesObserver from band.core.exceptions import ( CleanupIncompleteError, InputNotAccepted, @@ -431,6 +432,10 @@ def __init__( self._activity_owner: TurnScope | None = None self._activity_stop_scope: TurnScope | None = None self._activity_completion: asyncio.Task[None] | None = None + # Tool names follow the adapter's tool-narration choice; the default + # matches emit's opt-out semantics, so only adapters that drop + # Emit.TOOL_CALLS (via sync_activity_tool_names) hide them. + self._activity_tool_names = True self._control_revision = 0 self._on_platform_stop = on_platform_stop self._shutdown_task: asyncio.Task[bool] | None = None @@ -1961,12 +1966,21 @@ async def _start_activity(self, scope: TurnScope) -> None: self._activity_owner = scope await self._working_reporter.start(owner=scope) + def set_activity_tool_names(self, enabled: bool) -> None: + """Gate tool names in room activity to match tool-call narration. + + Permission waits and memory activity are unaffected: they name no + content. Observers created after a change pick it up immediately. + """ + self._activity_tool_names = enabled + def activity_observer(self, scope: TurnScope | None) -> ActivityObserver | None: - return ( - ScopeActivityObserver(self, scope) - if scope is not None and scope.activity_allowed - else None - ) + if scope is None or not scope.activity_allowed: + return None + observer: ActivityObserver = ScopeActivityObserver(self, scope) + if not self._activity_tool_names: + observer = ToolNamesObserver(observer) + return observer def owns_activity(self, scope: TurnScope) -> bool: return self._activity_owner is scope and not scope.aborted and not self._closing diff --git a/tests/runtime/activity/test_namegate.py b/tests/runtime/activity/test_namegate.py new file mode 100644 index 000000000..3051bcf30 --- /dev/null +++ b/tests/runtime/activity/test_namegate.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import httpx + +from band.core.activity import sync_activity_tool_names +from band.core.types import Emit +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from band.runtime.working_state import ( + end_tool_activity, + report_memory_activity, + report_tool_activity, + set_permission_wait, +) +from tests.conftest import make_message_event +from tests.runtime.activity.kit import ActivityPeer, activity_context + + +class DetailPeer(ActivityPeer): + """Capture the activity detail each report carries, alongside the base tuples.""" + + def __init__(self) -> None: + super().__init__() + self.details: list[dict[str, object] | None] = [] + + def answer(self, request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/activity"): + self.details.append(json.loads(request.content).get("activity")) + return super().answer(request) + + +def _reporting_handler( + *, tool: str | None = None, permission: bool = False, memory: bool = False +): + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + if permission: + await set_permission_wait(tools, True) + await set_permission_wait(tools, False) + if memory: + await report_memory_activity(tools) + if tool is not None: + await report_tool_activity(tools, tool) + await end_tool_activity(tools) + + return handler + + +class TestActivityToolNameGate: + async def test_tool_names_report_while_tool_calls_are_emitted(self) -> None: + peer = DetailPeer() + async with activity_context( + peer, + _reporting_handler(tool="Bash"), + AsyncMock(), + ) as ctx: + await ctx._process_event(make_message_event(msg_id="trigger")) + kinds = [detail and detail.get("kind") for detail in peer.details] + assert kinds.count("tool_call") == 1 + named = [d for d in peer.details if d and d.get("kind") == "tool_call"] + assert named[0]["name"] == "Bash" + + async def test_tool_names_are_dropped_when_tool_calls_are_not_emitted( + self, + ) -> None: + peer = DetailPeer() + async with activity_context( + peer, + _reporting_handler(tool="Bash"), + AsyncMock(), + ) as ctx: + ctx.set_activity_tool_names(False) + await ctx._process_event(make_message_event(msg_id="trigger")) + kinds = [detail and detail.get("kind") for detail in peer.details] + assert "tool_call" not in kinds + assert "reasoning" in kinds + for detail in peer.details: + assert detail is None or not detail.get("name") + + async def test_permission_and_memory_survive_the_name_gate(self) -> None: + peer = DetailPeer() + async with activity_context( + peer, + _reporting_handler(permission=True, memory=True), + AsyncMock(), + ) as ctx: + ctx.set_activity_tool_names(False) + await ctx._process_event(make_message_event(msg_id="trigger")) + kinds = [detail and detail.get("kind") for detail in peer.details] + assert "memory" in kinds + assert "permission" in kinds + + async def test_sync_resolves_names_from_the_emit_choice(self) -> None: + async with activity_context( + ActivityPeer(), _reporting_handler(), AsyncMock() + ) as ctx: + tools = SimpleNamespace(_ctx=ctx) + sync_activity_tool_names(tools, frozenset({Emit.TOOL_CALLS})) + assert ctx._activity_tool_names is True + sync_activity_tool_names(tools, frozenset({Emit.THOUGHTS})) + assert ctx._activity_tool_names is False + sync_activity_tool_names(SimpleNamespace(), frozenset()) + assert ctx._activity_tool_names is False From 9511b0f2a4ea278f850fe4249d6499b9d2e992fe Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Sun, 18 Oct 2026 15:50:05 +0300 Subject: [PATCH 18/47] feat(core): own host harness probes and share catalog metadata --- src/band/core/harness.py | 121 ++++++++++++++++++++++++++ src/band/core/model_catalog.py | 2 + src/band/core/simple_adapter.py | 33 +++++++ tests/core/test_harness_probe.py | 144 +++++++++++++++++++++++++++++++ 4 files changed, 300 insertions(+) create mode 100644 src/band/core/harness.py create mode 100644 tests/core/test_harness_probe.py diff --git a/src/band/core/harness.py b/src/band/core/harness.py new file mode 100644 index 000000000..12df54b10 --- /dev/null +++ b/src/band/core/harness.py @@ -0,0 +1,121 @@ +"""Host-facing handshake results and custody of throwaway harness probes.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable +from typing import Any, TypeVar + +from pydantic import BaseModel, ConfigDict + +from band.core.exceptions import CleanupIncompleteError +from band.core.retention.cleanup import rejoin_owned, report_owned + +_T = TypeVar("_T") +_Close = Callable[[], Awaitable[None]] + + +class PreflightResult(BaseModel): + """Whether a harness answered its handshake, not proof of inference access.""" + + model_config = ConfigDict(frozen=True) + + ok: bool + reason: str | None = None + remedy: str | None = None + + @classmethod + def passed(cls) -> PreflightResult: + return cls(ok=True) + + @classmethod + def failed(cls, reason: str, remedy: str) -> PreflightResult: + return cls(ok=False, reason=reason, remedy=remedy) + + +class HarnessProbe: + """One adapter's serialized probes and retryable resource cleanup. + + Operations register resource closers before starting their transports. A + timeout bounds reporting, not ownership: cleanup joins late startup before + closing anything it may still publish. A later probe or adapter cleanup + rejoins that same work and retries failed closers. + """ + + def __init__(self) -> None: + self._lock = asyncio.Lock() + self._operation: asyncio.Task[Any] | None = None + self._cleanup: asyncio.Task[None] | None = None + self._closers: list[_Close] = [] + self._cancel_requested = False + + def own(self, close: _Close) -> None: + """Retain an idempotent closer before awaiting resource startup.""" + if self._operation is None or self._operation.done(): + raise RuntimeError("A harness resource must belong to a running probe") + self._closers.append(close) + + async def run( + self, + operation: Callable[[], Awaitable[_T]], + *, + timeout_s: float, + ) -> _T: + """Run metadata discovery with separate operation and cleanup budgets.""" + async with self._lock: + await self.close(timeout_s=timeout_s) + + async def invoke() -> _T: + return await operation() + + task = asyncio.create_task(invoke()) + self._operation = task + self._cleanup = None + self._cancel_requested = False + try: + result = await asyncio.wait_for(asyncio.shield(task), timeout_s) + except BaseException as primary: + try: + await self.close(timeout_s=timeout_s) + except BaseException as cleanup: + if isinstance(primary, asyncio.CancelledError): + raise primary from cleanup + raise cleanup from primary + raise + await self.close(timeout_s=timeout_s) + return result + + async def close(self, *, timeout_s: float = 60.0) -> None: + """Join or retry settlement without abandoning an unfinished transport.""" + if self._operation is None and not self._closers: + return + self._cleanup = rejoin_owned( + self._cleanup, self._finish, label="Harness probe cleanup" + ) + await report_owned(self._cleanup, timeout_s) + + async def _finish(self) -> None: + if (operation := self._operation) is not None: + if not operation.done() and not self._cancel_requested: + self._cancel_requested = True + operation.cancel() + await asyncio.gather(operation, return_exceptions=True) + + retained: list[_Close] = [] + failures: list[Exception] = [] + for close in reversed(self._closers): + try: + await close() + except Exception as error: # noqa: BLE001 -- attempt every closer while retaining each failed resource + retained.append(close) + failures.append(error) + self._closers = list(reversed(retained)) + if failures: + raise CleanupIncompleteError("Harness probe resources remain owned") from ( + ExceptionGroup("Harness probe cleanup failed", failures) + ) + self._operation = None + self._cancel_requested = False + + +__all__ = ["PreflightResult"] diff --git a/src/band/core/model_catalog.py b/src/band/core/model_catalog.py index 8381b96b7..f0b95f4f1 100644 --- a/src/band/core/model_catalog.py +++ b/src/band/core/model_catalog.py @@ -52,6 +52,8 @@ class ModelChoice(BaseModel): label: str | None = None efforts: tuple[str, ...] | None = None default_effort: str | None = None + provider: str | None = None + is_default: bool = False class ModelCatalog(BaseModel): diff --git a/src/band/core/simple_adapter.py b/src/band/core/simple_adapter.py index 39623e2a0..5d1905683 100644 --- a/src/band/core/simple_adapter.py +++ b/src/band/core/simple_adapter.py @@ -15,6 +15,7 @@ from band.core.disposal import AdapterDisposal, CleanupCallback from band.core.exceptions import BandConfigError, InputNotAccepted from band.core.handoff import context_handoff +from band.core.harness import HarnessProbe, PreflightResult from band.core.model_catalog import ( ModelCatalog, ModelSelection, @@ -140,6 +141,7 @@ def __init__( # platform access read it via require_platform(). self.platform: PlatformConnection | None = None self.features = self._resolve_features(**features) + self._harness_probe_owner: HarnessProbe | None = None def add_cleanup(self, callback: CleanupCallback) -> None: """Register async resource cleanup before host startup, e.g. client.aclose. @@ -345,6 +347,8 @@ async def cleanup_all(self) -> None: Resources the caller supplies release through ``add_cleanup`` instead, only after work and this teardown settle; a direct call leaves them pending. """ + if self._harness_probe_owner is not None: + await self._harness_probe_owner.close() @property def model_selection(self) -> ModelSelection: @@ -366,6 +370,35 @@ async def list_models(self) -> ModelCatalog | None: """ return None + @property + def _harness_probe(self) -> HarnessProbe: + if self._harness_probe_owner is None: + self._harness_probe_owner = HarnessProbe() + return self._harness_probe_owner + + async def preflight(self, *, timeout_s: float = 60.0) -> PreflightResult: + """Check a throwaway harness handshake without starting a Band room. + + A passed handshake does not prove provider authentication or inference + access. Cleanup failures propagate and stay owned for ``cleanup_all``. + """ + del timeout_s + return PreflightResult.failed( + f"{type(self).__name__} does not support harness preflight.", + "Use the harness's own startup check.", + ) + + async def probe_models(self, *, timeout_s: float = 60.0) -> ModelCatalog: + """Discover models without using or creating a Band room. + + Unlike ``list_models``, this explicit host operation may start a + throwaway transport. Cleanup failures remain owned by this adapter. + """ + del timeout_s + raise BandConfigError( + f"{type(self).__name__} does not support standalone model discovery" + ) + async def apply_model_selection( self, selection: ModelSelection, *, room_id: str ) -> None: diff --git a/tests/core/test_harness_probe.py b/tests/core/test_harness_probe.py new file mode 100644 index 000000000..3354abb2f --- /dev/null +++ b/tests/core/test_harness_probe.py @@ -0,0 +1,144 @@ +"""Host probes keep resource custody through failed or interrupted cleanup.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from typing import Any + +import pytest + +from band.core.exceptions import BandConfigError, CleanupIncompleteError +from band.core.simple_adapter import SimpleAdapter +from band.core.types import PlatformMessage + + +class ProbeAdapter(SimpleAdapter[None]): + async def on_message( + self, + msg: PlatformMessage, + tools: Any, + history: None, + participants_msg: str | None, + contacts_msg: str | None, + *, + is_session_bootstrap: bool, + room_id: str, + ) -> None: + await tools.no_reply() + + +async def test_an_unsupported_preflight_cannot_report_success() -> None: + result = await ProbeAdapter().preflight() + assert not result.ok + assert result.reason is not None + assert result.remedy is not None + + +async def test_an_unsupported_catalog_is_not_an_empty_success() -> None: + with pytest.raises(BandConfigError): + await ProbeAdapter().probe_models() + + +async def test_failed_close_keeps_the_resource_for_a_cleanup_retry( + tmp_path: Path, +) -> None: + adapter = ProbeAdapter() + resource = tmp_path / "owned-resource" + refuse_close = True + + async def close() -> None: + if refuse_close: + raise PermissionError("resource still in use") + resource.unlink() + + async def operation() -> str: + adapter._harness_probe.own(close) + resource.write_text("private configuration", encoding="utf-8") + return "catalog" + + with pytest.raises(CleanupIncompleteError): + await adapter._harness_probe.run(operation, timeout_s=0.1) + assert resource.read_text(encoding="utf-8") == "private configuration" + refuse_close = False + await adapter.cleanup_all() + assert not resource.exists() + + async def next_operation() -> str: + return "next catalog" + + assert ( + await adapter._harness_probe.run(next_operation, timeout_s=0.1) + == "next catalog" + ) + + +async def test_timeout_retains_a_start_that_settles_after_cancellation( + tmp_path: Path, +) -> None: + adapter = ProbeAdapter() + resource = tmp_path / "late-resource" + entered = asyncio.Event() + settle = asyncio.Event() + + async def close() -> None: + resource.unlink() + + async def operation() -> str: + adapter._harness_probe.own(close) + entered.set() + try: + await settle.wait() + except asyncio.CancelledError: + await settle.wait() + resource.write_text("late configuration", encoding="utf-8") + return "late catalog" + + running = asyncio.create_task(adapter._harness_probe.run(operation, timeout_s=0.01)) + try: + await asyncio.wait_for(entered.wait(), 1) + with pytest.raises(CleanupIncompleteError): + await running + settle.set() + await adapter.cleanup_all() + assert not resource.exists() + finally: + settle.set() + await asyncio.gather(running, return_exceptions=True) + await adapter.cleanup_all() + + +async def test_caller_cancellation_does_not_abandon_late_startup( + tmp_path: Path, +) -> None: + adapter = ProbeAdapter() + resource = tmp_path / "cancelled-resource" + entered = asyncio.Event() + settle = asyncio.Event() + + async def close() -> None: + resource.unlink() + + async def operation() -> str: + adapter._harness_probe.own(close) + entered.set() + try: + await settle.wait() + except asyncio.CancelledError: + await settle.wait() + resource.write_text("configuration", encoding="utf-8") + return "catalog" + + running = asyncio.create_task(adapter._harness_probe.run(operation, timeout_s=0.01)) + try: + await asyncio.wait_for(entered.wait(), 1) + running.cancel() + with pytest.raises(asyncio.CancelledError): + await running + settle.set() + await adapter.cleanup_all() + assert not resource.exists() + finally: + settle.set() + await asyncio.gather(running, return_exceptions=True) + await adapter.cleanup_all() From 926fe70c1f4164353669f15f510836b7a5763e1a Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Sun, 18 Oct 2026 16:08:28 +0300 Subject: [PATCH 19/47] fix(core): keep cancelled probe diagnostics credential-safe --- README.md | 22 ++++++++++++++ src/band/core/harness.py | 5 +++- src/band/core/retention/cleanup.py | 30 ++++++++++++++----- tests/core/test_harness_probe.py | 47 ++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index abdb21f8d..6dc2cd29c 100644 --- a/README.md +++ b/README.md @@ -211,6 +211,28 @@ adapter = GeminiAdapter(GeminiAdapterConfig(model="gemini-3.8-flash")) Use [examples/run_agent.py](examples/run_agent.py) when you want one command that can switch between LangGraph, Pydantic AI, Anthropic, Claude SDK, Parlant, CrewAI, Codex, A2A bridge, and A2A gateway. Use the per-framework directories under [examples/](examples/) when you want the adapter-specific setup. +### Host-side Harness Probes + +`await adapter.preflight(timeout_s=60)` reports a harness handshake as a +`PreflightResult` (`ok`, `reason`, `remedy`). A successful handshake does not prove +that the account can run inference. Adapters without a standalone handshake +implementation explicitly report that it is unsupported. + +`await adapter.probe_models(timeout_s=60)` discovers a canonical `ModelCatalog` +without creating or selecting a Band room. Unsupported adapters raise +`BandConfigError`; an empty catalog is not a substitute for unsupported discovery. +The existing room-facing `list_models` hook remains separate. + +Catalog entries preserve advertised provider and default metadata. An entry's +`efforts=None` means the harness did not report effort support; `efforts=()` means +it explicitly offers none. `current_model` is populated only when the harness +reports an actual selection. + +Timeouts bound how long a caller waits, not ownership of a partially started +transport. Always call `await adapter.cleanup_all()` when disposing of an adapter. +An incomplete close raises `CleanupIncompleteError` and retains its resources for +a later cleanup attempt rather than reporting successful disposal. + ### Process Signals `agent.run()` and `agent.run_forever()` stop the agent on SIGTERM/SIGINT and restore the process's previous handlers when they return. A host that owns its process signals (a launchd/systemd service, a desktop app, a test runner) passes `install_signal_handlers=False` and calls `agent.stop()` from its own handler; `run_forever` then returns. `GracefulShutdown` is a ready-made handler that does this, and `run_with_graceful_shutdown(agent)` wraps it. Put `GracefulShutdown` outside `agent` so its handlers are registered before `start()` and a mid-initialize SIGTERM can still tear down: diff --git a/src/band/core/harness.py b/src/band/core/harness.py index 12df54b10..f7986b752 100644 --- a/src/band/core/harness.py +++ b/src/band/core/harness.py @@ -90,7 +90,10 @@ async def close(self, *, timeout_s: float = 60.0) -> None: if self._operation is None and not self._closers: return self._cleanup = rejoin_owned( - self._cleanup, self._finish, label="Harness probe cleanup" + self._cleanup, + self._finish, + label="Harness probe cleanup", + diagnostic_details=False, ) await report_owned(self._cleanup, timeout_s) diff --git a/src/band/core/retention/cleanup.py b/src/band/core/retention/cleanup.py index 0a0822300..fe9a6059b 100644 --- a/src/band/core/retention/cleanup.py +++ b/src/band/core/retention/cleanup.py @@ -20,16 +20,25 @@ _reporters: Counter[asyncio.Task[Any]] = Counter() # Finished tasks whose failure a caller re-raised or this module logged. _reported: weakref.WeakSet[asyncio.Task[Any]] = weakref.WeakSet() -_labels: weakref.WeakKeyDictionary[asyncio.Task[Any], str] = weakref.WeakKeyDictionary() +_labels: weakref.WeakKeyDictionary[asyncio.Task[Any], tuple[str, bool]] = ( + weakref.WeakKeyDictionary() +) def spawn_owned( - operation: Coroutine[object, object, None], *, label: str + operation: Coroutine[object, object, None], + *, + label: str, + diagnostic_details: bool = True, ) -> asyncio.Task[None]: """Run work that outlives a cancelled caller; log its failure only when - no caller is waiting to report it.""" + no caller is waiting to report it. + + Disable diagnostic details when transport errors may contain credentials; + the original exception remains on the owned task for callers and retries. + """ task = asyncio.create_task(operation) - _labels[task] = label + _labels[task] = (label, diagnostic_details) task.add_done_callback(_log_unreported) return task @@ -39,23 +48,30 @@ def rejoin_owned( operation: Callable[[], Coroutine[object, object, None]], *, label: str, + diagnostic_details: bool = True, ) -> asyncio.Task[None]: """Join the running or settled owned task; start a new one only after it failed, so concurrent callers share one attempt and a retry follows a settled failure.""" if current is None or (current.done() and task_failure(current) is not None): - return spawn_owned(operation(), label=label) + return spawn_owned( + operation(), label=label, diagnostic_details=diagnostic_details + ) return current def _log_unreported(task: asyncio.Task[Any]) -> None: if task in _reporters or task in _reported or task.cancelled(): return - if (label := _labels.get(task)) is None: + if (metadata := _labels.get(task)) is None: return _reported.add(task) if (error := task.exception()) is not None: - logger.warning("%s failed: %s", label, error, exc_info=error) + label, diagnostic_details = metadata + if diagnostic_details: + logger.warning("%s failed: %s", label, error, exc_info=error) + else: + logger.warning("%s failed (%s)", label, type(error).__name__) def task_failure(task: asyncio.Task[Any]) -> BaseException | None: diff --git a/tests/core/test_harness_probe.py b/tests/core/test_harness_probe.py index 3354abb2f..7bbd06fa8 100644 --- a/tests/core/test_harness_probe.py +++ b/tests/core/test_harness_probe.py @@ -142,3 +142,50 @@ async def operation() -> str: settle.set() await asyncio.gather(running, return_exceptions=True) await adapter.cleanup_all() + + +async def test_cancelled_cleanup_reports_no_raw_error_but_retains_retry( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + adapter = ProbeAdapter() + resource = tmp_path / "failed-resource" + secret = "provider-token-in-transport-error" + entered = asyncio.Event() + release = asyncio.Event() + failed = asyncio.Event() + refuse_close = True + + async def close() -> None: + entered.set() + await release.wait() + if refuse_close: + failed.set() + raise PermissionError(secret) + resource.unlink() + + async def operation() -> str: + adapter._harness_probe.own(close) + resource.write_text("owned configuration", encoding="utf-8") + return "catalog" + + running = asyncio.create_task(adapter._harness_probe.run(operation, timeout_s=1)) + try: + await asyncio.wait_for(entered.wait(), 1) + running.cancel() + with pytest.raises(asyncio.CancelledError): + await running + release.set() + await asyncio.wait_for(failed.wait(), 1) + await asyncio.sleep(0) + assert secret not in caplog.text + assert resource.exists() + with pytest.raises(CleanupIncompleteError): + await adapter.cleanup_all() + refuse_close = False + await adapter.cleanup_all() + assert not resource.exists() + finally: + refuse_close = False + release.set() + await asyncio.gather(running, return_exceptions=True) + await adapter.cleanup_all() From 88a0c6ff97897a8a24830081ced99f7830e75056 Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Sun, 18 Oct 2026 16:25:40 +0300 Subject: [PATCH 20/47] fix(core): enroll probes before eager execution --- src/band/core/harness.py | 3 +++ tests/core/test_harness_probe.py | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/band/core/harness.py b/src/band/core/harness.py index f7986b752..5d3e2998f 100644 --- a/src/band/core/harness.py +++ b/src/band/core/harness.py @@ -64,14 +64,17 @@ async def run( """Run metadata discovery with separate operation and cleanup budgets.""" async with self._lock: await self.close(timeout_s=timeout_s) + enrolled = asyncio.get_running_loop().create_future() async def invoke() -> _T: + await enrolled return await operation() task = asyncio.create_task(invoke()) self._operation = task self._cleanup = None self._cancel_requested = False + enrolled.set_result(None) try: result = await asyncio.wait_for(asyncio.shield(task), timeout_s) except BaseException as primary: diff --git a/tests/core/test_harness_probe.py b/tests/core/test_harness_probe.py index 7bbd06fa8..d2efa85be 100644 --- a/tests/core/test_harness_probe.py +++ b/tests/core/test_harness_probe.py @@ -11,6 +11,7 @@ from band.core.exceptions import BandConfigError, CleanupIncompleteError from band.core.simple_adapter import SimpleAdapter from band.core.types import PlatformMessage +from tests.conftest import eager_tasks, requires_eager_tasks class ProbeAdapter(SimpleAdapter[None]): @@ -189,3 +190,27 @@ async def operation() -> str: release.set() await asyncio.gather(running, return_exceptions=True) await adapter.cleanup_all() + + +@requires_eager_tasks +async def test_eager_execution_enrolls_resources_before_operation_start( + tmp_path: Path, +) -> None: + adapter = ProbeAdapter() + resource = tmp_path / "owned" + closed = tmp_path / "closed" + + async def close() -> None: + resource.rename(closed) + + async def operation() -> None: + adapter._harness_probe.own(close) + resource.write_text("owned payload", encoding="utf-8") + + try: + with eager_tasks(): + await adapter._harness_probe.run(operation, timeout_s=1) + assert closed.read_text(encoding="utf-8") == "owned payload" + assert not resource.exists() + finally: + await adapter.cleanup_all() From 8dcf009cd7a75f52db7eae12998e44b705b540de Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Sun, 18 Oct 2026 15:57:29 +0300 Subject: [PATCH 21/47] feat: add owned Codex host preflight --- docs/adapters/codex.md | 29 ++++ src/band/adapters/codex.py | 105 ++++++++++-- src/band/integrations/codex/stdio_client.py | 40 ++++- src/band/integrations/codex/types.py | 1 + tests/adapters/codex/conftest.py | 13 ++ tests/adapters/codex/probepeer.py | 55 +++++++ tests/adapters/codex/probes.py | 111 +++++++++++++ tests/adapters/codex/test_preflight.py | 160 +++++++++++++++++++ tests/adapters/codex/test_probe_cleanup.py | 76 +++++++++ tests/adapters/codex/test_probe_ownership.py | 84 ++++++++++ 10 files changed, 654 insertions(+), 20 deletions(-) create mode 100644 tests/adapters/codex/probepeer.py create mode 100644 tests/adapters/codex/probes.py create mode 100644 tests/adapters/codex/test_preflight.py create mode 100644 tests/adapters/codex/test_probe_cleanup.py create mode 100644 tests/adapters/codex/test_probe_ownership.py diff --git a/docs/adapters/codex.md b/docs/adapters/codex.md index 41d6e5ce8..0736d11d7 100644 --- a/docs/adapters/codex.md +++ b/docs/adapters/codex.md @@ -111,3 +111,32 @@ Approval commands use the token displayed in the room prompt. Native Codex request and approval IDs stay internal; a fresh request gets a new chat token after completion or room cleanup, even when Codex reuses its native ID. A redelivery of a still-pending request keeps its displayed token. + +## Host preflight + +`await adapter.preflight(timeout_s=60.0)` launches a throwaway app-server using +the adapter's `codex_command`, `codex_env`, client name/title/version, +`experimental_api` and `skill_roots`. It sends `initialize` and `account/read` +without creating a Codex thread, spending a model turn, or claiming a Band room +or workspace. + +The result is a `PreflightResult` with `ok`, `reason` and `remedy`. A missing +account fails only when Codex explicitly reports `requiresOpenaiAuth=true`; +an unsupported or refused `account/read` is not proof that login is missing. +A passed handshake is not proof of provider authentication, subscription or +inference entitlement. + +```python notest +from band.adapters.codex import CodexAdapter, CodexAdapterConfig + +adapter = CodexAdapter(CodexAdapterConfig()) +result = await adapter.preflight(timeout_s=30) +``` + +The timeout bounds handshake/RPC work and a separate cleanup wait. Cancellation +propagates. Cleanup failures raise `CleanupIncompleteError` instead of reporting +a successful or ordinary failed preflight, and the adapter retains the client, +including a partially created process or Windows job. Keep that adapter and +retry `await adapter.cleanup_all()` to join or retry cleanup. Shutdown attempts +both probe and room cleanup even when either fails. Successful calls close their +throwaway process before returning; a second preflight starts a fresh one. diff --git a/src/band/adapters/codex.py b/src/band/adapters/codex.py index 3a41024fd..ac0e4b1c0 100644 --- a/src/band/adapters/codex.py +++ b/src/band/adapters/codex.py @@ -7,7 +7,7 @@ import logging import time as _time from collections import OrderedDict -from collections.abc import AsyncIterator, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager, nullcontext from contextvars import ContextVar from dataclasses import dataclass, fields @@ -15,7 +15,7 @@ from datetime import UTC, datetime from enum import StrEnum from functools import partial -from typing import Any, ClassVar, Literal, NamedTuple, Protocol +from typing import Any, ClassVar, Literal, NamedTuple, Protocol, TypeVar from band_sdk_core import AgentFailure from pydantic import ( @@ -43,7 +43,12 @@ deliver_notice, reraise_delivery_cause, ) -from band.core.exceptions import InputNotAccepted +from band.core.exceptions import ( + BandConnectionError, + CleanupIncompleteError, + InputNotAccepted, +) +from band.core.harness import PreflightResult from band.core.protocols import ( FAILURE_CODE_TIMEOUT, GENERIC_PROVIDER_FAILURE_MESSAGE, @@ -101,6 +106,7 @@ build_agent_failure, parse_plan_steps, ) +from band.integrations.codex.winspawn import MISSING_EXECUTABLE, SpawnFailure from band.runtime.custom_tools import ( CustomToolDef, custom_tool_to_openai_schema, @@ -130,6 +136,8 @@ logger = logging.getLogger(__name__) +_ProbeResult = TypeVar("_ProbeResult") + def _image_content_items(result: dict[str, Any]) -> list[dict[str, Any]]: """An image tool result as Codex app-server ``inputImage`` content items. @@ -1891,9 +1899,10 @@ async def on_cleanup(self, room_id: str) -> None: await room.retained.dispose() async def cleanup_all(self) -> None: - """Stop the residency owner: admission closes at once, every room's - turn gets ``turn_settle_timeout_s`` to wind down, then its process - closes, all reported within one ``retention.transition_timeout_s``. + """Stop every room and any standalone probe, attempting both on failure. + + Rooms share ``retention.transition_timeout_s``; probe cleanup keeps its + independent budget and custody until the transport physically exits. Callers join the same stop, and a failed one is retried. An unfinished or failed stop keeps every room fenced with its process, thread and @@ -1901,8 +1910,15 @@ async def cleanup_all(self) -> None: """ for room_id in list(self._room_clients): self._clear_pending_approvals_for_room(room_id) - await self._retention.stop() + results = await asyncio.gather( + self._retention.stop(), super().cleanup_all(), return_exceptions=True + ) await self.backend_turns.drain() + failures = [result for result in results if isinstance(result, BaseException)] + if len(failures) == 1: + raise failures[0] + if failures: + raise BaseExceptionGroup("Codex adapter cleanup failed", failures) async def _acquire( self, room: RoomCodexClient, generation: int, resume: bool @@ -1919,13 +1935,7 @@ async def _acquire( self._read_native_events(room, client, generation), label=f"Room {room.room_id} Codex event reader", ) - await client.initialize( - client_name=self.config.client_name, - client_title=self.config.client_title, - client_version=self.config.client_version, - experimental_api=self.config.experimental_api, - ) - await self._register_skill_roots(client) + await self._initialize_client(client) if room.selected_model is None: room.selected_model = await self._select_model(client) logger.info( @@ -2181,6 +2191,71 @@ def _forget_room_thread(self, room_id: str) -> str | None: self._raw_history_by_room.pop(room_id, None) return thread_id + async def preflight(self, *, timeout_s: float = 60.0) -> PreflightResult: + """Check the configured app-server and its reported login without a turn.""" + try: + return await self._probe_harness(self._login_state, timeout_s=timeout_s) + except CleanupIncompleteError: + raise + except BandConnectionError as error: + cause = error.__cause__ + if isinstance(cause, FileNotFoundError) or ( + isinstance(cause, SpawnFailure) + and cause.winerror in MISSING_EXECUTABLE + ): + return PreflightResult.failed( + "Codex CLI binary not found.", + "Install Codex (`npm install -g @openai/codex`) or set " + "CodexAdapterConfig.codex_command.", + ) + return self._handshake_failed() + except Exception: # noqa: BLE001 -- ordinary probe failures are safe host-facing results + return self._handshake_failed() + + @staticmethod + def _handshake_failed() -> PreflightResult: + return PreflightResult.failed( + "Codex app-server did not complete its handshake.", + "Run `codex app-server` in a terminal to check the install, then retry.", + ) + + @staticmethod + async def _login_state(client: CodexClientProtocol) -> PreflightResult: + try: + account = await client.request(CodexRequestMethod.ACCOUNT_READ, {}) + except CodexJsonRpcError: + # Unsupported or refused metadata is not evidence of a missing login. + return PreflightResult.passed() + if account.get("account") is None and account.get("requiresOpenaiAuth") is True: + return PreflightResult.failed( + "Codex is not logged in.", "Run `codex login`, then retry." + ) + return PreflightResult.passed() + + async def _probe_harness( + self, + operation: Callable[[CodexClientProtocol], Awaitable[_ProbeResult]], + *, + timeout_s: float, + ) -> _ProbeResult: + async def inspect() -> _ProbeResult: + client = self._build_client(None) + self._harness_probe.own(client.close) + await client.connect() + await self._initialize_client(client) + return await operation(client) + + return await self._harness_probe.run(inspect, timeout_s=timeout_s) + + async def _initialize_client(self, client: CodexClientProtocol) -> None: + await client.initialize( + client_name=self.config.client_name, + client_title=self.config.client_title, + client_version=self.config.client_version, + experimental_api=self.config.experimental_api, + ) + await self._register_skill_roots(client) + async def _register_skill_roots(self, client: CodexClientProtocol) -> None: roots = self.config.skill_roots if not roots: @@ -2192,7 +2267,7 @@ async def _register_skill_roots(self, client: CodexClientProtocol) -> None: except CodexJsonRpcError as exc: raise RuntimeError(f"Codex rejected skill_roots {roots}: {exc}") from exc - def _build_client(self, workspace: str) -> CodexClientProtocol: + def _build_client(self, workspace: str | None) -> CodexClientProtocol: return CodexStdioClient( command=self.config.codex_command, cwd=workspace, diff --git a/src/band/integrations/codex/stdio_client.py b/src/band/integrations/codex/stdio_client.py index c87961e5a..e68773d05 100644 --- a/src/band/integrations/codex/stdio_client.py +++ b/src/band/integrations/codex/stdio_client.py @@ -95,7 +95,7 @@ async def _wait_readers(readers: Sequence[asyncio.Task[None]]) -> None: return _done, pending = await asyncio.wait(readers, timeout=_EXIT_OBSERVE_S) if pending: - logger.warning("Codex stdio readers still pending after tree kill") + raise TimeoutError("Codex stdio readers still pending after tree kill") async def kill_process_tree(process: asyncio.subprocess.Process) -> None: @@ -137,6 +137,7 @@ def __init__( self._proc: asyncio.subprocess.Process | None = None self._windows_process: WindowsProcess | None = None + self._creation_task: asyncio.Task[None] | None = None self._reader_task: asyncio.Task[None] | None = None self._stderr_task: asyncio.Task[None] | None = None self._close_task: asyncio.Task[None] | None = None @@ -158,6 +159,7 @@ async def connect(self) -> None: if self._close_task is not None: await self.close() self._close_task = None + self._creation_task = None if self._connected: return @@ -186,8 +188,24 @@ async def _spawn(self) -> asyncio.subprocess.Process: except BaseException: await self.close() raise + if self._creation_task is not None: + raise BandConnectionError("Codex process is already starting") + self._creation_task = asyncio.create_task(self._create_process()) + try: + # A cancelled caller cannot discard a child created before the + # subprocess coroutine returns its Process handle. + await asyncio.shield(self._creation_task) + if self._close_task is not None: + raise BandConnectionError("Codex process closed during startup") + assert self._proc is not None + return self._proc + except BaseException: + await self.close() + raise + + async def _create_process(self) -> None: # The child leads its own session, so its pid is the group id. - return await asyncio.create_subprocess_exec( + self._proc = await asyncio.create_subprocess_exec( *self.command, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, @@ -224,7 +242,11 @@ def _spawn_connection_error(self, failure: SpawnFailure) -> BandConnectionError: async def close(self) -> None: """Share retryable cleanup that survives a cancelled or timed-out caller.""" - if self._windows_process is None and self._proc is None: + if ( + self._windows_process is None + and self._proc is None + and self._creation_task is None + ): return if self._windows_process is not None: self._windows_process.closing = True @@ -251,6 +273,14 @@ async def _close_process(self) -> None: for task in readers: task.cancel() try: + if self._creation_task is not None: + done, _pending = await asyncio.wait( + [self._creation_task], timeout=_EXIT_OBSERVE_S + ) + if not done: + raise TimeoutError("Codex process creation is still pending") + if not self._creation_task.cancelled(): + self._creation_task.exception() # A cancelled pipe read on Windows stays pending while the child is # blocked in a synchronous stdin read, and awaiting that read never # times out. Terminate the tree and close our write end first. @@ -261,8 +291,8 @@ async def _close_process(self) -> None: await kill_process_tree(process) await close_stdin(process) if not await await_returncode(process, _EXIT_OBSERVE_S): - logger.warning( - "Codex process %s still alive after tree kill", process.pid + raise TimeoutError( + f"Codex process {process.pid} still alive after tree kill" ) self._fail_pending("Codex stdio client closed") finally: diff --git a/src/band/integrations/codex/types.py b/src/band/integrations/codex/types.py index 641fb9430..108c02269 100644 --- a/src/band/integrations/codex/types.py +++ b/src/band/integrations/codex/types.py @@ -71,6 +71,7 @@ class CodexRequestMethod(StrEnum): """Client-to-server request methods the SDK sends to ``codex app-server``.""" INITIALIZE = "initialize" + ACCOUNT_READ = "account/read" MODEL_LIST = "model/list" SKILLS_EXTRA_ROOTS_SET = "skills/extraRoots/set" THREAD_RESUME = "thread/resume" diff --git a/tests/adapters/codex/conftest.py b/tests/adapters/codex/conftest.py index bb9c8e8ee..4b5001f84 100644 --- a/tests/adapters/codex/conftest.py +++ b/tests/adapters/codex/conftest.py @@ -4,6 +4,7 @@ import asyncio from collections.abc import AsyncIterator, Awaitable, Callable +from pathlib import Path from typing import Any import looptime @@ -28,6 +29,7 @@ patch_codex_clients_by_room, send_bootstrap, ) +from tests.adapters.codex.probes import ProbeHost from tests.adapters.codexturns import ( FakeCodexClient, make_codex_adapter, @@ -129,3 +131,14 @@ async def start( yield start for adapter in adapters: await adapter.cleanup_all() + + +@pytest_asyncio.fixture(loop_scope="function") +async def probe_host( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> AsyncIterator[ProbeHost]: + host = ProbeHost(tmp_path, monkeypatch) + try: + yield host + finally: + await host.close() diff --git a/tests/adapters/codex/probepeer.py b/tests/adapters/codex/probepeer.py new file mode 100644 index 000000000..4aa93d6c8 --- /dev/null +++ b/tests/adapters/codex/probepeer.py @@ -0,0 +1,55 @@ +"""A real stdio app-server boundary for host probes without provider turns.""" + +from __future__ import annotations + +import json +import os +import sys +import threading +from pathlib import Path +from typing import Any + + +def emit(message: dict[str, Any]) -> None: + sys.stdout.write(json.dumps(message) + "\n") + sys.stdout.flush() + + +def main() -> None: + record = Path(sys.argv[1]) + settings = json.loads(sys.argv[2]) + for line in sys.stdin: + message = json.loads(line) + with record.open("a", encoding="utf-8") as stream: + stream.write( + json.dumps( + { + **message, + "pid": os.getpid(), + "marker": os.environ["CODEX_PROBE_MARKER"], + } + ) + + "\n" + ) + if "id" not in message: + continue + method = message["method"] + if method == "model/list": + cursor = message["params"].get("cursor") + pages = settings.get("pages", {"": {"data": [], "nextCursor": None}}) + reply = pages.get(cursor or "", {}) + if settings.get("endless_pages"): + page = int(cursor or "0") + reply = {"data": [], "nextCursor": str(page + 1)} + else: + reply = settings.get("replies", {}).get(method, {}) + if reply.get("hang"): + threading.Event().wait() + elif "error" in reply: + emit({"id": message["id"], "error": reply["error"]}) + else: + emit({"id": message["id"], "result": reply}) + + +if __name__ == "__main__": + main() diff --git a/tests/adapters/codex/probes.py b/tests/adapters/codex/probes.py new file mode 100644 index 000000000..1059e0f1b --- /dev/null +++ b/tests/adapters/codex/probes.py @@ -0,0 +1,111 @@ +"""Real Codex clients with controllable cleanup and a recorded wire boundary.""" + +from __future__ import annotations + +import json +import sys +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +import pytest + +from band.adapters import codex +from band.adapters.codex import CodexAdapter, CodexAdapterConfig +from band.integrations.codex.stdio_client import CodexStdioClient +from tests.adapters.waiting import until + +PROBE_MARKER = "configured-host-environment" +CLOSE_FAILURE = "probe process cleanup refused" + + +class ProbeClient(CodexStdioClient): + refuse_close = False + + async def _close_process(self) -> None: + if self.refuse_close: + raise RuntimeError(CLOSE_FAILURE) + await super()._close_process() + + +class ProbeHost: + def __init__(self, root: Path, monkeypatch: pytest.MonkeyPatch) -> None: + self.record = root / "probe.jsonl" + self.clients: list[ProbeClient] = [] + self.adapters: list[CodexAdapter] = [] + self.root = root + + def build( + *, + command: Sequence[str] | None = None, + cwd: str | None = None, + env: Mapping[str, str] | None = None, + ) -> ProbeClient: + client = ProbeClient(command=command, cwd=cwd, env=env) + self.clients.append(client) + return client + + monkeypatch.setattr(codex, "CodexStdioClient", build) + + def adapter( + self, + *, + replies: dict[str, Any] | None = None, + pages: dict[str, Any] | None = None, + endless_pages: bool = False, + **config: Any, + ) -> CodexAdapter: + settings = { + "replies": replies or {}, + "endless_pages": endless_pages, + } + if pages is not None: + settings["pages"] = pages + adapter = CodexAdapter( + CodexAdapterConfig( + codex_command=( + sys.executable, + "-u", + str(Path(__file__).with_name("probepeer.py")), + str(self.record), + json.dumps(settings), + ), + codex_env={"CODEX_PROBE_MARKER": PROBE_MARKER}, + workspace_for_room=lambda room: str(self.root / room), + **config, + ) + ) + self.adapters.append(adapter) + return adapter + + @property + def messages(self) -> list[dict[str, Any]]: + if not self.record.exists(): + return [] + return [json.loads(line) for line in self.record.read_text().splitlines()] + + @property + def methods(self) -> list[str]: + return [message["method"] for message in self.messages] + + async def received(self, method: str) -> None: + await until( + lambda: method in self.methods, + failure=f"The app-server never received {method}", + ) + + def assert_exited(self) -> None: + assert self.clients, "The configured app-server never started" + for client in self.clients: + assert client._proc is not None + assert client._proc.returncode is not None + assert client._windows_process is None + assert not client._connected + + async def close(self) -> None: + for client in self.clients: + client.refuse_close = False + for adapter in self.adapters: + await adapter.cleanup_all() + for client in self.clients: + await client.close() diff --git a/tests/adapters/codex/test_preflight.py b/tests/adapters/codex/test_preflight.py new file mode 100644 index 000000000..ae3e5ba85 --- /dev/null +++ b/tests/adapters/codex/test_preflight.py @@ -0,0 +1,160 @@ +"""Host preflight performs configured RPCs without claiming a room or turn.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +import pytest + +from band.adapters.codex import CodexAdapter, CodexAdapterConfig +from band.core.exceptions import CleanupIncompleteError +from band.integrations.codex.stdio_client import CodexStdioClient +from tests.adapters.codex.probes import CLOSE_FAILURE, PROBE_MARKER, ProbeHost + + +class TestPreflight: + async def test_configured_handshake_checks_login_without_room_or_turn( + self, probe_host: ProbeHost, tmp_path: Path + ) -> None: + skill_root = str(tmp_path / "skills") + adapter = probe_host.adapter( + replies={"account/read": {"account": {"type": "chatgpt"}}}, + client_name="host_name", + client_title="Host title", + client_version="2.3.4", + experimental_api=False, + skill_roots=[skill_root], + ) + + result = await adapter.preflight(timeout_s=5) + + assert result.ok + assert probe_host.methods == [ + "initialize", + "initialized", + "skills/extraRoots/set", + "account/read", + ] + initialize = probe_host.messages[0] + assert initialize["params"] == { + "clientInfo": { + "name": "host_name", + "title": "Host title", + "version": "2.3.4", + }, + "capabilities": {"experimentalApi": False}, + } + assert initialize["marker"] == PROBE_MARKER + assert probe_host.messages[2]["params"] == {"extraRoots": [skill_root]} + assert adapter._room_clients == {} + assert adapter._workspaces.rooms == () + probe_host.assert_exited() + + @pytest.mark.parametrize( + ("account", "ok"), + [ + ({"account": None, "requiresOpenaiAuth": True}, False), + ({"account": None, "requiresOpenaiAuth": False}, True), + ({"account": {"type": "apiKey"}, "requiresOpenaiAuth": True}, True), + ({"error": {"code": -32601, "message": "unsupported"}}, True), + ({"error": {"code": -32000, "message": "refused"}}, True), + ], + ids=["missing-required-login", "custom-provider", "api-key", "unsupported", "refused"], + ) + async def test_only_positively_missing_required_auth_fails( + self, probe_host: ProbeHost, account: dict, ok: bool + ) -> None: + adapter = probe_host.adapter(replies={"account/read": account}) + + result = await adapter.preflight(timeout_s=5) + + assert result.ok is ok + if not ok: + assert "not logged in" in result.reason + assert "codex login" in result.remedy + probe_host.assert_exited() + + async def test_missing_executable_names_install_remedy(self, tmp_path: Path) -> None: + adapter = CodexAdapter( + CodexAdapterConfig(codex_command=(str(tmp_path / "absent-codex"),)) + ) + + result = await adapter.preflight(timeout_s=5) + + assert not result.ok + assert "binary not found" in result.reason + assert "codex_command" in result.remedy + await adapter.cleanup_all() + + @pytest.mark.parametrize("method", ["initialize", "account/read"]) + async def test_hung_rpc_has_a_budget_and_reaps_the_owned_process( + self, probe_host: ProbeHost, method: str + ) -> None: + adapter = probe_host.adapter(replies={method: {"hang": True}}) + + result = await adapter.preflight(timeout_s=1) + + assert not result.ok + assert result.reason + probe_host.assert_exited() + + async def test_initialize_error_is_a_failed_result_after_cleanup( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter( + replies={"initialize": {"error": {"code": -32000, "message": "bad frame"}}} + ) + + result = await adapter.preflight(timeout_s=5) + + assert not result.ok + assert "handshake" in result.reason + assert result.remedy + probe_host.assert_exited() + + async def test_cancellation_propagates_and_reaps_the_process( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter(replies={"account/read": {"hang": True}}) + probing = asyncio.create_task(adapter.preflight(timeout_s=5)) + try: + await probe_host.received("account/read") + probing.cancel() + with pytest.raises(asyncio.CancelledError): + await probing + await adapter.cleanup_all() + probe_host.assert_exited() + finally: + probing.cancel() + await asyncio.gather(probing, return_exceptions=True) + + async def test_cleanup_failure_remains_owned_for_adapter_retry( + self, probe_host: ProbeHost, monkeypatch: pytest.MonkeyPatch + ) -> None: + adapter = probe_host.adapter() + + async def refuse(_client: CodexStdioClient) -> None: + raise RuntimeError(CLOSE_FAILURE) + + with monkeypatch.context() as patch: + patch.setattr(CodexStdioClient, "_close_process", refuse) + with pytest.raises(CleanupIncompleteError): + await adapter.preflight(timeout_s=5) + [client] = probe_host.clients + assert client._proc is not None and client._proc.returncode is None + + await adapter.cleanup_all() + probe_host.assert_exited() + + async def test_second_preflight_uses_a_fresh_process( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter() + + first = await adapter.preflight(timeout_s=5) + second = await adapter.preflight(timeout_s=5) + + assert first.ok and second.ok + assert len(probe_host.clients) == 2 + probe_host.assert_exited() diff --git a/tests/adapters/codex/test_probe_cleanup.py b/tests/adapters/codex/test_probe_cleanup.py new file mode 100644 index 000000000..53efbe180 --- /dev/null +++ b/tests/adapters/codex/test_probe_cleanup.py @@ -0,0 +1,76 @@ +"""Incomplete process and reader cleanup cannot become a successful preflight.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from band.core.exceptions import CleanupIncompleteError +from band.integrations.codex import stdio_client, winspawn +from tests.adapters.codex.probes import ProbeHost +from tests.adapters.waiting import spin + + +class TestProbeCleanup: + async def test_unconfirmed_process_exit_stays_owned_for_retry( + self, probe_host: ProbeHost, monkeypatch: pytest.MonkeyPatch + ) -> None: + adapter = probe_host.adapter() + + async def not_observed( + _process: asyncio.subprocess.Process, _timeout: float + ) -> bool: + return False + + with monkeypatch.context() as patch: + patch.setattr(stdio_client, "await_returncode", not_observed) + patch.setattr(winspawn, "await_returncode", not_observed) + with pytest.raises(CleanupIncompleteError): + await adapter.preflight(timeout_s=5) + + await adapter.cleanup_all() + probe_host.assert_exited() + + async def test_unsettled_reader_prevents_success_and_is_rejoined( + self, probe_host: ProbeHost, monkeypatch: pytest.MonkeyPatch + ) -> None: + adapter = probe_host.adapter(replies={"account/read": {"hang": True}}) + probing = asyncio.create_task(adapter.preflight(timeout_s=5)) + finish_reader = asyncio.Event() + reader_started = asyncio.Event() + + async def held_reader() -> None: + reader_started.set() + while not finish_reader.is_set(): + try: + await finish_reader.wait() + except asyncio.CancelledError: + continue + + reader = asyncio.create_task(held_reader()) + try: + await reader_started.wait() + await probe_host.received("account/read") + [client] = probe_host.clients + assert client._stderr_task is not None + client._stderr_task.cancel() + await asyncio.gather(client._stderr_task, return_exceptions=True) + client._stderr_task = reader + probing.cancel() + await spin() + with pytest.raises(asyncio.CancelledError): + await probing + with monkeypatch.context() as patch: + patch.setattr(stdio_client, "_EXIT_OBSERVE_S", 0) + with pytest.raises(CleanupIncompleteError): + await adapter._harness_probe.close(timeout_s=0.1) + finish_reader.set() + await reader + await adapter.cleanup_all() + probe_host.assert_exited() + finally: + finish_reader.set() + await asyncio.gather(reader, return_exceptions=True) + probing.cancel() + await asyncio.gather(probing, return_exceptions=True) diff --git a/tests/adapters/codex/test_probe_ownership.py b/tests/adapters/codex/test_probe_ownership.py new file mode 100644 index 000000000..288400f7b --- /dev/null +++ b/tests/adapters/codex/test_probe_ownership.py @@ -0,0 +1,84 @@ +"""A probe keeps partial spawns and failed cleanup reachable through the adapter.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +from band.core.exceptions import CleanupIncompleteError +from tests.adapters.codex.kit import ROOM_ID, send_bootstrap +from tests.adapters.codex.probes import ProbeHost +from tests.adapters.waiting import spin + + +class TestProbeOwnership: + async def test_cancel_during_late_subprocess_creation_reaps_the_eventual_child( + self, probe_host: ProbeHost, monkeypatch: pytest.MonkeyPatch + ) -> None: + adapter = probe_host.adapter() + create_process = asyncio.create_subprocess_exec + created = asyncio.Event() + return_process = asyncio.Event() + processes: list[asyncio.subprocess.Process] = [] + + async def late_creation(*args: str, **kwargs: Any) -> asyncio.subprocess.Process: + process = await create_process(*args, **kwargs) + processes.append(process) + created.set() + await return_process.wait() + return process + + monkeypatch.setattr(asyncio, "create_subprocess_exec", late_creation) + probing = asyncio.create_task(adapter.preflight(timeout_s=5)) + try: + await asyncio.wait_for(created.wait(), timeout=5) + probing.cancel() + await spin() + return_process.set() + with pytest.raises(asyncio.CancelledError): + await probing + await adapter.cleanup_all() + assert processes + assert all(process.returncode is not None for process in processes) + assert adapter._room_clients == {} + finally: + return_process.set() + probing.cancel() + await asyncio.gather(probing, return_exceptions=True) + await adapter.cleanup_all() + for process in processes: + if process.returncode is None: + process.kill() + await process.wait() + + async def test_probe_failure_does_not_skip_existing_room_cleanup( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter(replies={"account/read": {"hang": True}}) + await adapter.on_started("Agent", "A coding agent") + await send_bootstrap(adapter, content="/model") + [room_client] = probe_host.clients + assert ROOM_ID in adapter._room_clients + probing = asyncio.create_task(adapter.preflight(timeout_s=5)) + try: + await probe_host.received("account/read") + probe_client = probe_host.clients[-1] + probe_client.refuse_close = True + probing.cancel() + with pytest.raises(asyncio.CancelledError): + await probing + with pytest.raises(CleanupIncompleteError): + await adapter.cleanup_all() + assert room_client._proc is not None + assert room_client._proc.returncode is not None + assert adapter._workspaces.rooms == () + assert probe_client._proc is not None + assert probe_client._proc.returncode is None + probe_client.refuse_close = False + await adapter.cleanup_all() + probe_host.assert_exited() + finally: + probing.cancel() + await asyncio.gather(probing, return_exceptions=True) From 8c46dba7aec1aa64677bfdbbaa3bb30190c20d36 Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Sun, 18 Oct 2026 16:52:49 +0300 Subject: [PATCH 22/47] fix(codex): isolate probe diagnostics and preserve shutdown requester --- docs/adapters/codex.md | 9 +- src/band/adapters/codex.py | 15 ++-- src/band/integrations/codex/rpc_base.py | 28 ++++-- src/band/integrations/codex/stdio_client.py | 29 +++++-- tests/adapters/codex/lifecycleprofile.py | 8 +- tests/adapters/codex/probepeer.py | 12 +++ tests/adapters/codex/probes.py | 10 ++- .../adapters/codex/test_callback_shutdown.py | 73 ++++++++++++++++ tests/adapters/codex/test_preflight.py | 12 ++- .../adapters/codex/test_probe_diagnostics.py | 87 +++++++++++++++++++ tests/adapters/codex/test_probe_ownership.py | 4 +- 11 files changed, 262 insertions(+), 25 deletions(-) create mode 100644 tests/adapters/codex/test_callback_shutdown.py create mode 100644 tests/adapters/codex/test_probe_diagnostics.py diff --git a/docs/adapters/codex.md b/docs/adapters/codex.md index 0736d11d7..4771f6cee 100644 --- a/docs/adapters/codex.md +++ b/docs/adapters/codex.md @@ -126,6 +126,11 @@ an unsupported or refused `account/read` is not proof that login is missing. A passed handshake is not proof of provider authentication, subscription or inference entitlement. +Standalone probes emit metadata-only diagnostics: raw app-server stdout, stderr, +and exception text are not logged, including cleanup that finishes after caller +cancellation. Original cleanup errors remain available to callers and retries. +Ordinary room transports keep their existing diagnostic policy. + ```python notest from band.adapters.codex import CodexAdapter, CodexAdapterConfig @@ -138,5 +143,7 @@ propagates. Cleanup failures raise `CleanupIncompleteError` instead of reporting a successful or ordinary failed preflight, and the adapter retains the client, including a partially created process or Windows job. Keep that adapter and retry `await adapter.cleanup_all()` to join or retry cleanup. Shutdown attempts -both probe and room cleanup even when either fails. Successful calls close their +both probe and room cleanup even when either fails. A custom-tool callback +requesting shutdown can receive a retryable pending-cleanup error while its own +turn unwinds; it is not cancelled by that stop. Successful calls close their throwaway process before returning; a second preflight starts a fresh one. diff --git a/src/band/adapters/codex.py b/src/band/adapters/codex.py index ac0e4b1c0..affcfdf12 100644 --- a/src/band/adapters/codex.py +++ b/src/band/adapters/codex.py @@ -1910,11 +1910,14 @@ async def cleanup_all(self) -> None: """ for room_id in list(self._room_clients): self._clear_pending_approvals_for_room(room_id) - results = await asyncio.gather( - self._retention.stop(), super().cleanup_all(), return_exceptions=True - ) + failures: list[BaseException] = [] + # Retention must see an enrolled callback as the stop requester. + for cleanup in (self._retention.stop, super().cleanup_all): + try: + await cleanup() + except BaseException as error: # noqa: BLE001 -- attempt both cleanups before propagating cancellation or failures + failures.append(error) await self.backend_turns.drain() - failures = [result for result in results if isinstance(result, BaseException)] if len(failures) == 1: raise failures[0] if failures: @@ -2200,8 +2203,7 @@ async def preflight(self, *, timeout_s: float = 60.0) -> PreflightResult: except BandConnectionError as error: cause = error.__cause__ if isinstance(cause, FileNotFoundError) or ( - isinstance(cause, SpawnFailure) - and cause.winerror in MISSING_EXECUTABLE + isinstance(cause, SpawnFailure) and cause.winerror in MISSING_EXECUTABLE ): return PreflightResult.failed( "Codex CLI binary not found.", @@ -2272,6 +2274,7 @@ def _build_client(self, workspace: str | None) -> CodexClientProtocol: command=self.config.codex_command, cwd=workspace, env=self.config.codex_env, + diagnostic_details=workspace is not None, ) async def _select_model(self, client: CodexClientProtocol) -> str: diff --git a/src/band/integrations/codex/rpc_base.py b/src/band/integrations/codex/rpc_base.py index 2fd379ff3..2d24860bb 100644 --- a/src/band/integrations/codex/rpc_base.py +++ b/src/band/integrations/codex/rpc_base.py @@ -82,9 +82,11 @@ def __init__( *, retry_policy: OverloadRetryPolicy | None = None, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + diagnostic_details: bool = True, ) -> None: self.retry_policy = retry_policy or OverloadRetryPolicy() self._sleep = sleep + self.diagnostic_details = diagnostic_details self._pending: dict[JsonRpcId, asyncio.Future[dict[str, Any]]] = {} self._events: asyncio.Queue[RpcEvent] = asyncio.Queue(maxsize=10_000) @@ -263,11 +265,18 @@ async def _dispatch_rpc_message(self, text: str) -> None: try: message = json.loads(text) except json.JSONDecodeError: - logger.warning("Skipping non-JSON %s: %s", self._transport_label, text) + logger.warning( + "Skipping non-JSON %s: %s", + self._transport_label, + text if self.diagnostic_details else "", + ) return if not isinstance(message, dict): - logger.warning("Skipping non-object JSON-RPC payload: %s", message) + logger.warning( + "Skipping non-object JSON-RPC payload: %s", + message if self.diagnostic_details else "", + ) return msg_id = message.get("id") @@ -285,7 +294,10 @@ async def _dispatch_rpc_message(self, text: str) -> None: ) ) except asyncio.QueueFull: - logger.warning("Event queue full; dropping server request %s", method) + logger.warning( + "Event queue full; dropping server request %s", + method if self.diagnostic_details else "", + ) return if method: @@ -300,7 +312,10 @@ async def _dispatch_rpc_message(self, text: str) -> None: ) ) except asyncio.QueueFull: - logger.warning("Event queue full; dropping notification %s", method) + logger.warning( + "Event queue full; dropping notification %s", + method if self.diagnostic_details else "", + ) return if msg_id is not None: @@ -309,7 +324,10 @@ async def _dispatch_rpc_message(self, text: str) -> None: future.set_result(message) return - logger.debug("Ignoring unknown JSON-RPC payload: %s", message) + logger.debug( + "Ignoring unknown JSON-RPC payload: %s", + message if self.diagnostic_details else "", + ) def _next_request_id(self) -> int: self._request_id += 1 diff --git a/src/band/integrations/codex/stdio_client.py b/src/band/integrations/codex/stdio_client.py index e68773d05..5f1de7e24 100644 --- a/src/band/integrations/codex/stdio_client.py +++ b/src/band/integrations/codex/stdio_client.py @@ -113,7 +113,11 @@ async def kill_process_tree(process: asyncio.subprocess.Process) -> None: class CodexStdioClient(BaseJsonRpcClient): - """Bidirectional JSON-RPC client with a single stdout read loop.""" + """Bidirectional JSON-RPC client with a single stdout read loop. + + Disabling diagnostic details keeps wire data and exception text out of logs + throughout startup and retryable cleanup; callers still receive the errors. + """ def __init__( self, @@ -123,8 +127,13 @@ def __init__( env: Mapping[str, str] | None = None, retry_policy: OverloadRetryPolicy | None = None, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + diagnostic_details: bool = True, ) -> None: - super().__init__(retry_policy=retry_policy, sleep=sleep) + super().__init__( + retry_policy=retry_policy, + sleep=sleep, + diagnostic_details=diagnostic_details, + ) self._transport_label = "codex output line" self.command = tuple(command) if command else self._default_codex_command() self.cwd = cwd @@ -261,10 +270,14 @@ async def close(self) -> None: self._close_task.add_done_callback(self._log_close_failure) await asyncio.shield(self._close_task) - @staticmethod - def _log_close_failure(task: asyncio.Task[None]) -> None: + def _log_close_failure(self, task: asyncio.Task[None]) -> None: if not task.cancelled() and (error := task.exception()) is not None: - logger.error("Codex subprocess cleanup failed", exc_info=error) + if self.diagnostic_details: + logger.error("Codex subprocess cleanup failed", exc_info=error) + else: + logger.error( + "Codex subprocess cleanup failed (%s)", type(error).__name__ + ) async def _close_process(self) -> None: readers = [ @@ -322,7 +335,7 @@ async def _read_stdout_loop(self) -> None: except asyncio.CancelledError: raise except Exception: - logger.exception("Codex stdout loop failed") + logger.error("Codex stdout loop failed", exc_info=self.diagnostic_details) self._fail_pending("Codex stdout loop failed") async def _read_stderr_loop(self) -> None: @@ -334,9 +347,9 @@ async def _read_stderr_loop(self) -> None: if not line: return text = line.decode("utf-8", errors="replace").rstrip() - if text: + if text and self.diagnostic_details: logger.debug("codex stderr: %s", text) except asyncio.CancelledError: raise except Exception: - logger.exception("Codex stderr loop failed") + logger.error("Codex stderr loop failed", exc_info=self.diagnostic_details) diff --git a/tests/adapters/codex/lifecycleprofile.py b/tests/adapters/codex/lifecycleprofile.py index 34ca58b5e..a637319a0 100644 --- a/tests/adapters/codex/lifecycleprofile.py +++ b/tests/adapters/codex/lifecycleprofile.py @@ -75,8 +75,14 @@ def child( command: Sequence[str] | None, cwd: str | None, env: Mapping[str, str] | None, + diagnostic_details: bool = True, ) -> CodexPeer: - peer = CodexPeer(command=command, cwd=cwd, env=env) + peer = CodexPeer( + command=command, + cwd=cwd, + env=env, + diagnostic_details=diagnostic_details, + ) host.spawned(peer, cwd) return peer diff --git a/tests/adapters/codex/probepeer.py b/tests/adapters/codex/probepeer.py index 4aa93d6c8..247e30108 100644 --- a/tests/adapters/codex/probepeer.py +++ b/tests/adapters/codex/probepeer.py @@ -18,6 +18,18 @@ def emit(message: dict[str, Any]) -> None: def main() -> None: record = Path(sys.argv[1]) settings = json.loads(sys.argv[2]) + if diagnostic := settings.get("diagnostic"): + sys.stdout.write( + diagnostic + + "\n" + + json.dumps([diagnostic]) + + "\n" + + json.dumps({"diagnostic": diagnostic}) + + "\n" + ) + sys.stdout.flush() + sys.stderr.write(diagnostic + "\n") + sys.stderr.flush() for line in sys.stdin: message = json.loads(line) with record.open("a", encoding="utf-8") as stream: diff --git a/tests/adapters/codex/probes.py b/tests/adapters/codex/probes.py index 1059e0f1b..ab70c764b 100644 --- a/tests/adapters/codex/probes.py +++ b/tests/adapters/codex/probes.py @@ -40,8 +40,14 @@ def build( command: Sequence[str] | None = None, cwd: str | None = None, env: Mapping[str, str] | None = None, + diagnostic_details: bool = True, ) -> ProbeClient: - client = ProbeClient(command=command, cwd=cwd, env=env) + client = ProbeClient( + command=command, + cwd=cwd, + env=env, + diagnostic_details=diagnostic_details, + ) self.clients.append(client) return client @@ -53,11 +59,13 @@ def adapter( replies: dict[str, Any] | None = None, pages: dict[str, Any] | None = None, endless_pages: bool = False, + diagnostic: str | None = None, **config: Any, ) -> CodexAdapter: settings = { "replies": replies or {}, "endless_pages": endless_pages, + "diagnostic": diagnostic, } if pages is not None: settings["pages"] = pages diff --git a/tests/adapters/codex/test_callback_shutdown.py b/tests/adapters/codex/test_callback_shutdown.py new file mode 100644 index 000000000..258e8c336 --- /dev/null +++ b/tests/adapters/codex/test_callback_shutdown.py @@ -0,0 +1,73 @@ +"""A custom tool can request shutdown without being cancelled by its own stop.""" + +from __future__ import annotations + +import asyncio + +from pydantic import BaseModel + +from band.adapters.codex import CodexAdapterConfig +from band.core.exceptions import CleanupIncompleteError +from band.core.retention import RoomRetentionConfig +from tests.adapters.codex.kit import ToolSchemaFakeTools, send_bootstrap +from tests.adapters.codexturns import ( + FakeCodexClient, + event_request, + make_codex_adapter, + turn_completed, +) + + +class TestCallbackShutdown: + async def test_enrolled_tool_gets_pending_cleanup_and_unwinds_without_cancellation( + self, + ) -> None: + class ShutdownInput(BaseModel): + """Request orderly host shutdown.""" + + observed: list[str] = [] + + async def shutdown(_input: ShutdownInput) -> str: + try: + await adapter.cleanup_all() + except CleanupIncompleteError: + observed.append("pending") + except asyncio.CancelledError: + observed.append("cancelled") + raise + return "shutdown requested" + + client = FakeCodexClient( + events=[ + event_request( + 99, + "item/tool/call", + {"tool": "shutdown", "arguments": {}, "callId": "shutdown-99"}, + ), + turn_completed(), + ] + ) + adapter = make_codex_adapter( + client, + config=CodexAdapterConfig( + turn_settle_timeout_s=0.05, + retention=RoomRetentionConfig(transition_timeout_s=1), + ), + additional_tools=[(ShutdownInput, shutdown)], + ) + tools = ToolSchemaFakeTools() + await adapter.on_started("Codex Agent", "A coding agent") + try: + results = await asyncio.gather( + send_bootstrap(adapter, tools=tools), return_exceptions=True + ) + assert observed == ["pending"] + assert results == [None] + assert client.responses[0][1]["success"] is True + assert ( + client.responses[0][1]["contentItems"][0]["text"] + == "shutdown requested" + ) + finally: + await adapter.cleanup_all() + assert client.closed diff --git a/tests/adapters/codex/test_preflight.py b/tests/adapters/codex/test_preflight.py index ae3e5ba85..45be4ad1e 100644 --- a/tests/adapters/codex/test_preflight.py +++ b/tests/adapters/codex/test_preflight.py @@ -60,7 +60,13 @@ async def test_configured_handshake_checks_login_without_room_or_turn( ({"error": {"code": -32601, "message": "unsupported"}}, True), ({"error": {"code": -32000, "message": "refused"}}, True), ], - ids=["missing-required-login", "custom-provider", "api-key", "unsupported", "refused"], + ids=[ + "missing-required-login", + "custom-provider", + "api-key", + "unsupported", + "refused", + ], ) async def test_only_positively_missing_required_auth_fails( self, probe_host: ProbeHost, account: dict, ok: bool @@ -75,7 +81,9 @@ async def test_only_positively_missing_required_auth_fails( assert "codex login" in result.remedy probe_host.assert_exited() - async def test_missing_executable_names_install_remedy(self, tmp_path: Path) -> None: + async def test_missing_executable_names_install_remedy( + self, tmp_path: Path + ) -> None: adapter = CodexAdapter( CodexAdapterConfig(codex_command=(str(tmp_path / "absent-codex"),)) ) diff --git a/tests/adapters/codex/test_probe_diagnostics.py b/tests/adapters/codex/test_probe_diagnostics.py new file mode 100644 index 000000000..9cf7a7ad4 --- /dev/null +++ b/tests/adapters/codex/test_probe_diagnostics.py @@ -0,0 +1,87 @@ +"""Standalone probes retain failures without copying raw transport diagnostics.""" + +from __future__ import annotations + +import asyncio +import logging + +import pytest + +from tests.adapters.codex.kit import send_bootstrap +from tests.adapters.codex.probes import ProbeHost +from tests.adapters.waiting import spin + +SENSITIVE_DIAGNOSTIC = "configured-token-that-must-not-reach-probe-logs" + + +class TestProbeDiagnostics: + async def test_real_probe_wire_and_stderr_are_metadata_only( + self, probe_host: ProbeHost, caplog: pytest.LogCaptureFixture + ) -> None: + adapter = probe_host.adapter(diagnostic=SENSITIVE_DIAGNOSTIC) + + with caplog.at_level(logging.DEBUG): + result = await adapter.preflight(timeout_s=5) + + assert result.ok + assert "initialize" in probe_host.methods + assert "account/read" in probe_host.methods + probe_host.assert_exited() + assert SENSITIVE_DIAGNOSTIC not in caplog.text + + async def test_existing_room_transport_keeps_its_diagnostic_policy( + self, probe_host: ProbeHost, caplog: pytest.LogCaptureFixture + ) -> None: + adapter = probe_host.adapter( + diagnostic=SENSITIVE_DIAGNOSTIC, model="configured-model" + ) + await adapter.on_started("Codex Agent", "A coding agent") + with caplog.at_level(logging.DEBUG): + await send_bootstrap(adapter, content="/model") + await adapter.cleanup_all() + + assert "initialize" in probe_host.methods + probe_host.assert_exited() + assert SENSITIVE_DIAGNOSTIC in caplog.text + + async def test_late_close_failure_keeps_the_original_error_and_retry_without_raw_log( + self, + probe_host: ProbeHost, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + adapter = probe_host.adapter(replies={"account/read": {"hang": True}}) + probing = asyncio.create_task(adapter.preflight(timeout_s=5)) + close_started = asyncio.Event() + finish_close = asyncio.Event() + + async def refused_close() -> None: + close_started.set() + await finish_close.wait() + raise RuntimeError(SENSITIVE_DIAGNOSTIC) + + try: + with caplog.at_level(logging.DEBUG): + await probe_host.received("account/read") + [client] = probe_host.clients + with monkeypatch.context() as patch: + patch.setattr(client, "_close_process", refused_close) + probing.cancel() + await asyncio.wait_for(close_started.wait(), timeout=5) + probing.cancel() + with pytest.raises(asyncio.CancelledError): + await probing + finish_close.set() + assert client._close_task is not None + with pytest.raises(RuntimeError, match=SENSITIVE_DIAGNOSTIC): + await client._close_task + await spin() + assert client._proc is not None + assert client._proc.returncode is None + assert SENSITIVE_DIAGNOSTIC not in caplog.text + finally: + finish_close.set() + probing.cancel() + await asyncio.gather(probing, return_exceptions=True) + await adapter.cleanup_all() + probe_host.assert_exited() diff --git a/tests/adapters/codex/test_probe_ownership.py b/tests/adapters/codex/test_probe_ownership.py index 288400f7b..3cda9ed18 100644 --- a/tests/adapters/codex/test_probe_ownership.py +++ b/tests/adapters/codex/test_probe_ownership.py @@ -23,7 +23,9 @@ async def test_cancel_during_late_subprocess_creation_reaps_the_eventual_child( return_process = asyncio.Event() processes: list[asyncio.subprocess.Process] = [] - async def late_creation(*args: str, **kwargs: Any) -> asyncio.subprocess.Process: + async def late_creation( + *args: str, **kwargs: Any + ) -> asyncio.subprocess.Process: process = await create_process(*args, **kwargs) processes.append(process) created.set() From c1ef05409567d2290705170198be67b7b85260a3 Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Sun, 18 Oct 2026 21:54:47 +0300 Subject: [PATCH 23/47] fix(codex): await physical Windows job member exit --- docs/adapters/codex.md | 8 +- src/band/integrations/codex/winspawn.py | 126 ++++++++++++++++- tests/integrations/codex/conftest.py | 10 ++ tests/integrations/codex/jobpeer.py | 134 ++++++++++++++++++ tests/integrations/codex/test_cleanup.py | 5 +- tests/integrations/codex/test_job.py | 165 +++++++++++++++++++++-- 6 files changed, 426 insertions(+), 22 deletions(-) create mode 100644 tests/integrations/codex/jobpeer.py diff --git a/docs/adapters/codex.md b/docs/adapters/codex.md index 4771f6cee..315c59aeb 100644 --- a/docs/adapters/codex.md +++ b/docs/adapters/codex.md @@ -38,9 +38,11 @@ For existing integrations, follow the [SDK 4.x to 5.x migration](../acp.md#upgra On Windows, the client owns the helper and its process-tree job from startup through cleanup. It starts the interpreter image directly so a virtual-environment launcher cannot hide another helper process. Cleanup confirms the helper has - exited before terminating and draining the job; an empty job alone does not - establish helper exit. - Failed cleanup retains that same owner and job for retry. + exited, seals the job against new children, and retains member process handles + before requesting termination. It waits for those handles to signal physical + exit as well as empty job accounting; accounting alone does not prove directory + and file handles were released. Failed cleanup retains that same owner, job, + and any pending process handles for retry. - **Leaving is final.** Leaving a room stops it admitting work at once. A leave that fails or runs over keeps the process, thread and workspace claim, and the room refuses work until a retried leave succeeds; a later rejoin diff --git a/src/band/integrations/codex/winspawn.py b/src/band/integrations/codex/winspawn.py index a7306ba9c..75520e9d0 100644 --- a/src/band/integrations/codex/winspawn.py +++ b/src/band/integrations/codex/winspawn.py @@ -32,12 +32,16 @@ _JOB_POLL_S = 0.05 _ERROR_ALREADY_EXISTS = 183 _ERROR_BROKEN_PIPE = 109 +_ERROR_INVALID_PARAMETER = 87 +_ERROR_MORE_DATA = 234 _STATUS_READ_SIZE = 256 _PATH_BUFFER_SIZE = 260 _JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x00002000 +_JOB_OBJECT_LIMIT_ACTIVE_PROCESS = 0x00000008 _JOB_OBJECT_EXTENDED_LIMIT_INFORMATION = 9 _JOB_OBJECT_BASIC_ACCOUNTING_INFORMATION = 1 +_JOB_OBJECT_BASIC_PROCESS_ID_LIST = 3 _JOB_OBJECT_ASSIGN_PROCESS = 0x0001 _CREATE_SUSPENDED = 0x00000004 @@ -46,7 +50,11 @@ _HANDLE_FLAG_INHERIT = 0x00000001 _PROC_THREAD_ATTRIBUTE_HANDLE_LIST = 0x00020002 _PROC_THREAD_ATTRIBUTE_JOB_LIST = 0x0002000D +_PROCESS_QUERY_LIMITED_INFORMATION = 0x00001000 +_SYNCHRONIZE = 0x00100000 _INFINITE = 0xFFFFFFFF +_WAIT_OBJECT_0 = 0 +_WAIT_TIMEOUT = 0x00000102 _WAIT_FAILED = 0xFFFFFFFF @@ -60,7 +68,7 @@ def __init__(self, winerror: int | None) -> None: class JobNotDrained(Exception): - """Termination was requested and members were still active afterwards.""" + """Termination has not established that every process exited.""" def __init__(self, active: int | None) -> None: self.active = active @@ -116,6 +124,14 @@ class BasicAccountingInformation(ctypes.Structure): ) +class BasicProcessIdList(ctypes.Structure): + _fields_ = ( + ("NumberOfAssignedProcesses", wintypes.DWORD), + ("NumberOfProcessIdsInList", wintypes.DWORD), + ("ProcessIdList", ctypes.c_size_t * 1), + ) + + class StartupInfo(ctypes.Structure): _fields_ = ( ("cb", wintypes.DWORD), @@ -201,6 +217,10 @@ def _kernel32() -> Any: library.CloseHandle.restype = wintypes.BOOL library.OpenJobObjectW.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.LPCWSTR] library.OpenJobObjectW.restype = handle + library.OpenProcess.argtypes = [wintypes.DWORD, wintypes.BOOL, wintypes.DWORD] + library.OpenProcess.restype = handle + library.IsProcessInJob.argtypes = [handle, handle, ctypes.POINTER(wintypes.BOOL)] + library.IsProcessInJob.restype = wintypes.BOOL library.CreateProcessW.argtypes = [ wintypes.LPCWSTR, ctypes.POINTER(ctypes.c_wchar), @@ -363,8 +383,12 @@ class Job: def __init__(self, handle: ctypes.c_void_p, name: str) -> None: self._handle: ctypes.c_void_p | None = handle self.name = name + self._processes: dict[int, ctypes.c_void_p] = {} + self._members_captured = False def close(self) -> None: + if self._processes: + raise JobNotDrained(None) handle = self._handle if handle is not None and not _kernel32().CloseHandle(handle): raise _win_os_error(_last_error()) @@ -394,26 +418,118 @@ def active_processes(self) -> int | None: return None return int(info.ActiveProcesses) + def _seal_membership(self) -> None: + # A zero limit denies late children without terminating existing members. + limits = ExtendedLimitInformation() + limits.BasicLimitInformation.LimitFlags = ( + _JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | _JOB_OBJECT_LIMIT_ACTIVE_PROCESS + ) + limits.BasicLimitInformation.ActiveProcessLimit = 0 + if not _kernel32().SetInformationJobObject( + self._open_handle(), + _JOB_OBJECT_EXTENDED_LIMIT_INFORMATION, + ctypes.byref(limits), + ctypes.sizeof(limits), + ): + raise _win_os_error(_last_error()) + + def _member_pids(self) -> tuple[int, ...]: + capacity = 1 + while True: + buffer = ctypes.create_string_buffer( + ctypes.sizeof(BasicProcessIdList) + + (capacity - 1) * ctypes.sizeof(ctypes.c_size_t) + ) + info = ctypes.cast(buffer, ctypes.POINTER(BasicProcessIdList)).contents + complete = _kernel32().QueryInformationJobObject( + self._open_handle(), + _JOB_OBJECT_BASIC_PROCESS_ID_LIST, + buffer, + ctypes.sizeof(buffer), + None, + ) + if not complete and (code := _last_error()) != _ERROR_MORE_DATA: + raise _win_os_error(code) + assigned = int(info.NumberOfAssignedProcesses) + count = int(info.NumberOfProcessIdsInList) + if not complete or count < assigned: + capacity = max(capacity * 2, assigned) + continue + values = ctypes.cast( + ctypes.byref(buffer, BasicProcessIdList.ProcessIdList.offset), + ctypes.POINTER(ctypes.c_size_t), + ) + return tuple(int(values[index]) for index in range(count)) + + def _capture_members(self) -> None: + for pid in self._member_pids(): + if pid in self._processes: + continue + process = _kernel32().OpenProcess( + _SYNCHRONIZE | _PROCESS_QUERY_LIMITED_INFORMATION, False, pid + ) + if not process: + code = _last_error() + if code == _ERROR_INVALID_PARAMETER and pid not in self._member_pids(): + continue + raise _win_os_error(code) + self._processes[pid] = process + # Retained handles keep object identity stable across PID reuse. Failed + # membership checks keep the handles owned, but cannot authorize termination. + for pid, process in tuple(self._processes.items()): + member = wintypes.BOOL() + if not _kernel32().IsProcessInJob( + process, self._open_handle(), ctypes.byref(member) + ): + raise _win_os_error(_last_error()) + if not member.value: + self._close_process_handle(pid) + + def _close_process_handle(self, pid: int) -> None: + if not _kernel32().CloseHandle(self._processes[pid]): + raise _win_os_error(_last_error()) + del self._processes[pid] + + def _reap_members(self) -> None: + for pid, process in tuple(self._processes.items()): + state = _kernel32().WaitForSingleObject(process, 0) + if state == _WAIT_TIMEOUT: + continue + if state != _WAIT_OBJECT_0: + raise _win_os_error(_last_error()) + self._close_process_handle(pid) + def terminate(self) -> None: - """Mark every member for termination. + """Retain physical-exit evidence before marking members for termination. Members blocked in a synchronous pipe read stay inside that call until the other end closes. Wait only after that end is closed. """ + if not self._members_captured: + self._seal_membership() + self._capture_members() + self._members_captured = True if not _kernel32().TerminateJobObject(self._open_handle(), 1): raise _win_os_error(_last_error()) async def wait_until_idle(self, timeout: float) -> None: - """Observe tree exit without releasing ownership of the job.""" + """Observe physical exit and job accounting without releasing the job.""" + if self._processes and not self._members_captured: + raise JobNotDrained(None) loop = asyncio.get_running_loop() deadline = loop.time() + timeout while True: + self._reap_members() active = self.active_processes() - if active == 0: + if active == 0 and not self._processes: return remaining = deadline - loop.time() if active is None or remaining <= 0: - logger.warning("Codex job still has %s active processes", active) + logger.warning( + "Codex job still has %s active processes and %s pending exits", + active, + len(self._processes), + ) raise JobNotDrained(active) await asyncio.sleep(min(_JOB_POLL_S, remaining)) diff --git a/tests/integrations/codex/conftest.py b/tests/integrations/codex/conftest.py index 5d3fc862c..d7ffc14f5 100644 --- a/tests/integrations/codex/conftest.py +++ b/tests/integrations/codex/conftest.py @@ -12,10 +12,20 @@ import pytest_asyncio from band.integrations.codex import stdio_client, winspawn +from tests.integrations.codex.jobpeer import NativeJobPeer from tests.integrations.codex.lifecycle import WindowsLifecycle from tests.integrations.codex.lookup import WindowsLookup +@pytest.fixture +def native_job(monkeypatch: pytest.MonkeyPatch) -> NativeJobPeer: + peer = NativeJobPeer() + monkeypatch.setattr(winspawn, "_kernel32", lambda: peer.kernel) + monkeypatch.setattr(winspawn, "_last_error", lambda: peer.error) + monkeypatch.setattr(winspawn, "_win_os_error", lambda code: OSError(code)) + return peer + + @pytest.fixture def windows_lookup(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> WindowsLookup: peer = WindowsLookup(tmp_path) diff --git a/tests/integrations/codex/jobpeer.py b/tests/integrations/codex/jobpeer.py new file mode 100644 index 000000000..247eccad1 --- /dev/null +++ b/tests/integrations/codex/jobpeer.py @@ -0,0 +1,134 @@ +"""Native job boundary where accounting can finish before physical process exit.""" + +from __future__ import annotations + +import ctypes +from ctypes import wintypes +from types import SimpleNamespace + +from band.integrations.codex.winspawn import ( + BasicAccountingInformation, + ExtendedLimitInformation, +) + + +class NativeJobPeer: + def __init__(self) -> None: + self.handle = ctypes.c_void_p(1) + self.pids = [101] + self.exited: set[int] = set() + self.reused: set[int] = set() + self.opened: set[int] = set() + self.waited: list[int] = [] + self.active = 1 + self.error = 0 + self.open_error = 0 + self.wait_error = False + self.close_error = False + self.query_error = False + self.terminated = False + self.closed = False + self.sealed = False + self.kernel = SimpleNamespace( + SetInformationJobObject=self.set_limits, + QueryInformationJobObject=self.query, + TerminateJobObject=self.terminate, + OpenProcess=self.open_process, + IsProcessInJob=self.is_member, + WaitForSingleObject=self.wait, + CloseHandle=self.close, + ) + + def set_limits( + self, handle: ctypes.c_void_p, kind: int, data: object, size: int + ) -> bool: + assert handle is self.handle + assert kind == 9 + assert size == ctypes.sizeof(ExtendedLimitInformation) + limits = ctypes.cast(data, ctypes.POINTER(ExtendedLimitInformation)).contents + basic = limits.BasicLimitInformation + assert basic.LimitFlags & 0x2000, "sealing must preserve kill-on-close" + self.sealed = bool(basic.LimitFlags & 8 and basic.ActiveProcessLimit == 0) + return True + + def query( + self, + handle: ctypes.c_void_p, + kind: int, + data: object, + size: int, + returned: object, + ) -> bool: + assert handle is self.handle + if self.query_error: + self.error = 5 + return False + if kind == 1: + info = ctypes.cast( + data, ctypes.POINTER(BasicAccountingInformation) + ).contents + info.ActiveProcesses = self.active + return True + assert kind == 3 + assert self.sealed, "new descendants must be excluded before the snapshot" + assert not self.terminated, "termination can remove unsignaled members" + header_size = 2 * ctypes.sizeof(wintypes.DWORD) + header = ctypes.cast(data, ctypes.POINTER(wintypes.DWORD)) + capacity = (size - header_size) // ctypes.sizeof(ctypes.c_size_t) + header[0] = len(self.pids) + header[1] = min(capacity, len(self.pids)) + address = ctypes.cast(data, ctypes.c_void_p).value + assert address is not None + entries = (ctypes.c_size_t * header[1]).from_address(address + header_size) + entries[:] = self.pids[: header[1]] + if capacity < len(self.pids): + self.error = 234 + return False + return True + + def terminate(self, handle: ctypes.c_void_p, code: int) -> bool: + assert handle is self.handle + assert code == 1 + self.terminated = True + self.active = 0 + return True + + def open_process( + self, access: int, inherit: bool, pid: int + ) -> ctypes.c_void_p | None: + assert access == 0x101000 + assert not inherit + if self.open_error: + self.error = self.open_error + return None + self.opened.add(pid) + return ctypes.c_void_p(pid) + + def is_member( + self, process: ctypes.c_void_p, job: ctypes.c_void_p, result: object + ) -> bool: + assert job is self.handle + ctypes.cast(result, ctypes.POINTER(wintypes.BOOL))[0] = ( + process.value not in self.reused + ) + return True + + def wait(self, handle: ctypes.c_void_p, timeout: int) -> int: + assert timeout == 0, "process waits must not block the event loop" + assert handle.value is not None + self.waited.append(handle.value) + if self.wait_error: + self.error = 6 + return 0xFFFFFFFF + return 0 if handle.value in self.exited else 258 + + def close(self, handle: ctypes.c_void_p) -> bool: + if self.close_error: + self.error = 6 + return False + if handle is self.handle: + self.closed = True + else: + assert handle.value is not None + self.opened.remove(handle.value) + return True diff --git a/tests/integrations/codex/test_cleanup.py b/tests/integrations/codex/test_cleanup.py index b382d7644..f287f31c3 100644 --- a/tests/integrations/codex/test_cleanup.py +++ b/tests/integrations/codex/test_cleanup.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio +import ctypes import logging import os import shutil @@ -311,9 +312,7 @@ def deny(_name: str) -> object: async def test_idle_wait_preserves_ownership_until_explicit_release( self, active: int | None, drained: bool ) -> None: - job = Job.__new__(Job) - job._handle = object() # type: ignore[assignment] - job.name = "test" + job = Job(ctypes.c_void_p(1), "test") closed = False def active_processes() -> int | None: diff --git a/tests/integrations/codex/test_job.py b/tests/integrations/codex/test_job.py index c801264cf..d907231b2 100644 --- a/tests/integrations/codex/test_job.py +++ b/tests/integrations/codex/test_job.py @@ -8,6 +8,7 @@ import pytest from band.integrations.codex import winspawn +from tests.integrations.codex.jobpeer import NativeJobPeer class TestJobHandleOwnership: @@ -32,25 +33,167 @@ def close_handle(current: ctypes.c_void_p) -> bool: job.close() assert attempted == [handle, handle] - def test_failed_termination_is_reported_without_releasing_the_job( - self, monkeypatch: pytest.MonkeyPatch + async def test_failed_termination_is_reported_without_releasing_the_job( + self, native_job: NativeJobPeer ) -> None: - handle = ctypes.c_void_p(1) + handle = native_job.handle job = winspawn.Job(handle, "test") terminated: list[ctypes.c_void_p] = [] def terminate(current: ctypes.c_void_p, _code: int) -> bool: terminated.append(current) - return len(terminated) > 1 + if len(terminated) == 1: + native_job.error = 5 + return False + return native_job.terminate(current, _code) - monkeypatch.setattr( - winspawn, - "_kernel32", - lambda: SimpleNamespace(TerminateJobObject=terminate), - ) - monkeypatch.setattr(winspawn, "_last_error", lambda: 5) - monkeypatch.setattr(winspawn, "_win_os_error", lambda code: OSError(code)) + native_job.kernel.TerminateJobObject = terminate with pytest.raises(OSError): job.terminate() job.terminate() assert terminated == [handle, handle] + native_job.exited.add(101) + await job.wait_until_idle(0) + job.close() + + async def test_zero_accounting_retains_unsignaled_member_until_retry( + self, native_job: NativeJobPeer + ) -> None: + job = winspawn.Job(native_job.handle, "test") + job.terminate() + assert native_job.active == 0 + with pytest.raises(winspawn.JobNotDrained): + await job.wait_until_idle(0) + assert not native_job.closed + assert native_job.opened == {101} + native_job.exited.add(101) + job.terminate() + await job.wait_until_idle(0) + job.close() + assert native_job.closed + assert not native_job.opened + + async def test_all_members_must_exit_when_the_pid_buffer_grows( + self, native_job: NativeJobPeer + ) -> None: + native_job.pids = list(range(101, 166)) + native_job.active = len(native_job.pids) + job = winspawn.Job(native_job.handle, "test") + job.terminate() + native_job.exited.update(native_job.pids[:-1]) + with pytest.raises(winspawn.JobNotDrained): + await job.wait_until_idle(0) + assert native_job.opened == {165} + native_job.exited.add(165) + await job.wait_until_idle(0) + job.close() + assert not native_job.opened + + async def test_reused_pid_is_not_waited_as_a_job_member( + self, native_job: NativeJobPeer + ) -> None: + native_job.reused.add(101) + job = winspawn.Job(native_job.handle, "test") + job.terminate() + await job.wait_until_idle(0) + job.close() + assert not native_job.opened + assert not native_job.waited + + def test_denied_member_handle_preserves_job_custody( + self, native_job: NativeJobPeer + ) -> None: + native_job.open_error = 5 + job = winspawn.Job(native_job.handle, "test") + with pytest.raises(OSError): + job.terminate() + assert not native_job.terminated + assert not native_job.closed + + async def test_native_wait_failure_retains_the_member_handle( + self, native_job: NativeJobPeer + ) -> None: + job = winspawn.Job(native_job.handle, "test") + job.terminate() + native_job.wait_error = True + with pytest.raises(OSError): + await job.wait_until_idle(0) + assert native_job.opened == {101} + assert not native_job.closed + native_job.wait_error = False + native_job.exited.add(101) + await job.wait_until_idle(0) + job.close() + assert not native_job.opened + + async def test_failed_member_handle_close_keeps_retryable_ownership( + self, native_job: NativeJobPeer + ) -> None: + job = winspawn.Job(native_job.handle, "test") + job.terminate() + native_job.exited.add(101) + native_job.close_error = True + with pytest.raises(OSError): + await job.wait_until_idle(0) + assert native_job.opened == {101} + assert not native_job.closed + native_job.close_error = False + await job.wait_until_idle(0) + job.close() + assert not native_job.opened + + async def test_job_close_cannot_discard_pending_physical_exit( + self, native_job: NativeJobPeer + ) -> None: + job = winspawn.Job(native_job.handle, "test") + job.terminate() + with pytest.raises(winspawn.JobNotDrained): + job.close() + assert not native_job.closed + native_job.exited.add(101) + await job.wait_until_idle(0) + job.close() + assert native_job.closed + + async def test_failed_membership_query_keeps_the_handle_for_retry( + self, native_job: NativeJobPeer + ) -> None: + native_job.kernel.IsProcessInJob = lambda *_args: False + native_job.error = 5 + job = winspawn.Job(native_job.handle, "test") + with pytest.raises(OSError): + job.terminate() + assert native_job.opened == {101} + assert not native_job.terminated + native_job.kernel.IsProcessInJob = native_job.is_member + job.terminate() + native_job.exited.add(101) + await job.wait_until_idle(0) + job.close() + assert not native_job.opened + + async def test_vanished_member_pid_is_confirmed_against_the_job( + self, native_job: NativeJobPeer + ) -> None: + def gone(_access: int, _inherit: bool, _pid: int) -> None: + native_job.pids.clear() + native_job.active = 0 + native_job.error = 87 + + native_job.kernel.OpenProcess = gone + job = winspawn.Job(native_job.handle, "test") + job.terminate() + await job.wait_until_idle(0) + job.close() + assert not native_job.opened + assert not native_job.waited + + def test_invalid_member_pid_still_in_the_job_is_not_suppressed( + self, native_job: NativeJobPeer + ) -> None: + native_job.open_error = 87 + job = winspawn.Job(native_job.handle, "test") + with pytest.raises(OSError): + job.terminate() + assert not native_job.terminated + assert not native_job.closed From ba4c876737fd0c193eb28f7909ab7a6ed5c9329e Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Mon, 19 Oct 2026 04:12:57 +0300 Subject: [PATCH 24/47] test(codex): require metadata-only transport diagnostics --- .../adapters/codex/test_probe_diagnostics.py | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/adapters/codex/test_probe_diagnostics.py b/tests/adapters/codex/test_probe_diagnostics.py index 9cf7a7ad4..e9a205a42 100644 --- a/tests/adapters/codex/test_probe_diagnostics.py +++ b/tests/adapters/codex/test_probe_diagnostics.py @@ -3,10 +3,12 @@ from __future__ import annotations import asyncio +import json import logging import pytest +from band.integrations.codex.stdio_client import CodexStdioClient from tests.adapters.codex.kit import send_bootstrap from tests.adapters.codex.probes import ProbeHost from tests.adapters.waiting import spin @@ -29,7 +31,7 @@ async def test_real_probe_wire_and_stderr_are_metadata_only( probe_host.assert_exited() assert SENSITIVE_DIAGNOSTIC not in caplog.text - async def test_existing_room_transport_keeps_its_diagnostic_policy( + async def test_room_wire_and_stderr_are_also_metadata_only( self, probe_host: ProbeHost, caplog: pytest.LogCaptureFixture ) -> None: adapter = probe_host.adapter( @@ -42,7 +44,26 @@ async def test_existing_room_transport_keeps_its_diagnostic_policy( assert "initialize" in probe_host.methods probe_host.assert_exited() - assert SENSITIVE_DIAGNOSTIC in caplog.text + assert SENSITIVE_DIAGNOSTIC not in caplog.text + + @pytest.mark.parametrize("request_id", [None, 7], ids=["notification", "request"]) + async def test_full_transport_queue_logs_only_the_dropped_event_kind( + self, caplog: pytest.LogCaptureFixture, request_id: int | None + ) -> None: + client = CodexStdioClient() + client._events = asyncio.Queue(maxsize=1) + payload = {"method": SENSITIVE_DIAGNOSTIC, "id": request_id} + + with caplog.at_level(logging.WARNING): + await client._dispatch_rpc_message(json.dumps(payload)) + await client._dispatch_rpc_message(json.dumps(payload)) + + event = await client.recv_event(timeout_s=1) + assert event.method == SENSITIVE_DIAGNOSTIC + assert event.kind == ("notification" if request_id is None else "request") + assert "Event queue full" in caplog.text + assert event.kind in caplog.text + assert SENSITIVE_DIAGNOSTIC not in caplog.text async def test_late_close_failure_keeps_the_original_error_and_retry_without_raw_log( self, From 06157aae97ce1045816eefa4dfbf712e2eb4d4b2 Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Mon, 19 Oct 2026 04:14:25 +0300 Subject: [PATCH 25/47] fix(codex): decouple host probes from retained MCP startup --- docs/adapters/codex.md | 88 +++++++++++---------- src/band/adapters/codex.py | 48 +++++------ src/band/integrations/codex/rpc_base.py | 29 +++---- src/band/integrations/codex/stdio_client.py | 34 +++----- tests/adapters/codex/lifecycleprofile.py | 8 +- tests/adapters/codex/probes.py | 4 +- 6 files changed, 102 insertions(+), 109 deletions(-) diff --git a/docs/adapters/codex.md b/docs/adapters/codex.md index 315c59aeb..11b5cc512 100644 --- a/docs/adapters/codex.md +++ b/docs/adapters/codex.md @@ -8,6 +8,51 @@ see [idle room resource retention](../idle-room-release.md). For existing integrations, follow the [SDK 4.x to 5.x migration](../acp.md#upgrading-from-sdk-4x-to-5x). +## Host preflight + +`await adapter.preflight(timeout_s=60.0)` launches a throwaway app-server using +the adapter's `codex_command`, `codex_env`, client name/title/version, +`experimental_api` and `skill_roots`. It sends `initialize` and `account/read` +without creating a Codex thread, spending a model turn, or claiming a Band room +or workspace. + +The result is a `PreflightResult` with `ok`, `reason` and `remedy`. A missing +account fails only when Codex explicitly reports `requiresOpenaiAuth=true`; +an unsupported or refused `account/read` is not proof that login is missing. +A passed handshake is not proof of provider authentication, subscription or +inference entitlement. + +Room and standalone-probe transport diagnostics omit raw app-server stdout, +stderr and exception text, including cleanup that finishes after caller +cancellation. Malformed frames report transport, type and length; full queues +report only the dropped event kind. Original cleanup errors remain available to +callers and retries. + +```python notest +from band.adapters.codex import CodexAdapter, CodexAdapterConfig + +adapter = CodexAdapter(CodexAdapterConfig()) +result = await adapter.preflight(timeout_s=30) +``` + +The timeout bounds handshake/RPC work and a separate cleanup wait. Cancellation +propagates. Cleanup failures raise `CleanupIncompleteError` instead of reporting +a successful or ordinary failed preflight, and the adapter retains the client, +including a partially created process or Windows job. Keep that adapter and +retry `await adapter.cleanup_all()` to join or retry cleanup. Shutdown attempts +both probe and room cleanup even when either fails. A custom-tool callback +requesting shutdown can receive a retryable pending-cleanup error while its own +turn unwinds; it is not cancelled by that stop. Successful calls close their +throwaway process before returning; a second preflight starts a fresh one. + +Windows room and probe cleanup seals each owned job against new children and +retains member process handles before requesting termination. It waits for +physical member exit as well as empty job accounting; accounting alone does not +prove directory and file handles were released. Failed cleanup retains that same +owner, job and any pending process handles for retry. + +## Room behavior + - **Where settings go.** Runtime settings live in `CodexAdapterConfig`, which forbids unknown fields, so a misplaced `emit=` or a typo fails construction. `emit=`, `capabilities=` and `additional_tools=` go on `CodexAdapter`. @@ -38,11 +83,9 @@ For existing integrations, follow the [SDK 4.x to 5.x migration](../acp.md#upgra On Windows, the client owns the helper and its process-tree job from startup through cleanup. It starts the interpreter image directly so a virtual-environment launcher cannot hide another helper process. Cleanup confirms the helper has - exited, seals the job against new children, and retains member process handles - before requesting termination. It waits for those handles to signal physical - exit as well as empty job accounting; accounting alone does not prove directory - and file handles were released. Failed cleanup retains that same owner, job, - and any pending process handles for retry. + exited before terminating and draining the job; an empty job alone does not + establish helper exit. + Failed cleanup retains that same owner and job for retry. - **Leaving is final.** Leaving a room stops it admitting work at once. A leave that fails or runs over keeps the process, thread and workspace claim, and the room refuses work until a retried leave succeeds; a later rejoin @@ -114,38 +157,3 @@ request and approval IDs stay internal; a fresh request gets a new chat token after completion or room cleanup, even when Codex reuses its native ID. A redelivery of a still-pending request keeps its displayed token. -## Host preflight - -`await adapter.preflight(timeout_s=60.0)` launches a throwaway app-server using -the adapter's `codex_command`, `codex_env`, client name/title/version, -`experimental_api` and `skill_roots`. It sends `initialize` and `account/read` -without creating a Codex thread, spending a model turn, or claiming a Band room -or workspace. - -The result is a `PreflightResult` with `ok`, `reason` and `remedy`. A missing -account fails only when Codex explicitly reports `requiresOpenaiAuth=true`; -an unsupported or refused `account/read` is not proof that login is missing. -A passed handshake is not proof of provider authentication, subscription or -inference entitlement. - -Standalone probes emit metadata-only diagnostics: raw app-server stdout, stderr, -and exception text are not logged, including cleanup that finishes after caller -cancellation. Original cleanup errors remain available to callers and retries. -Ordinary room transports keep their existing diagnostic policy. - -```python notest -from band.adapters.codex import CodexAdapter, CodexAdapterConfig - -adapter = CodexAdapter(CodexAdapterConfig()) -result = await adapter.preflight(timeout_s=30) -``` - -The timeout bounds handshake/RPC work and a separate cleanup wait. Cancellation -propagates. Cleanup failures raise `CleanupIncompleteError` instead of reporting -a successful or ordinary failed preflight, and the adapter retains the client, -including a partially created process or Windows job. Keep that adapter and -retry `await adapter.cleanup_all()` to join or retry cleanup. Shutdown attempts -both probe and room cleanup even when either fails. A custom-tool callback -requesting shutdown can receive a retryable pending-cleanup error while its own -turn unwinds; it is not cancelled by that stop. Successful calls close their -throwaway process before returning; a second preflight starts a fresh one. diff --git a/src/band/adapters/codex.py b/src/band/adapters/codex.py index affcfdf12..b17bde8b5 100644 --- a/src/band/adapters/codex.py +++ b/src/band/adapters/codex.py @@ -35,6 +35,7 @@ from band.converters.codex import CodexHistoryConverter from band.converters.helpers import build_replay_messages +from band.core import exceptions, harness from band.core.adapterconfig import EnvAdapterConfig from band.core.content import has_visible_content from band.core.defaultmodels import OPENAI_MODEL @@ -43,12 +44,7 @@ deliver_notice, reraise_delivery_cause, ) -from band.core.exceptions import ( - BandConnectionError, - CleanupIncompleteError, - InputNotAccepted, -) -from band.core.harness import PreflightResult +from band.core.exceptions import InputNotAccepted from band.core.protocols import ( FAILURE_CODE_TIMEOUT, GENERIC_PROVIDER_FAILURE_MESSAGE, @@ -1938,7 +1934,13 @@ async def _acquire( self._read_native_events(room, client, generation), label=f"Room {room.room_id} Codex event reader", ) - await self._initialize_client(client) + await client.initialize( + client_name=self.config.client_name, + client_title=self.config.client_title, + client_version=self.config.client_version, + experimental_api=self.config.experimental_api, + ) + await self._register_skill_roots(client) if room.selected_model is None: room.selected_model = await self._select_model(client) logger.info( @@ -2194,18 +2196,18 @@ def _forget_room_thread(self, room_id: str) -> str | None: self._raw_history_by_room.pop(room_id, None) return thread_id - async def preflight(self, *, timeout_s: float = 60.0) -> PreflightResult: + async def preflight(self, *, timeout_s: float = 60.0) -> harness.PreflightResult: """Check the configured app-server and its reported login without a turn.""" try: return await self._probe_harness(self._login_state, timeout_s=timeout_s) - except CleanupIncompleteError: + except exceptions.CleanupIncompleteError: raise - except BandConnectionError as error: + except exceptions.BandConnectionError as error: cause = error.__cause__ if isinstance(cause, FileNotFoundError) or ( isinstance(cause, SpawnFailure) and cause.winerror in MISSING_EXECUTABLE ): - return PreflightResult.failed( + return harness.PreflightResult.failed( "Codex CLI binary not found.", "Install Codex (`npm install -g @openai/codex`) or set " "CodexAdapterConfig.codex_command.", @@ -2215,24 +2217,24 @@ async def preflight(self, *, timeout_s: float = 60.0) -> PreflightResult: return self._handshake_failed() @staticmethod - def _handshake_failed() -> PreflightResult: - return PreflightResult.failed( + def _handshake_failed() -> harness.PreflightResult: + return harness.PreflightResult.failed( "Codex app-server did not complete its handshake.", "Run `codex app-server` in a terminal to check the install, then retry.", ) @staticmethod - async def _login_state(client: CodexClientProtocol) -> PreflightResult: + async def _login_state(client: CodexClientProtocol) -> harness.PreflightResult: try: account = await client.request(CodexRequestMethod.ACCOUNT_READ, {}) except CodexJsonRpcError: # Unsupported or refused metadata is not evidence of a missing login. - return PreflightResult.passed() + return harness.PreflightResult.passed() if account.get("account") is None and account.get("requiresOpenaiAuth") is True: - return PreflightResult.failed( + return harness.PreflightResult.failed( "Codex is not logged in.", "Run `codex login`, then retry." ) - return PreflightResult.passed() + return harness.PreflightResult.passed() async def _probe_harness( self, @@ -2241,15 +2243,18 @@ async def _probe_harness( timeout_s: float, ) -> _ProbeResult: async def inspect() -> _ProbeResult: - client = self._build_client(None) + client = CodexStdioClient( + command=self.config.codex_command, + env=self.config.codex_env, + ) self._harness_probe.own(client.close) await client.connect() - await self._initialize_client(client) + await self._initialize_probe(client) return await operation(client) return await self._harness_probe.run(inspect, timeout_s=timeout_s) - async def _initialize_client(self, client: CodexClientProtocol) -> None: + async def _initialize_probe(self, client: CodexClientProtocol) -> None: await client.initialize( client_name=self.config.client_name, client_title=self.config.client_title, @@ -2269,12 +2274,11 @@ async def _register_skill_roots(self, client: CodexClientProtocol) -> None: except CodexJsonRpcError as exc: raise RuntimeError(f"Codex rejected skill_roots {roots}: {exc}") from exc - def _build_client(self, workspace: str | None) -> CodexClientProtocol: + def _build_client(self, workspace: str) -> CodexClientProtocol: return CodexStdioClient( command=self.config.codex_command, cwd=workspace, env=self.config.codex_env, - diagnostic_details=workspace is not None, ) async def _select_model(self, client: CodexClientProtocol) -> str: diff --git a/src/band/integrations/codex/rpc_base.py b/src/band/integrations/codex/rpc_base.py index 2d24860bb..3a2e62e04 100644 --- a/src/band/integrations/codex/rpc_base.py +++ b/src/band/integrations/codex/rpc_base.py @@ -82,11 +82,9 @@ def __init__( *, retry_policy: OverloadRetryPolicy | None = None, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, - diagnostic_details: bool = True, ) -> None: self.retry_policy = retry_policy or OverloadRetryPolicy() self._sleep = sleep - self.diagnostic_details = diagnostic_details self._pending: dict[JsonRpcId, asyncio.Future[dict[str, Any]]] = {} self._events: asyncio.Queue[RpcEvent] = asyncio.Queue(maxsize=10_000) @@ -266,16 +264,19 @@ async def _dispatch_rpc_message(self, text: str) -> None: message = json.loads(text) except json.JSONDecodeError: logger.warning( - "Skipping non-JSON %s: %s", + "Skipping non-JSON %s (type=%s, length=%d)", self._transport_label, - text if self.diagnostic_details else "", + type(text).__name__, + len(text), ) return if not isinstance(message, dict): logger.warning( - "Skipping non-object JSON-RPC payload: %s", - message if self.diagnostic_details else "", + "Skipping non-object JSON-RPC %s (type=%s, length=%d)", + self._transport_label, + type(message).__name__, + len(text), ) return @@ -294,10 +295,7 @@ async def _dispatch_rpc_message(self, text: str) -> None: ) ) except asyncio.QueueFull: - logger.warning( - "Event queue full; dropping server request %s", - method if self.diagnostic_details else "", - ) + logger.warning("Event queue full; dropping server request") return if method: @@ -312,10 +310,7 @@ async def _dispatch_rpc_message(self, text: str) -> None: ) ) except asyncio.QueueFull: - logger.warning( - "Event queue full; dropping notification %s", - method if self.diagnostic_details else "", - ) + logger.warning("Event queue full; dropping notification") return if msg_id is not None: @@ -325,8 +320,10 @@ async def _dispatch_rpc_message(self, text: str) -> None: return logger.debug( - "Ignoring unknown JSON-RPC payload: %s", - message if self.diagnostic_details else "", + "Ignoring unknown JSON-RPC %s (type=%s, length=%d)", + self._transport_label, + type(message).__name__, + len(text), ) def _next_request_id(self) -> int: diff --git a/src/band/integrations/codex/stdio_client.py b/src/band/integrations/codex/stdio_client.py index 5f1de7e24..1248d523c 100644 --- a/src/band/integrations/codex/stdio_client.py +++ b/src/band/integrations/codex/stdio_client.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import Any -from band.core.exceptions import BandConnectionError +from band.core.exceptions import BandConnectionError, CleanupIncompleteError from band.integrations.codex.rpc_base import BaseJsonRpcClient, OverloadRetryPolicy from band.integrations.codex.winspawn import ( MISSING_EXECUTABLE, @@ -115,8 +115,7 @@ async def kill_process_tree(process: asyncio.subprocess.Process) -> None: class CodexStdioClient(BaseJsonRpcClient): """Bidirectional JSON-RPC client with a single stdout read loop. - Disabling diagnostic details keeps wire data and exception text out of logs - throughout startup and retryable cleanup; callers still receive the errors. + Transport logs report metadata only; callers still receive original errors. """ def __init__( @@ -127,13 +126,8 @@ def __init__( env: Mapping[str, str] | None = None, retry_policy: OverloadRetryPolicy | None = None, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, - diagnostic_details: bool = True, ) -> None: - super().__init__( - retry_policy=retry_policy, - sleep=sleep, - diagnostic_details=diagnostic_details, - ) + super().__init__(retry_policy=retry_policy, sleep=sleep) self._transport_label = "codex output line" self.command = tuple(command) if command else self._default_codex_command() self.cwd = cwd @@ -272,12 +266,9 @@ async def close(self) -> None: def _log_close_failure(self, task: asyncio.Task[None]) -> None: if not task.cancelled() and (error := task.exception()) is not None: - if self.diagnostic_details: - logger.error("Codex subprocess cleanup failed", exc_info=error) - else: - logger.error( - "Codex subprocess cleanup failed (%s)", type(error).__name__ - ) + logger.error( + "Codex subprocess cleanup failed (type=%s)", type(error).__name__ + ) async def _close_process(self) -> None: readers = [ @@ -304,8 +295,8 @@ async def _close_process(self) -> None: await kill_process_tree(process) await close_stdin(process) if not await await_returncode(process, _EXIT_OBSERVE_S): - raise TimeoutError( - f"Codex process {process.pid} still alive after tree kill" + raise CleanupIncompleteError( + "Codex process did not report exit after tree cleanup" ) self._fail_pending("Codex stdio client closed") finally: @@ -335,7 +326,7 @@ async def _read_stdout_loop(self) -> None: except asyncio.CancelledError: raise except Exception: - logger.error("Codex stdout loop failed", exc_info=self.diagnostic_details) + logger.error("Codex stdout loop failed") self._fail_pending("Codex stdout loop failed") async def _read_stderr_loop(self) -> None: @@ -346,10 +337,9 @@ async def _read_stderr_loop(self) -> None: line = await self._proc.stderr.readline() if not line: return - text = line.decode("utf-8", errors="replace").rstrip() - if text and self.diagnostic_details: - logger.debug("codex stderr: %s", text) + if line.rstrip(): + logger.debug("codex stderr (type=bytes, length=%d)", len(line)) except asyncio.CancelledError: raise except Exception: - logger.error("Codex stderr loop failed", exc_info=self.diagnostic_details) + logger.error("Codex stderr loop failed") diff --git a/tests/adapters/codex/lifecycleprofile.py b/tests/adapters/codex/lifecycleprofile.py index a637319a0..34ca58b5e 100644 --- a/tests/adapters/codex/lifecycleprofile.py +++ b/tests/adapters/codex/lifecycleprofile.py @@ -75,14 +75,8 @@ def child( command: Sequence[str] | None, cwd: str | None, env: Mapping[str, str] | None, - diagnostic_details: bool = True, ) -> CodexPeer: - peer = CodexPeer( - command=command, - cwd=cwd, - env=env, - diagnostic_details=diagnostic_details, - ) + peer = CodexPeer(command=command, cwd=cwd, env=env) host.spawned(peer, cwd) return peer diff --git a/tests/adapters/codex/probes.py b/tests/adapters/codex/probes.py index ab70c764b..675b8c4c1 100644 --- a/tests/adapters/codex/probes.py +++ b/tests/adapters/codex/probes.py @@ -40,13 +40,13 @@ def build( command: Sequence[str] | None = None, cwd: str | None = None, env: Mapping[str, str] | None = None, - diagnostic_details: bool = True, + **transport: Any, ) -> ProbeClient: client = ProbeClient( command=command, cwd=cwd, env=env, - diagnostic_details=diagnostic_details, + **transport, ) self.clients.append(client) return client From b2a9a4d580539c617a7f028b63429ef8b601f9af Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Mon, 19 Oct 2026 04:16:31 +0300 Subject: [PATCH 26/47] refactor(codex): isolate physical exit proof from MCP transport --- docs/adapters/codex.md | 1 - src/band/integrations/codex/stdio_client.py | 13 ++++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/adapters/codex.md b/docs/adapters/codex.md index 11b5cc512..441b3d46f 100644 --- a/docs/adapters/codex.md +++ b/docs/adapters/codex.md @@ -156,4 +156,3 @@ Approval commands use the token displayed in the room prompt. Native Codex request and approval IDs stay internal; a fresh request gets a new chat token after completion or room cleanup, even when Codex reuses its native ID. A redelivery of a still-pending request keeps its displayed token. - diff --git a/src/band/integrations/codex/stdio_client.py b/src/band/integrations/codex/stdio_client.py index 1248d523c..f4c2c8486 100644 --- a/src/band/integrations/codex/stdio_client.py +++ b/src/band/integrations/codex/stdio_client.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import Any -from band.core.exceptions import BandConnectionError, CleanupIncompleteError +from band.core.exceptions import BandConnectionError from band.integrations.codex.rpc_base import BaseJsonRpcClient, OverloadRetryPolicy from band.integrations.codex.winspawn import ( MISSING_EXECUTABLE, @@ -271,6 +271,13 @@ def _log_close_failure(self, task: asyncio.Task[None]) -> None: ) async def _close_process(self) -> None: + await self._settle_process() + if self._proc is not None and self._proc.returncode is None: + raise TimeoutError( + f"Codex process {self._proc.pid} still alive after tree kill" + ) + + async def _settle_process(self) -> None: readers = [ task for task in (self._reader_task, self._stderr_task) if task is not None ] @@ -295,8 +302,8 @@ async def _close_process(self) -> None: await kill_process_tree(process) await close_stdin(process) if not await await_returncode(process, _EXIT_OBSERVE_S): - raise CleanupIncompleteError( - "Codex process did not report exit after tree cleanup" + logger.warning( + "Codex process %s still alive after tree kill", process.pid ) self._fail_pending("Codex stdio client closed") finally: From da754a5c03a26fa80d16bd48a6be9fc4ed05d71c Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Mon, 19 Oct 2026 04:27:27 +0300 Subject: [PATCH 27/47] test(codex): retain one-second cleanup for late probe spawns --- tests/adapters/codex/probepeer.py | 2 + tests/adapters/codex/probes.py | 2 + tests/adapters/codex/test_probe_ownership.py | 53 ++++++++++++++++++++ 3 files changed, 57 insertions(+) diff --git a/tests/adapters/codex/probepeer.py b/tests/adapters/codex/probepeer.py index 247e30108..18fd20790 100644 --- a/tests/adapters/codex/probepeer.py +++ b/tests/adapters/codex/probepeer.py @@ -30,6 +30,8 @@ def main() -> None: sys.stdout.flush() sys.stderr.write(diagnostic + "\n") sys.stderr.flush() + if settings.get("startup_hang"): + threading.Event().wait() for line in sys.stdin: message = json.loads(line) with record.open("a", encoding="utf-8") as stream: diff --git a/tests/adapters/codex/probes.py b/tests/adapters/codex/probes.py index 675b8c4c1..45cfbbfb2 100644 --- a/tests/adapters/codex/probes.py +++ b/tests/adapters/codex/probes.py @@ -60,12 +60,14 @@ def adapter( pages: dict[str, Any] | None = None, endless_pages: bool = False, diagnostic: str | None = None, + startup_hang: bool = False, **config: Any, ) -> CodexAdapter: settings = { "replies": replies or {}, "endless_pages": endless_pages, "diagnostic": diagnostic, + "startup_hang": startup_hang, } if pages is not None: settings["pages"] = pages diff --git a/tests/adapters/codex/test_probe_ownership.py b/tests/adapters/codex/test_probe_ownership.py index 3cda9ed18..c3edd3e79 100644 --- a/tests/adapters/codex/test_probe_ownership.py +++ b/tests/adapters/codex/test_probe_ownership.py @@ -14,6 +14,59 @@ class TestProbeOwnership: + async def test_one_second_probe_reaps_a_child_hung_before_handshake( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter(startup_hang=True) + + result = await adapter.preflight(timeout_s=1) + + assert not result.ok + assert "handshake" in result.reason + probe_host.assert_exited() + + async def test_one_second_probe_reaps_a_late_published_hung_child_on_cancel( + self, probe_host: ProbeHost, monkeypatch: pytest.MonkeyPatch + ) -> None: + adapter = probe_host.adapter(startup_hang=True) + create_process = asyncio.create_subprocess_exec + created = asyncio.Event() + return_process = asyncio.Event() + processes: list[asyncio.subprocess.Process] = [] + + async def late_creation( + *args: str, **kwargs: Any + ) -> asyncio.subprocess.Process: + process = await create_process(*args, **kwargs) + processes.append(process) + created.set() + await return_process.wait() + return process + + monkeypatch.setattr(asyncio, "create_subprocess_exec", late_creation) + probing = asyncio.create_task(adapter.preflight(timeout_s=1)) + try: + await asyncio.wait_for(created.wait(), timeout=5) + probing.cancel() + await spin() + return_process.set() + with pytest.raises(asyncio.CancelledError) as stopped: + await probing + assert stopped.value.__cause__ is None + assert processes + assert all(process.returncode is not None for process in processes) + assert all(client._windows_process is None for client in probe_host.clients) + assert adapter._room_clients == {} + finally: + return_process.set() + probing.cancel() + await asyncio.gather(probing, return_exceptions=True) + await adapter.cleanup_all() + for process in processes: + if process.returncode is None: + process.kill() + await process.wait() + async def test_cancel_during_late_subprocess_creation_reaps_the_eventual_child( self, probe_host: ProbeHost, monkeypatch: pytest.MonkeyPatch ) -> None: From 09f4dff5d53ecccdcd1dc83d3cb19c39302548c4 Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Mon, 19 Oct 2026 04:28:47 +0300 Subject: [PATCH 28/47] fix(codex): expedite owned standalone probe shutdown --- docs/adapters/codex.md | 7 +++++++ src/band/adapters/codex.py | 1 + src/band/integrations/codex/stdio_client.py | 11 +++++++++++ 3 files changed, 19 insertions(+) diff --git a/docs/adapters/codex.md b/docs/adapters/codex.md index 441b3d46f..ce27441a9 100644 --- a/docs/adapters/codex.md +++ b/docs/adapters/codex.md @@ -45,6 +45,13 @@ requesting shutdown can receive a retryable pending-cleanup error while its own turn unwinds; it is not cancelled by that stop. Successful calls close their throwaway process before returning; a second preflight starts a fresh one. +Standalone probes set the transport's `graceful_shutdown=False`. After joining +any pending subprocess creation, their retained close owner closes stdin and +requests immediate owned POSIX termination before ordinary settlement. The default +`graceful_shutdown=True` leaves room settlement policy unchanged. Process exit and +reader settlement still require confirmation; an unfinished cleanup remains owned +for retry within the existing reporting bound. + Windows room and probe cleanup seals each owned job against new children and retains member process handles before requesting termination. It waits for physical member exit as well as empty job accounting; accounting alone does not diff --git a/src/band/adapters/codex.py b/src/band/adapters/codex.py index b17bde8b5..2ef834dac 100644 --- a/src/band/adapters/codex.py +++ b/src/band/adapters/codex.py @@ -2246,6 +2246,7 @@ async def inspect() -> _ProbeResult: client = CodexStdioClient( command=self.config.codex_command, env=self.config.codex_env, + graceful_shutdown=False, ) self._harness_probe.own(client.close) await client.connect() diff --git a/src/band/integrations/codex/stdio_client.py b/src/band/integrations/codex/stdio_client.py index f4c2c8486..bd72d6a86 100644 --- a/src/band/integrations/codex/stdio_client.py +++ b/src/band/integrations/codex/stdio_client.py @@ -116,11 +116,14 @@ class CodexStdioClient(BaseJsonRpcClient): """Bidirectional JSON-RPC client with a single stdout read loop. Transport logs report metadata only; callers still receive original errors. + ``graceful_shutdown=False`` requests immediate owned POSIX termination before + ordinary settlement; the default leaves that settlement policy unchanged. """ def __init__( self, *, + graceful_shutdown: bool = True, command: Sequence[str] | None = None, cwd: str | None = None, env: Mapping[str, str] | None = None, @@ -141,6 +144,7 @@ def __init__( self._proc: asyncio.subprocess.Process | None = None self._windows_process: WindowsProcess | None = None self._creation_task: asyncio.Task[None] | None = None + self._graceful_shutdown = graceful_shutdown self._reader_task: asyncio.Task[None] | None = None self._stderr_task: asyncio.Task[None] | None = None self._close_task: asyncio.Task[None] | None = None @@ -292,6 +296,13 @@ async def _settle_process(self) -> None: raise TimeoutError("Codex process creation is still pending") if not self._creation_task.cancelled(): self._creation_task.exception() + if ( + not self._graceful_shutdown + and os.name != "nt" + and (process := self._proc) is not None + ): + await close_stdin(process) + await kill_process_tree(process) # A cancelled pipe read on Windows stays pending while the child is # blocked in a synchronous stdin read, and awaiting that read never # times out. Terminate the tree and close our write end first. From 45bc6ed2ca7addecfeb7ded1b64baf212c28f5c6 Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Mon, 19 Oct 2026 04:31:11 +0300 Subject: [PATCH 29/47] docs(codex): retain canonical Windows process custody wording --- docs/adapters/codex.md | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/docs/adapters/codex.md b/docs/adapters/codex.md index ce27441a9..12887b544 100644 --- a/docs/adapters/codex.md +++ b/docs/adapters/codex.md @@ -52,12 +52,6 @@ requests immediate owned POSIX termination before ordinary settlement. The defau reader settlement still require confirmation; an unfinished cleanup remains owned for retry within the existing reporting bound. -Windows room and probe cleanup seals each owned job against new children and -retains member process handles before requesting termination. It waits for -physical member exit as well as empty job accounting; accounting alone does not -prove directory and file handles were released. Failed cleanup retains that same -owner, job and any pending process handles for retry. - ## Room behavior - **Where settings go.** Runtime settings live in `CodexAdapterConfig`, which @@ -90,9 +84,11 @@ owner, job and any pending process handles for retry. On Windows, the client owns the helper and its process-tree job from startup through cleanup. It starts the interpreter image directly so a virtual-environment launcher cannot hide another helper process. Cleanup confirms the helper has - exited before terminating and draining the job; an empty job alone does not - establish helper exit. - Failed cleanup retains that same owner and job for retry. + exited, seals the job against new children, and retains member process handles + before requesting termination. It waits for those handles to signal physical + exit as well as empty job accounting; accounting alone does not prove directory + and file handles were released. Failed cleanup retains that same owner, job, + and any pending process handles for retry. - **Leaving is final.** Leaving a room stops it admitting work at once. A leave that fails or runs over keeps the process, thread and workspace claim, and the room refuses work until a retried leave succeeds; a later rejoin From f824903299ea1e25d734fa39d83bb0cf5dfa1872 Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Mon, 19 Oct 2026 04:36:19 +0300 Subject: [PATCH 30/47] test(codex): assert probe outcomes without diagnostic wording --- tests/adapters/codex/test_probe_diagnostics.py | 2 +- tests/adapters/codex/test_probe_ownership.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/adapters/codex/test_probe_diagnostics.py b/tests/adapters/codex/test_probe_diagnostics.py index e9a205a42..b0900f2a9 100644 --- a/tests/adapters/codex/test_probe_diagnostics.py +++ b/tests/adapters/codex/test_probe_diagnostics.py @@ -61,7 +61,7 @@ async def test_full_transport_queue_logs_only_the_dropped_event_kind( event = await client.recv_event(timeout_s=1) assert event.method == SENSITIVE_DIAGNOSTIC assert event.kind == ("notification" if request_id is None else "request") - assert "Event queue full" in caplog.text + assert any(record.levelno == logging.WARNING for record in caplog.records) assert event.kind in caplog.text assert SENSITIVE_DIAGNOSTIC not in caplog.text diff --git a/tests/adapters/codex/test_probe_ownership.py b/tests/adapters/codex/test_probe_ownership.py index c3edd3e79..c6648357a 100644 --- a/tests/adapters/codex/test_probe_ownership.py +++ b/tests/adapters/codex/test_probe_ownership.py @@ -22,7 +22,6 @@ async def test_one_second_probe_reaps_a_child_hung_before_handshake( result = await adapter.preflight(timeout_s=1) assert not result.ok - assert "handshake" in result.reason probe_host.assert_exited() async def test_one_second_probe_reaps_a_late_published_hung_child_on_cancel( From 23f633d5c0d11cd7b01779193da131585781dfd6 Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Mon, 19 Oct 2026 04:42:20 +0300 Subject: [PATCH 31/47] docs(codex): explain privacy-safe reader fault containment --- src/band/integrations/codex/stdio_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/band/integrations/codex/stdio_client.py b/src/band/integrations/codex/stdio_client.py index bd72d6a86..bd2007157 100644 --- a/src/band/integrations/codex/stdio_client.py +++ b/src/band/integrations/codex/stdio_client.py @@ -343,7 +343,7 @@ async def _read_stdout_loop(self) -> None: await self._dispatch_rpc_message(line.decode("utf-8", errors="replace")) except asyncio.CancelledError: raise - except Exception: + except Exception: # noqa: BLE001 -- fail pending RPCs without logging sensitive exception text logger.error("Codex stdout loop failed") self._fail_pending("Codex stdout loop failed") @@ -359,5 +359,5 @@ async def _read_stderr_loop(self) -> None: logger.debug("codex stderr (type=bytes, length=%d)", len(line)) except asyncio.CancelledError: raise - except Exception: + except Exception: # noqa: BLE001 -- contain reader faults without logging sensitive exception text logger.error("Codex stderr loop failed") From 5725e28160aa7ebfcad256b3621c83f8b682a1f8 Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Sun, 18 Oct 2026 16:01:01 +0300 Subject: [PATCH 32/47] feat: discover and paginate Codex model catalogs --- docs/adapters/codex.md | 32 ++++ src/band/adapters/codex.py | 134 +++++++++----- tests/adapters/codex/test_model_pagination.py | 91 ++++++++++ tests/adapters/codex/test_model_probe.py | 167 ++++++++++++++++++ 4 files changed, 379 insertions(+), 45 deletions(-) create mode 100644 tests/adapters/codex/test_model_pagination.py create mode 100644 tests/adapters/codex/test_model_probe.py diff --git a/docs/adapters/codex.md b/docs/adapters/codex.md index 12887b544..4ff46eab8 100644 --- a/docs/adapters/codex.md +++ b/docs/adapters/codex.md @@ -52,6 +52,38 @@ requests immediate owned POSIX termination before ordinary settlement. The defau reader settlement still require confirmation; an unfinished cleanup remains owned for retry within the existing reporting bound. +## Host model discovery + +`await adapter.probe_models(timeout_s=60.0)` uses the same owned throwaway +app-server handshake, including the configured skill roots, then paginates +`model/list`. It creates no thread or turn and leaves existing rooms, their +selected models and their workspace claims unchanged. + +The result is the canonical `band.core.model_catalog.ModelCatalog`. Its visible +`ModelChoice` entries preserve Codex IDs, display names, default-model flags, +advertised reasoning efforts and default efforts. Unknown efforts remain `None`; +an explicitly empty effort list becomes `()`. A default-model flag does not +invent `current_model`: a standalone app-server has no selected room model. + +```python notest +catalog = await adapter.probe_models(timeout_s=30) +model_ids = catalog.model_ids +default_choices = tuple(choice for choice in catalog.models if choice.is_default) +``` + +A successful empty catalog has `models=()`. Unsupported or failed RPCs raise +instead of masquerading as an empty catalog. Cancellation and failed cleanup +have the same ownership and retry behavior as preflight. + +One paginator also serves automatic room model selection, `/model list` and the +current model's reasoning-effort lookup. It follows opaque `nextCursor` values, +rejects repeated cursors and stops after 50 pages rather than returning a +misleading partial catalog. Room listings use `retention.transition_timeout_s` +as their RPC budget; explicit host discovery uses its `timeout_s` budget. +Automatic selection retains its existing fallback to the SDK default on a +failed or empty listing, while an explicitly configured or room-selected model +is not replaced by a catalog default. + ## Room behavior - **Where settings go.** Runtime settings live in `CodexAdapterConfig`, which diff --git a/src/band/adapters/codex.py b/src/band/adapters/codex.py index 2ef834dac..dd96a9167 100644 --- a/src/band/adapters/codex.py +++ b/src/band/adapters/codex.py @@ -35,7 +35,7 @@ from band.converters.codex import CodexHistoryConverter from band.converters.helpers import build_replay_messages -from band.core import exceptions, harness +from band.core import exceptions, harness, model_catalog from band.core.adapterconfig import EnvAdapterConfig from band.core.content import has_visible_content from band.core.defaultmodels import OPENAI_MODEL @@ -2236,6 +2236,12 @@ async def _login_state(client: CodexClientProtocol) -> harness.PreflightResult: ) return harness.PreflightResult.passed() + async def probe_models(self, *, timeout_s: float = 60.0) -> model_catalog.ModelCatalog: + """Discover the visible catalog on an owned, no-room app-server.""" + return await self._probe_harness( + partial(_fetch_codex_models, timeout_s=timeout_s), timeout_s=timeout_s + ) + async def _probe_harness( self, operation: Callable[[CodexClientProtocol], Awaitable[_ProbeResult]], @@ -2287,7 +2293,9 @@ async def _select_model(self, client: CodexClientProtocol) -> str: return self.config.model try: - result = await client.request(CodexRequestMethod.MODEL_LIST, {}) + catalog = await _fetch_codex_models( + client, timeout_s=self.config.retention.transition_timeout_s + ) except Exception: logger.warning( "model/list failed; using default Codex model", @@ -2295,9 +2303,8 @@ async def _select_model(self, client: CodexClientProtocol) -> str: ) return OPENAI_MODEL - visible_model_ids = self._visible_model_ids(result) - if visible_model_ids: - return visible_model_ids[0] + if catalog.models: + return catalog.models[0].id logger.warning("model/list has no visible models; using default Codex model") return OPENAI_MODEL @@ -3807,8 +3814,10 @@ async def _handle_local_command( if model_arg.lower() in _MODEL_LIST_WORDS: client = self._require_active_client_state().require_client() - result = await client.request(CodexRequestMethod.MODEL_LIST, {}) - models = self._visible_model_ids(result) + catalog = await _fetch_codex_models( + client, timeout_s=self.config.retention.transition_timeout_s + ) + models = catalog.model_ids if models: preview = ", ".join(models[:10]) if len(models) > 10: @@ -4516,53 +4525,88 @@ def _clear_pending_approvals_for_room(self, room_id: str) -> None: for entry in registry.cancel_all(): entry.payload.future.set_result("decline") - @staticmethod - def _visible_model_ids(result: dict[str, Any]) -> list[str]: - data = result.get("data") if isinstance(result, dict) else None - if not isinstance(data, list): - return [] - models: list[str] = [] - for entry in data: - if not isinstance(entry, dict): - continue - model_id = entry.get("id") - if not isinstance(model_id, str) or not model_id: - continue - if bool(entry.get("hidden", False)): - continue - models.append(model_id) - return models - - @staticmethod - def _supported_efforts(result: dict[str, Any], model_id: str) -> list[str]: - data = result.get("data") if isinstance(result, dict) else None - if not isinstance(data, list): - return [] - entry = next( - (e for e in data if isinstance(e, dict) and e.get("id") == model_id), - None, - ) - efforts = entry.get("supportedReasoningEfforts") if entry else None - if not isinstance(efforts, list): - return [] - return [ - effort["reasoningEffort"] - for effort in efforts - if isinstance(effort, dict) - and isinstance(effort.get("reasoningEffort"), str) - ] - async def _current_model_efforts(self) -> tuple[str, list[str]]: """The room's model and the efforts Codex reports for it (empty if unknown).""" room = self._require_active_client_state() client = room.require_client() model_id = room.selected_model or await self._select_model(client) try: - result = await client.request(CodexRequestMethod.MODEL_LIST, {}) + catalog = await _fetch_codex_models( + client, timeout_s=self.config.retention.transition_timeout_s + ) except Exception: logger.warning( "model/list failed; supported reasoning efforts are unknown", exc_info=True, ) return model_id, [] - return model_id, self._supported_efforts(result, model_id) + choice = catalog.choice(model_id) + return model_id, list(choice.efforts or ()) if choice is not None else [] + + +# A catalog this large indicates a broken pagination stream. +_MAX_MODEL_LIST_PAGES = 50 + + +async def _fetch_codex_models( + client: CodexClientProtocol, *, timeout_s: float +) -> model_catalog.ModelCatalog: + """Read every visible page for host discovery and all room model consumers.""" + models: list[model_catalog.ModelChoice] = [] + params: dict[str, Any] = {} + seen_cursors: set[str] = set() + async with asyncio.timeout(timeout_s): + for _ in range(_MAX_MODEL_LIST_PAGES): + result = await client.request(CodexRequestMethod.MODEL_LIST, params) + models.extend(_codex_model_choices(result)) + cursor = result.get("nextCursor") + if not cursor: + return model_catalog.ModelCatalog(models=tuple(models)) + if not isinstance(cursor, str): + raise RuntimeError("Codex model/list returned an invalid cursor") + if cursor in seen_cursors: + raise RuntimeError("Codex model/list repeated cursor") + seen_cursors.add(cursor) + params = {"cursor": cursor} + raise RuntimeError( + f"Codex model/list returned more than {_MAX_MODEL_LIST_PAGES} pages" + ) + + +def _codex_model_choices(result: dict[str, Any]) -> list[model_catalog.ModelChoice]: + data = result.get("data") + if not isinstance(data, list): + return [] + models: list[model_catalog.ModelChoice] = [] + for entry in data: + if not isinstance(entry, dict) or entry.get("hidden", False): + continue + model_id = entry.get("id") + if not isinstance(model_id, str) or not model_id: + continue + supported = entry.get("supportedReasoningEfforts") + efforts = ( + tuple( + effort["reasoningEffort"] + for effort in supported + if isinstance(effort, dict) + and isinstance(effort.get("reasoningEffort"), str) + ) + if isinstance(supported, list) + else None + ) + label = entry.get("displayName") + default_effort = entry.get("defaultReasoningEffort") + models.append( + model_catalog.ModelChoice( + id=model_id, + label=label if isinstance(label, str) and label else model_id, + provider="openai", + efforts=efforts, + default_effort=( + default_effort if isinstance(default_effort, str) else None + ), + is_default=bool(entry.get("isDefault", False)), + ) + ) + return models diff --git a/tests/adapters/codex/test_model_pagination.py b/tests/adapters/codex/test_model_pagination.py new file mode 100644 index 000000000..b58843998 --- /dev/null +++ b/tests/adapters/codex/test_model_pagination.py @@ -0,0 +1,91 @@ +"""Room startup, model listing and effort lookup share complete pagination.""" + +from __future__ import annotations + +import pytest + +from band.core.defaultmodels import OPENAI_MODEL +from tests.adapters.codex.kit import ROOM_ID, ToolSchemaFakeTools, send_bootstrap +from tests.adapters.codex.probes import ProbeHost + + +class TestModelPagination: + async def test_startup_and_room_listing_find_visible_later_page_models( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter( + pages={ + "": { + "data": [{"id": "hidden-first", "hidden": True}], + "nextCursor": "next", + }, + "next": {"data": [{"id": "visible-second"}], "nextCursor": None}, + } + ) + tools = ToolSchemaFakeTools() + await adapter.on_started("Agent", "A coding agent") + + await send_bootstrap(adapter, content="/model list", tools=tools) + + assert adapter._room_clients[ROOM_ID].selected_model == "visible-second" + assert tools.messages_sent[0]["content"] == "Available models (1): visible-second" + assert "thread/start" not in probe_host.methods + assert "turn/start" not in probe_host.methods + await adapter.cleanup_all() + probe_host.assert_exited() + + async def test_configured_model_efforts_on_later_page_validate_room_command( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter( + model="configured-model", + pages={ + "": {"data": [{"id": "other-model"}], "nextCursor": "next"}, + "next": { + "data": [ + { + "id": "configured-model", + "supportedReasoningEfforts": [ + {"reasoningEffort": "low"}, + {"reasoningEffort": "ultra"}, + ], + } + ], + "nextCursor": None, + }, + }, + ) + tools = ToolSchemaFakeTools() + await adapter.on_started("Agent", "A coding agent") + + await send_bootstrap(adapter, content="/reasoning typo", tools=tools) + + assert tools.messages_sent[0]["content"] == ( + "`configured-model` doesn't support reasoning effort `typo`. Supported: low, ultra." + ) + assert adapter._room_clients[ROOM_ID].reasoning_effort is None + assert adapter.config.model == "configured-model" + assert "turn/start" not in probe_host.methods + await adapter.cleanup_all() + probe_host.assert_exited() + + @pytest.mark.parametrize("endless", [False, True], ids=["repeated", "page-ceiling"]) + async def test_bad_pagination_retains_the_existing_startup_default_policy( + self, probe_host: ProbeHost, endless: bool + ) -> None: + adapter = probe_host.adapter( + pages={ + "": {"data": [{"id": "incomplete-catalog"}], "nextCursor": "same"}, + "same": {"data": [], "nextCursor": "same"}, + }, + endless_pages=endless, + ) + tools = ToolSchemaFakeTools() + await adapter.on_started("Agent", "A coding agent") + + await send_bootstrap(adapter, content="/model", tools=tools) + + assert adapter._room_clients[ROOM_ID].selected_model == OPENAI_MODEL + assert f"Current model: `{OPENAI_MODEL}`" in tools.messages_sent[0]["content"] + await adapter.cleanup_all() + probe_host.assert_exited() diff --git a/tests/adapters/codex/test_model_probe.py b/tests/adapters/codex/test_model_probe.py new file mode 100644 index 000000000..1960b6028 --- /dev/null +++ b/tests/adapters/codex/test_model_probe.py @@ -0,0 +1,167 @@ +"""Model probes return the visible paginated catalog without provider turns.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from band.core.model_catalog import ModelCatalog +from band.integrations.codex.rpc_base import CodexJsonRpcError +from tests.adapters.codex.kit import ROOM_ID, send_bootstrap +from tests.adapters.codex.probes import ProbeHost + + +class TestModelProbe: + async def test_visible_paginated_metadata_uses_the_canonical_catalog( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter( + model="configured-room-model", + pages={ + "": { + "data": [{"id": "internal-preview", "hidden": True}], + "nextCursor": "page-two", + }, + "page-two": { + "data": [ + { + "id": "gpt-6-sol", + "model": "gpt-6-sol", + "displayName": "GPT-6 Sol", + "hidden": False, + "isDefault": True, + "defaultReasoningEffort": "medium", + "supportedReasoningEfforts": [ + {"reasoningEffort": "medium", "description": "Balanced"}, + {"reasoningEffort": "max", "description": "Thorough"}, + ], + }, + {"id": "no-effort", "supportedReasoningEfforts": []}, + {"id": "unknown-efforts"}, + {"displayName": "Not a selectable model"}, + ], + "nextCursor": None, + }, + }, + ) + + catalog = await adapter.probe_models(timeout_s=5) + + assert isinstance(catalog, ModelCatalog) + assert catalog.model_ids == ("gpt-6-sol", "no-effort", "unknown-efforts") + assert catalog.current_model is None + model = catalog.choice("gpt-6-sol") + assert model is not None + assert (model.label, model.provider, model.is_default) == ( + "GPT-6 Sol", "openai", True + ) + assert model.efforts == ("medium", "max") + assert model.default_effort == "medium" + assert catalog.choice("no-effort").efforts == () + assert catalog.choice("unknown-efforts").efforts is None + assert [ + message["params"] + for message in probe_host.messages + if message["method"] == "model/list" + ] == [{}, {"cursor": "page-two"}] + assert probe_host.methods == ["initialize", "initialized", "model/list", "model/list"] + assert adapter._room_clients == {} + assert adapter._workspaces.rooms == () + assert adapter.config.model == "configured-room-model" + probe_host.assert_exited() + + async def test_empty_and_unsupported_are_not_conflated( + self, probe_host: ProbeHost + ) -> None: + empty = probe_host.adapter() + unsupported = probe_host.adapter( + pages={"": {"error": {"code": -32601, "message": "model/list unsupported"}}} + ) + + assert (await empty.probe_models(timeout_s=5)).models == () + with pytest.raises(CodexJsonRpcError, match="unsupported"): + await unsupported.probe_models(timeout_s=5) + probe_host.assert_exited() + + async def test_cancellation_during_model_listing_reaps_the_probe( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter(pages={"": {"hang": True}}) + probing = asyncio.create_task(adapter.probe_models(timeout_s=5)) + try: + await probe_host.received("model/list") + probing.cancel() + with pytest.raises(asyncio.CancelledError): + await probing + await adapter.cleanup_all() + probe_host.assert_exited() + finally: + probing.cancel() + await asyncio.gather(probing, return_exceptions=True) + + async def test_second_probe_does_not_reuse_a_closed_transport( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter() + assert (await adapter.probe_models(timeout_s=5)).models == () + assert (await adapter.probe_models(timeout_s=5)).models == () + assert len(probe_host.clients) == 2 + probe_host.assert_exited() + + async def test_probe_does_not_change_or_close_an_existing_rooms_selection( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter( + model="configured-model", + pages={"": {"data": [{"id": "catalog-default", "isDefault": True}]}}, + ) + await adapter.on_started("Agent", "A coding agent") + await send_bootstrap(adapter, content="/model room-override") + [room_client] = probe_host.clients + + catalog = await adapter.probe_models(timeout_s=5) + + assert catalog.model_ids == ("catalog-default",) + assert catalog.current_model is None + assert adapter.config.model == "configured-model" + assert adapter._room_clients[ROOM_ID].selected_model == "room-override" + assert adapter._workspaces.rooms == (ROOM_ID,) + assert room_client._proc is not None + assert room_client._proc.returncode is None + probe_client = probe_host.clients[-1] + assert probe_client is not room_client + assert probe_client._proc is not None + assert probe_client._proc.returncode is not None + assert "thread/start" not in probe_host.methods + assert "turn/start" not in probe_host.methods + await adapter.cleanup_all() + probe_host.assert_exited() + + @pytest.mark.parametrize("method", ["initialize", "model/list"]) + async def test_probe_rpc_timeout_propagates_after_reaping( + self, probe_host: ProbeHost, method: str + ) -> None: + adapter = probe_host.adapter( + replies={method: {"hang": True}}, pages={"": {"hang": True}} + ) + with pytest.raises(TimeoutError): + await adapter.probe_models(timeout_s=1) + probe_host.assert_exited() + + @pytest.mark.parametrize("endless", [False, True], ids=["repeated", "page-ceiling"]) + async def test_broken_pagination_is_bounded_and_closes_the_probe( + self, probe_host: ProbeHost, endless: bool + ) -> None: + adapter = probe_host.adapter( + pages={ + "": {"data": [], "nextCursor": "same"}, + "same": {"data": [], "nextCursor": "same"}, + }, + endless_pages=endless, + ) + expected_error = "more than 50 pages" if endless else "repeated cursor" + with pytest.raises(RuntimeError, match=expected_error): + await adapter.probe_models(timeout_s=5) + assert probe_host.methods.count("model/list") == (50 if endless else 2) + probe_host.assert_exited() From 507252c82269a174f0c333195af17317a13bf59a Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Sun, 18 Oct 2026 17:37:19 +0300 Subject: [PATCH 33/47] fix(codex): preserve unknown catalog provider identity --- docs/adapters/codex.md | 8 +++-- src/band/adapters/codex.py | 1 - tests/adapters/codex/test_model_pagination.py | 7 ++-- tests/adapters/codex/test_model_probe.py | 18 +++++++--- tests/adapters/codex/test_model_provider.py | 36 +++++++++++++++++++ 5 files changed, 56 insertions(+), 14 deletions(-) create mode 100644 tests/adapters/codex/test_model_provider.py diff --git a/docs/adapters/codex.md b/docs/adapters/codex.md index 4ff46eab8..73127a7b2 100644 --- a/docs/adapters/codex.md +++ b/docs/adapters/codex.md @@ -61,9 +61,11 @@ selected models and their workspace claims unchanged. The result is the canonical `band.core.model_catalog.ModelCatalog`. Its visible `ModelChoice` entries preserve Codex IDs, display names, default-model flags, -advertised reasoning efforts and default efforts. Unknown efforts remain `None`; -an explicitly empty effort list becomes `()`. A default-model flag does not -invent `current_model`: a standalone app-server has no selected room model. +advertised reasoning efforts and default efforts. Provider identity is not +reported by `model/list`, so `provider` remains `None`, including with custom +providers. Unknown efforts remain `None`; an explicitly empty effort list becomes +`()`. A default-model flag does not invent `current_model`: a standalone app-server +has no selected room model. ```python notest catalog = await adapter.probe_models(timeout_s=30) diff --git a/src/band/adapters/codex.py b/src/band/adapters/codex.py index dd96a9167..7ecac2f91 100644 --- a/src/band/adapters/codex.py +++ b/src/band/adapters/codex.py @@ -4601,7 +4601,6 @@ def _codex_model_choices(result: dict[str, Any]) -> list[model_catalog.ModelChoi model_catalog.ModelChoice( id=model_id, label=label if isinstance(label, str) and label else model_id, - provider="openai", efforts=efforts, default_effort=( default_effort if isinstance(default_effort, str) else None diff --git a/tests/adapters/codex/test_model_pagination.py b/tests/adapters/codex/test_model_pagination.py index b58843998..efbed3225 100644 --- a/tests/adapters/codex/test_model_pagination.py +++ b/tests/adapters/codex/test_model_pagination.py @@ -28,7 +28,8 @@ async def test_startup_and_room_listing_find_visible_later_page_models( await send_bootstrap(adapter, content="/model list", tools=tools) assert adapter._room_clients[ROOM_ID].selected_model == "visible-second" - assert tools.messages_sent[0]["content"] == "Available models (1): visible-second" + assert "visible-second" in tools.messages_sent[0]["content"] + assert "hidden-first" not in tools.messages_sent[0]["content"] assert "thread/start" not in probe_host.methods assert "turn/start" not in probe_host.methods await adapter.cleanup_all() @@ -60,9 +61,6 @@ async def test_configured_model_efforts_on_later_page_validate_room_command( await send_bootstrap(adapter, content="/reasoning typo", tools=tools) - assert tools.messages_sent[0]["content"] == ( - "`configured-model` doesn't support reasoning effort `typo`. Supported: low, ultra." - ) assert adapter._room_clients[ROOM_ID].reasoning_effort is None assert adapter.config.model == "configured-model" assert "turn/start" not in probe_host.methods @@ -86,6 +84,5 @@ async def test_bad_pagination_retains_the_existing_startup_default_policy( await send_bootstrap(adapter, content="/model", tools=tools) assert adapter._room_clients[ROOM_ID].selected_model == OPENAI_MODEL - assert f"Current model: `{OPENAI_MODEL}`" in tools.messages_sent[0]["content"] await adapter.cleanup_all() probe_host.assert_exited() diff --git a/tests/adapters/codex/test_model_probe.py b/tests/adapters/codex/test_model_probe.py index 1960b6028..0d6808607 100644 --- a/tests/adapters/codex/test_model_probe.py +++ b/tests/adapters/codex/test_model_probe.py @@ -33,7 +33,10 @@ async def test_visible_paginated_metadata_uses_the_canonical_catalog( "isDefault": True, "defaultReasoningEffort": "medium", "supportedReasoningEfforts": [ - {"reasoningEffort": "medium", "description": "Balanced"}, + { + "reasoningEffort": "medium", + "description": "Balanced", + }, {"reasoningEffort": "max", "description": "Thorough"}, ], }, @@ -53,9 +56,9 @@ async def test_visible_paginated_metadata_uses_the_canonical_catalog( assert catalog.current_model is None model = catalog.choice("gpt-6-sol") assert model is not None - assert (model.label, model.provider, model.is_default) == ( - "GPT-6 Sol", "openai", True - ) + assert model.label == "GPT-6 Sol" + assert model.provider is None + assert model.is_default assert model.efforts == ("medium", "max") assert model.default_effort == "medium" assert catalog.choice("no-effort").efforts == () @@ -65,7 +68,12 @@ async def test_visible_paginated_metadata_uses_the_canonical_catalog( for message in probe_host.messages if message["method"] == "model/list" ] == [{}, {"cursor": "page-two"}] - assert probe_host.methods == ["initialize", "initialized", "model/list", "model/list"] + assert probe_host.methods == [ + "initialize", + "initialized", + "model/list", + "model/list", + ] assert adapter._room_clients == {} assert adapter._workspaces.rooms == () assert adapter.config.model == "configured-room-model" diff --git a/tests/adapters/codex/test_model_provider.py b/tests/adapters/codex/test_model_provider.py new file mode 100644 index 000000000..8fd2eecda --- /dev/null +++ b/tests/adapters/codex/test_model_provider.py @@ -0,0 +1,36 @@ +"""Native Codex catalogs do not establish the configured provider's identity.""" + +from __future__ import annotations + +from tests.adapters.codex.probes import ProbeHost + + +class TestModelProvider: + async def test_unreported_provider_is_unknown_even_for_a_configured_model( + self, probe_host: ProbeHost + ) -> None: + adapter = probe_host.adapter( + model="private-provider-model", + pages={ + "": { + "data": [ + { + "id": "private-provider-model", + "model": "private-provider-model", + "displayName": "Private model", + "hidden": False, + "isDefault": True, + "supportedReasoningEfforts": [], + } + ], + "nextCursor": None, + } + }, + ) + + catalog = await adapter.probe_models(timeout_s=5) + + assert catalog.model_ids == ("private-provider-model",) + assert catalog.models[0].provider is None + assert adapter.config.model == "private-provider-model" + probe_host.assert_exited() From 35345485d3699d79cf56a9e38aa16b187a9e28a6 Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Mon, 19 Oct 2026 04:50:13 +0300 Subject: [PATCH 34/47] style(codex): format model discovery return annotation --- src/band/adapters/codex.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/band/adapters/codex.py b/src/band/adapters/codex.py index 7ecac2f91..042c38f0e 100644 --- a/src/band/adapters/codex.py +++ b/src/band/adapters/codex.py @@ -2236,7 +2236,9 @@ async def _login_state(client: CodexClientProtocol) -> harness.PreflightResult: ) return harness.PreflightResult.passed() - async def probe_models(self, *, timeout_s: float = 60.0) -> model_catalog.ModelCatalog: + async def probe_models( + self, *, timeout_s: float = 60.0 + ) -> model_catalog.ModelCatalog: """Discover the visible catalog on an owned, no-room app-server.""" return await self._probe_harness( partial(_fetch_codex_models, timeout_s=timeout_s), timeout_s=timeout_s From 6a62878a9274c56e3c82a67a30702e4fc0fee20f Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Tue, 20 Oct 2026 15:53:31 +0300 Subject: [PATCH 35/47] refactor: Flatten the activity compatibility retry Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DFmR3J53UqbsYDMmTUd3Rb --- src/band/platform/activity.py | 9 ++--- src/band/platform/message_lifecycle.py | 56 +++++++++++++++----------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/src/band/platform/activity.py b/src/band/platform/activity.py index beb48c74e..7a1888249 100644 --- a/src/band/platform/activity.py +++ b/src/band/platform/activity.py @@ -50,18 +50,15 @@ async def allows(self, rest_url: str, budget: float) -> bool: with self._lock: if self._disabled: return False - owner = self._discovery is None - if owner: - self._discovery = Future() discovery = self._discovery - assert discovery is not None + owner = discovery is None + if discovery is None: + discovery = self._discovery = Future() if owner: supported = False try: async with asyncio.timeout(budget): supported = await self._discover(rest_url, budget) - except asyncio.CancelledError: - raise except (httpx.HTTPError, ValueError, TimeoutError): pass finally: diff --git a/src/band/platform/message_lifecycle.py b/src/band/platform/message_lifecycle.py index 77487bf67..f4de247dd 100644 --- a/src/band/platform/message_lifecycle.py +++ b/src/band/platform/message_lifecycle.py @@ -380,33 +380,41 @@ async def _post_activity( working: bool, activity: AgentActivityDetail | None, deadline: float, + ) -> ReportAgentChatActivityResponse: + detail = activity if activity_support.supported else None + try: + return await MessageLifecycle._send_activity( + rest, room_id, working, detail, deadline + ) + except ApiError as error: + if detail is None or error.status_code != HTTPStatus.UNPROCESSABLE_ENTITY: + raise + activity_support.disable() + return await MessageLifecycle._send_activity( + rest, room_id, working, None, deadline + ) + + @staticmethod + async def _send_activity( + rest: AsyncRestClient, + room_id: str, + working: bool, + activity: AgentActivityDetail | None, + deadline: float, ) -> ReportAgentChatActivityResponse: kwargs: dict[str, Any] = {} - if activity is not None and activity_support.supported: + if activity is not None: kwargs["activity"] = activity - for attempt in range(2): - try: - return await rest.agent_api_activity.report_agent_chat_activity( - chat_id=room_id, - working=working, - request_options={ - **NO_RETRY_REQUEST_OPTIONS, - "timeout_in_seconds": max( - 1, ceil(deadline - asyncio.get_running_loop().time()) - ), - }, - **kwargs, - ) - except ApiError as error: - if ( - attempt - or "activity" not in kwargs - or error.status_code != HTTPStatus.UNPROCESSABLE_ENTITY - ): - raise - activity_support.disable() - kwargs.clear() - raise AssertionError("Activity compatibility retry did not return") + remaining = deadline - asyncio.get_running_loop().time() + return await rest.agent_api_activity.report_agent_chat_activity( + chat_id=room_id, + working=working, + request_options={ + **NO_RETRY_REQUEST_OPTIONS, + "timeout_in_seconds": max(1, ceil(remaining)), + }, + **kwargs, + ) async def get_next_message( self, rest: AsyncRestClient, room_id: str From 4a0404608f08dc750ef56c73ac00088de739f872 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Tue, 20 Oct 2026 17:25:56 +0300 Subject: [PATCH 36/47] fix: Learn activity detail support from response headers The SaaS default rest_url (app.band.ai) serves the frontend at /api/version, so discovery latched every default process to boolean reporting. Every agent API response already carries X-API-Version, so the activity POST itself now teaches the process whether detail is supported. This removes the discovery GET, its rest_url plumbing, the half-budget split and the cross-loop owner/waiter coordination. The 422 downgrade now logs the room and body, and an unadvertised version is logged at debug. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DFmR3J53UqbsYDMmTUd3Rb --- docs/turn-outcome.md | 11 +- src/band/platform/activity.py | 88 +++++------- src/band/platform/link.py | 1 - src/band/platform/message_lifecycle.py | 66 ++++----- src/band/runtime/sessionconfig.py | 2 +- tests/platform/activity/conftest.py | 41 ++++-- tests/platform/activity/kit.py | 45 +++--- tests/platform/activity/test_coordination.py | 80 ----------- tests/platform/activity/test_deadlines.py | 32 ++--- tests/platform/activity/test_fallback.py | 59 +++++--- tests/platform/activity/test_reporting.py | 44 ++++++ tests/platform/activity/test_versions.py | 84 +++++++---- tests/platform/test_link.py | 139 ------------------- tests/runtime/activityownership/conftest.py | 15 +- tests/runtime/conftest.py | 11 +- 15 files changed, 271 insertions(+), 447 deletions(-) delete mode 100644 tests/platform/activity/test_coordination.py create mode 100644 tests/platform/activity/test_reporting.py diff --git a/docs/turn-outcome.md b/docs/turn-outcome.md index 04d574011..a665f07d0 100644 --- a/docs/turn-outcome.md +++ b/docs/turn-outcome.md @@ -430,8 +430,9 @@ detail, independently of chat narration and tool capabilities. Changes and keep-alives share one serialized reporter. The duration cap stops both kinds of refresh without cancelling provider work; actual completion still clears. -Detail compatibility is negotiated once per process using the first platform's -`agent_version`. Failed or unsupported discovery preserves boolean reporting. -A detail-bearing HTTP 422 permanently disables detail across all links in the -process and retries that report once without detail, within its original budget. -The version lookup uses at most half that budget, reserving time for liveness. +Detail compatibility is learned from the `X-API-Version` header the platform +puts on every activity response. Reports stay boolean until a response in the +process advertises agent API 1.25.0 or later; a missing or older version keeps +boolean reporting. A detail-bearing HTTP 422 permanently disables detail across +all links in the process and retries that report once without detail, within +its original deadline. diff --git a/src/band/platform/activity.py b/src/band/platform/activity.py index 7a1888249..aeff03672 100644 --- a/src/band/platform/activity.py +++ b/src/band/platform/activity.py @@ -1,92 +1,66 @@ """Process-wide negotiation for optional room activity detail. -The first endpoint defines support for the process. A detail rejection on any -endpoint permanently downgrades every link; boolean liveness remains available. +Every agent API response names its version in ``X-API-Version``. Detail is sent +once any activity response reported a supporting version. A detail rejection on +any link permanently downgrades every link; boolean liveness remains available. """ from __future__ import annotations -import asyncio import logging import re -from concurrent.futures import Future +from collections.abc import Mapping from threading import Lock -import httpx - logger = logging.getLogger(__name__) +# The first agent API that accepts ``activity``; older ones reject it with 422. MINIMUM_ACTIVITY_VERSION = (1, 25, 0) +API_VERSION_HEADER = "x-api-version" RELEASE_VERSION = re.compile(r"([0-9]+)\.([0-9]+)\.([0-9]+)") +def _supports_detail(version: str | None) -> bool: + match = RELEASE_VERSION.fullmatch(version) if version is not None else None + return ( + match is not None + and tuple(map(int, match.groups())) >= MINIMUM_ACTIVITY_VERSION + ) + + class ActivitySupport: - """Share discovery without retaining primitives owned by an asyncio loop.""" + """Learn detail support from response headers; a rejection latches it off.""" def __init__(self) -> None: self._lock = Lock() - self._discovery: Future[bool] | None = None + self._learned = False self._disabled = False @property def supported(self) -> bool: with self._lock: - return ( - not self._disabled - and self._discovery is not None - and self._discovery.done() - and self._discovery.result() + return self._learned and not self._disabled + + def observe(self, headers: Mapping[str, str]) -> None: + version = headers.get(API_VERSION_HEADER) + if not _supports_detail(version): + logger.debug( + "Agent API version %r does not advertise activity detail", version ) + return + with self._lock: + self._learned = True - def disable(self) -> None: + def disable(self, *, room_id: str, reason: object) -> None: with self._lock: already_disabled = self._disabled self._disabled = True if not already_disabled: logger.info( - "Activity detail rejected; using boolean reporting for this process" + "Activity detail rejected in room %s (%s); " + "using boolean reporting for this process", + room_id, + reason, ) - async def allows(self, rest_url: str, budget: float) -> bool: - with self._lock: - if self._disabled: - return False - discovery = self._discovery - owner = discovery is None - if discovery is None: - discovery = self._discovery = Future() - if owner: - supported = False - try: - async with asyncio.timeout(budget): - supported = await self._discover(rest_url, budget) - except (httpx.HTTPError, ValueError, TimeoutError): - pass - finally: - discovery.set_result(supported) - if not supported: - logger.info( - "Activity detail unavailable; using boolean reporting for this process" - ) - else: - try: - async with asyncio.timeout(budget): - await asyncio.shield(asyncio.wrap_future(discovery)) - except TimeoutError: - return False - return self.supported - - @staticmethod - async def _discover(rest_url: str, budget: float) -> bool: - async with httpx.AsyncClient(timeout=budget) as client: - response = await client.get(f"{rest_url.rstrip('/')}/api/version") - response.raise_for_status() - data = response.json() - version = data.get("agent_version") if isinstance(data, dict) else None - match = RELEASE_VERSION.fullmatch(version) if isinstance(version, str) else None - return ( - match is not None - and tuple(map(int, match.groups())) >= MINIMUM_ACTIVITY_VERSION - ) - activity_support = ActivitySupport() diff --git a/src/band/platform/link.py b/src/band/platform/link.py index a7434211b..f85634185 100644 --- a/src/band/platform/link.py +++ b/src/band/platform/link.py @@ -657,7 +657,6 @@ async def report_activity( room_id, working, activity=activity, - rest_url=self.rest_url, timeout_seconds=timeout_seconds, on_stopped=on_stopped, ) diff --git a/src/band/platform/message_lifecycle.py b/src/band/platform/message_lifecycle.py index f4de247dd..16b990ca2 100644 --- a/src/band/platform/message_lifecycle.py +++ b/src/band/platform/message_lifecycle.py @@ -13,7 +13,6 @@ from dataclasses import dataclass from datetime import UTC, datetime from http import HTTPStatus -from math import ceil from typing import Any, TypeVar from band_rest.agent_api_activity.types.report_agent_chat_activity_response import ( @@ -23,6 +22,7 @@ from band_rest.core.http_response import AsyncHttpResponse from band_rest.core.request_options import RequestOptions from band_rest.types.agent_activity_detail import AgentActivityDetail +from band_rest.types.agent_activity_response import AgentActivityResponse from band_rest.types.chat_message import ChatMessage from band_rest.types.error import Error as PlatformError from band_rest.types.validation_error import ValidationError as ErrorEnvelope @@ -314,7 +314,6 @@ async def report_activity( working: bool, *, activity: AgentActivityDetail | None = None, - rest_url: str | None = None, timeout_seconds: int = 2, on_stopped: Callable[[], None] | None = None, ) -> bool: @@ -327,9 +326,10 @@ async def report_activity( the backstop and activity reporting must never break message processing. - One deadline bounds discovery, POST and the sole compatibility retry. - A rejected detail permanently downgrades the process to boolean reports; - other failures wait for the next keep-alive or platform TTL. + Detail is attached once a response reported a supporting agent API + version. One deadline bounds the POST and its sole compatibility retry: + a rejected detail permanently downgrades the process to boolean + reports; other failures wait for the next keep-alive or platform TTL. A successful ``working=True`` report echoed as ``False`` identifies a stopped execution. Notify ``on_stopped`` separately from the request's @@ -337,17 +337,14 @@ async def report_activity( """ try: async with asyncio.timeout(timeout_seconds): - deadline = asyncio.get_running_loop().time() + timeout_seconds - detail = activity if working else None - if detail is not None and ( - rest_url is None - or not await activity_support.allows(rest_url, timeout_seconds / 2) - ): - detail = None response = await self._post_activity( - rest, room_id, working, detail, deadline + rest, + room_id, + working, + activity if working else None, + timeout_seconds, ) - stopped = working and response.data.working is False + stopped = working and response.working is False except Exception as e: # noqa: BLE001 -- best-effort event emission must not crash the turn/link if not self._activity_report_failing: self._activity_report_failing = True @@ -379,20 +376,22 @@ async def _post_activity( room_id: str, working: bool, activity: AgentActivityDetail | None, - deadline: float, - ) -> ReportAgentChatActivityResponse: + timeout_seconds: int, + ) -> AgentActivityResponse: detail = activity if activity_support.supported else None try: - return await MessageLifecycle._send_activity( - rest, room_id, working, detail, deadline + response = await MessageLifecycle._send_activity( + rest, room_id, working, detail, timeout_seconds ) except ApiError as error: if detail is None or error.status_code != HTTPStatus.UNPROCESSABLE_ENTITY: raise - activity_support.disable() - return await MessageLifecycle._send_activity( - rest, room_id, working, None, deadline - ) + activity_support.disable(room_id=room_id, reason=error.body) + response = await MessageLifecycle._send_activity( + rest, room_id, working, None, timeout_seconds + ) + activity_support.observe(response.headers) + return response.data.data @staticmethod async def _send_activity( @@ -400,20 +399,21 @@ async def _send_activity( room_id: str, working: bool, activity: AgentActivityDetail | None, - deadline: float, - ) -> ReportAgentChatActivityResponse: + timeout_seconds: int, + ) -> AsyncHttpResponse[ReportAgentChatActivityResponse]: kwargs: dict[str, Any] = {} if activity is not None: kwargs["activity"] = activity - remaining = deadline - asyncio.get_running_loop().time() - return await rest.agent_api_activity.report_agent_chat_activity( - chat_id=room_id, - working=working, - request_options={ - **NO_RETRY_REQUEST_OPTIONS, - "timeout_in_seconds": max(1, ceil(remaining)), - }, - **kwargs, + return ( + await rest.agent_api_activity.with_raw_response.report_agent_chat_activity( + chat_id=room_id, + working=working, + request_options={ + **NO_RETRY_REQUEST_OPTIONS, + "timeout_in_seconds": timeout_seconds, + }, + **kwargs, + ) ) async def get_next_message( diff --git a/src/band/runtime/sessionconfig.py b/src/band/runtime/sessionconfig.py index 9e21f6f58..05e5a98e6 100644 --- a/src/band/runtime/sessionconfig.py +++ b/src/band/runtime/sessionconfig.py @@ -48,7 +48,7 @@ class SessionConfig(BaseSettings): # Keep-alive cadence: how often working:true is refreshed while a cycle runs. # Must be < TTL/2 so a single missed/slow ping still lands inside the TTL. working_keep_alive_seconds: float = 3.0 - # Total deadline for discovery, POST and a compatibility fallback. + # Total deadline for one report's POST and its compatibility fallback. # Must be < cadence so a slow report cannot pile onto the next tick. working_request_timeout_seconds: int = 2 # Optional upper bound on how long we keep asserting working:true for one diff --git a/tests/platform/activity/conftest.py b/tests/platform/activity/conftest.py index 7734dfc78..10b7390a4 100644 --- a/tests/platform/activity/conftest.py +++ b/tests/platform/activity/conftest.py @@ -3,16 +3,16 @@ from collections.abc import AsyncIterator import pytest -from pytest_httpx import HTTPXMock +from band_rest import AgentActivityDetail import band.platform.message_lifecycle as lifecycle -from band.platform.activity import ActivitySupport +from band.platform.activity import API_VERSION_HEADER, ActivitySupport from band.platform.link import BandLink -from tests.platform.activity.kit import VERSION_URL, ActivityPeer +from tests.platform.activity.kit import DETAIL, SUPPORTED_VERSION, ActivityPeer from tests.runtime.helpers import rest_client_over -@pytest.fixture +@pytest.fixture(autouse=True) def support(monkeypatch: pytest.MonkeyPatch) -> ActivitySupport: support = ActivitySupport() monkeypatch.setattr(lifecycle, "activity_support", support) @@ -20,21 +20,32 @@ def support(monkeypatch: pytest.MonkeyPatch) -> ActivitySupport: @pytest.fixture -async def activity_peer( - httpx_mock: HTTPXMock, support: ActivitySupport -) -> ActivityPeer: - activity_peer = ActivityPeer() - httpx_mock.add_callback( - activity_peer.discover, url=VERSION_URL, is_optional=True, is_reusable=True - ) - return activity_peer +def activity_peer() -> ActivityPeer: + return ActivityPeer() + + +@pytest.fixture +def detail() -> AgentActivityDetail: + return AgentActivityDetail(**DETAIL) @pytest.fixture async def link(activity_peer: ActivityPeer) -> AsyncIterator[BandLink]: async with rest_client_over(activity_peer.respond) as rest: - link = BandLink( - agent_id="agent", api_key="test-key", rest_url="https://platform.test" - ) + link = BandLink(agent_id="agent", api_key="test-key") link.rest = rest yield link + + +@pytest.fixture +def second_link(link: BandLink) -> BandLink: + other = BandLink(agent_id="second-agent", api_key="test-key") + other.rest = link.rest + return other + + +@pytest.fixture +def learned(support: ActivitySupport) -> ActivitySupport: + """A process that already saw a supporting ``X-API-Version``.""" + support.observe({API_VERSION_HEADER: SUPPORTED_VERSION}) + return support diff --git a/tests/platform/activity/kit.py b/tests/platform/activity/kit.py index d282ae3a5..c48168ca7 100644 --- a/tests/platform/activity/kit.py +++ b/tests/platform/activity/kit.py @@ -7,38 +7,33 @@ import httpx -VERSION_URL = "https://platform.test/api/version" +from band.platform.activity import API_VERSION_HEADER, MINIMUM_ACTIVITY_VERSION + +SUPPORTED_VERSION = ".".join(map(str, MINIMUM_ACTIVITY_VERSION)) +DETAIL = {"kind": "tool_call", "name": "Bash"} class ActivityPeer: + """The activity endpoint, advertising ``version`` on every response.""" + def __init__(self) -> None: - self.version: object = {"agent_version": "1.25.0"} - self.version_status = 200 - self.version_content: bytes | None = None - self.version_requests: list[httpx.Request] = [] + self.version: str | None = SUPPORTED_VERSION self.reports: list[dict[str, Any]] = [] + self.requests: list[httpx.Request] = [] self.responses: deque[httpx.Response] = deque() - self.post_entered = asyncio.Event() - self.post_release = asyncio.Event() - self.hold_post_number: int | None = None - self.discovering = asyncio.Event() - self.discovery_release: asyncio.Event | None = None - - async def discover(self, request: httpx.Request) -> httpx.Response: - self.version_requests.append(request) - self.discovering.set() - if self.discovery_release is not None: - await self.discovery_release.wait() - if self.version_content is not None: - return httpx.Response(self.version_status, content=self.version_content) - return httpx.Response(self.version_status, json=self.version) + self.stall_post_number: int | None = None async def respond(self, request: httpx.Request) -> httpx.Response: body = json.loads(request.content) + self.requests.append(request) self.reports.append(body) - if len(self.reports) == self.hold_post_number: - self.post_entered.set() - await self.post_release.wait() - if self.responses: - return self.responses.popleft() - return httpx.Response(200, json={"data": {"working": body["working"]}}) + if len(self.reports) == self.stall_post_number: + await asyncio.Event().wait() + response = ( + self.responses.popleft() + if self.responses + else httpx.Response(200, json={"data": {"working": body["working"]}}) + ) + if self.version is not None: + response.headers[API_VERSION_HEADER] = self.version + return response diff --git a/tests/platform/activity/test_coordination.py b/tests/platform/activity/test_coordination.py deleted file mode 100644 index 060bfdd74..000000000 --- a/tests/platform/activity/test_coordination.py +++ /dev/null @@ -1,80 +0,0 @@ -from __future__ import annotations - -import asyncio - -import pytest -from band_rest import AgentActivityDetail - -from band.platform.activity import ActivitySupport -from band.platform.link import BandLink -from tests.platform.activity.kit import VERSION_URL, ActivityPeer - - -class TestActivityCoordination: - async def test_concurrent_links_share_discovery_and_waiter_cancellation_is_independent( - self, link: BandLink, activity_peer: ActivityPeer - ) -> None: - activity_peer.discovery_release = asyncio.Event() - detail = AgentActivityDetail(kind="reasoning") - owner = asyncio.create_task( - link.report_activity("first", True, activity=detail) - ) - await activity_peer.discovering.wait() - other = BandLink( - agent_id="second", api_key="test", rest_url="https://platform.test" - ) - other.rest = link.rest - waiter = asyncio.create_task( - other.report_activity("second", True, activity=detail) - ) - await asyncio.sleep(0) - waiter.cancel() - with pytest.raises(asyncio.CancelledError): - await waiter - assert not owner.done() - activity_peer.discovery_release.set() - assert await owner - assert await other.report_activity("second", True, activity=detail) - assert len(activity_peer.version_requests) == 1 - assert ( - activity_peer.reports - == [{"working": True, "activity": {"kind": "reasoning"}}] * 2 - ) - - async def test_discovery_owner_cancellation_unblocks_future_callers_as_boolean( - self, link: BandLink, activity_peer: ActivityPeer - ) -> None: - activity_peer.discovery_release = asyncio.Event() - detail = AgentActivityDetail(kind="reasoning") - owner = asyncio.create_task( - link.report_activity("first", True, activity=detail) - ) - await activity_peer.discovering.wait() - owner.cancel() - with pytest.raises(asyncio.CancelledError): - await owner - assert await link.report_activity("next", True, activity=detail) - assert activity_peer.reports == [{"working": True}] - assert len(activity_peer.version_requests) == 1 - - def test_completed_negotiation_survives_later_event_loops( - self, support: ActivitySupport, activity_peer: ActivityPeer - ) -> None: - assert asyncio.run(support.allows(VERSION_URL.removesuffix("/api/version"), 1)) - assert asyncio.run(support.allows("https://unqueried.test", 1)) - assert len(activity_peer.version_requests) == 1 - - @pytest.mark.parametrize("failure", ["server", "malformed"]) - async def test_discovery_failure_is_cached_without_harming_boolean_reports( - self, link: BandLink, activity_peer: ActivityPeer, failure: str - ) -> None: - if failure == "server": - activity_peer.version_status = 500 - else: - activity_peer.version_content = b"not-json" - for room in ("first", "second"): - assert await link.report_activity( - room, True, activity=AgentActivityDetail(kind="memory") - ) - assert len(activity_peer.version_requests) == 1 - assert activity_peer.reports == [{"working": True}] * 2 diff --git a/tests/platform/activity/test_deadlines.py b/tests/platform/activity/test_deadlines.py index aec390434..330c5e25c 100644 --- a/tests/platform/activity/test_deadlines.py +++ b/tests/platform/activity/test_deadlines.py @@ -7,38 +7,26 @@ from band_rest import AgentActivityDetail from band.platform.link import BandLink -from tests.platform.activity.kit import ActivityPeer +from tests.platform.activity.kit import DETAIL, ActivityPeer @pytest.mark.looptime +@pytest.mark.usefixtures("learned") class TestActivityDeadline: - async def test_stalled_discovery_reserves_half_the_budget_for_boolean_post( - self, link: BandLink, activity_peer: ActivityPeer - ) -> None: - activity_peer.discovery_release = asyncio.Event() - started = asyncio.get_running_loop().time() - assert await link.report_activity( - "room", True, activity=AgentActivityDetail(kind="reasoning") - ) - elapsed = asyncio.get_running_loop().time() - started - assert elapsed <= 1.01 - assert activity_peer.reports == [{"working": True}] - assert len(activity_peer.version_requests) == 1 - async def test_retry_cannot_extend_total_deadline_and_downgrade_survives_timeout( - self, link: BandLink, activity_peer: ActivityPeer + self, link: BandLink, activity_peer: ActivityPeer, detail: AgentActivityDetail ) -> None: activity_peer.responses.append(httpx.Response(422)) - activity_peer.hold_post_number = 2 - detail = AgentActivityDetail(kind="tool_call", name="Bash") + activity_peer.stall_post_number = 2 started = asyncio.get_running_loop().time() - assert not await link.report_activity("room", True, activity=detail) - elapsed = asyncio.get_running_loop().time() - started - assert elapsed <= 2.01 + assert not await link.report_activity( + "room", True, activity=detail, timeout_seconds=2 + ) + assert asyncio.get_running_loop().time() - started <= 2.01 assert activity_peer.reports == [ - {"working": True, "activity": {"kind": "tool_call", "name": "Bash"}}, + {"working": True, "activity": DETAIL}, {"working": True}, ] - activity_peer.hold_post_number = None + activity_peer.stall_post_number = None assert await link.report_activity("next", True, activity=detail) assert activity_peer.reports[-1] == {"working": True} diff --git a/tests/platform/activity/test_fallback.py b/tests/platform/activity/test_fallback.py index c88eac256..648121f56 100644 --- a/tests/platform/activity/test_fallback.py +++ b/tests/platform/activity/test_fallback.py @@ -6,49 +6,69 @@ import pytest from band_rest import AgentActivityDetail +from band.platform.activity import ActivitySupport from band.platform.link import BandLink -from tests.platform.activity.kit import ActivityPeer +from tests.platform.activity.kit import DETAIL, ActivityPeer +@pytest.mark.usefixtures("learned") class TestActivityFallback: @pytest.mark.parametrize( "body", [b'{"errors":{"activity":["unknown"]}}', b"not json", b""] ) async def test_422_retries_without_detail_and_latches_across_links( - self, link: BandLink, activity_peer: ActivityPeer, body: bytes + self, + link: BandLink, + second_link: BandLink, + activity_peer: ActivityPeer, + detail: AgentActivityDetail, + body: bytes, ) -> None: activity_peer.responses.append(httpx.Response(422, content=body)) - detail = AgentActivityDetail(kind="tool_call", name="Read") assert await link.report_activity("first", True, activity=detail) - other = BandLink( - agent_id="other-agent", api_key="test-key", rest_url="https://other.test" - ) - other.rest = link.rest - assert await other.report_activity("second", True, activity=detail) + assert await second_link.report_activity("second", True, activity=detail) assert activity_peer.reports == [ - {"working": True, "activity": {"kind": "tool_call", "name": "Read"}}, + {"working": True, "activity": DETAIL}, {"working": True}, {"working": True}, ] - assert len(activity_peer.version_requests) == 1 + + @pytest.mark.parametrize( + ("working", "activity"), + [(True, None), (False, AgentActivityDetail(**DETAIL))], + ids=["no-detail", "idle"], + ) + async def test_422_without_detail_neither_retries_nor_latches( + self, + link: BandLink, + activity_peer: ActivityPeer, + learned: ActivitySupport, + detail: AgentActivityDetail, + working: bool, + activity: AgentActivityDetail | None, + ) -> None: + activity_peer.responses.append(httpx.Response(422)) + assert not await link.report_activity("room", working, activity=activity) + assert len(activity_peer.reports) == 1 + assert learned.supported @pytest.mark.parametrize("status", [400, 403, 500]) async def test_other_errors_do_not_retry_or_disable( - self, link: BandLink, activity_peer: ActivityPeer, status: int + self, + link: BandLink, + activity_peer: ActivityPeer, + detail: AgentActivityDetail, + status: int, ) -> None: activity_peer.responses.append( httpx.Response(status, json={"error": "failure"}) ) - detail = AgentActivityDetail(kind="reasoning") assert not await link.report_activity("room", True, activity=detail) assert await link.report_activity("room", True, activity=detail) - assert ( - activity_peer.reports - == [{"working": True, "activity": {"kind": "reasoning"}}] * 2 - ) + assert activity_peer.reports == [{"working": True, "activity": DETAIL}] * 2 async def test_fallback_echoed_false_keeps_stop_notification( - self, link: BandLink, activity_peer: ActivityPeer + self, link: BandLink, activity_peer: ActivityPeer, detail: AgentActivityDetail ) -> None: activity_peer.responses.extend( [ @@ -58,10 +78,7 @@ async def test_fallback_echoed_false_keeps_stop_notification( ) stopped = Mock() assert await link.report_activity( - "room", - True, - activity=AgentActivityDetail(kind="reasoning"), - on_stopped=stopped, + "room", True, activity=detail, on_stopped=stopped ) stopped.assert_called_once_with() assert activity_peer.reports[-1] == {"working": True} diff --git a/tests/platform/activity/test_reporting.py b/tests/platform/activity/test_reporting.py new file mode 100644 index 000000000..bfc8bb51f --- /dev/null +++ b/tests/platform/activity/test_reporting.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import logging + +import httpx +import pytest + +from band.platform.link import BandLink +from tests.platform.activity.kit import ActivityPeer + + +class TestActivityReporting: + @pytest.mark.parametrize("working", [True, False]) + async def test_reports_the_boolean_state_to_the_room( + self, link: BandLink, activity_peer: ActivityPeer, working: bool + ) -> None: + assert await link.report_activity("room-1", working) + assert activity_peer.reports == [{"working": working}] + assert activity_peer.requests[-1].url.path.endswith("/chats/room-1/activity") + + async def test_post_is_bounded_and_never_retried( + self, link: BandLink, activity_peer: ActivityPeer + ) -> None: + activity_peer.responses.append(httpx.Response(500)) + assert not await link.report_activity("room-1", True, timeout_seconds=2) + [request] = activity_peer.requests + assert request.extensions["timeout"]["read"] == 2 + + async def test_repeated_failures_warn_once_then_recover( + self, + link: BandLink, + activity_peer: ActivityPeer, + caplog: pytest.LogCaptureFixture, + ) -> None: + activity_peer.responses.extend([httpx.Response(500), httpx.Response(500)]) + with caplog.at_level(logging.INFO, logger="band.platform.message_lifecycle"): + assert not await link.report_activity("room-1", True) + assert not await link.report_activity("room-1", True) + assert await link.report_activity("room-1", True) + assert [record.levelno for record in caplog.records] == [ + logging.WARNING, + logging.INFO, + ] + assert "recovered" in caplog.records[-1].getMessage() diff --git a/tests/platform/activity/test_versions.py b/tests/platform/activity/test_versions.py index 1eb74f30e..c8bcf135f 100644 --- a/tests/platform/activity/test_versions.py +++ b/tests/platform/activity/test_versions.py @@ -1,62 +1,90 @@ from __future__ import annotations +import logging + import pytest from band_rest import AgentActivityDetail +from band.config.settings import DEFAULT_REST_URL from band.platform.link import BandLink -from tests.platform.activity.kit import ActivityPeer +from tests.platform.activity.kit import DETAIL, SUPPORTED_VERSION, ActivityPeer class TestActivityVersion: @pytest.mark.parametrize( ("version", "supported"), [ - ({"agent_version": "1.24.99"}, False), - ({"agent_version": "1.25.0"}, True), - ({"agent_version": "1.26.10"}, True), - ({"agent_version": "2.0.0"}, True), - ({"platform_version": "999.0.0"}, False), - ({"agent_version": "1.25.0-beta"}, False), - ({"agent_version": "broken"}, False), - ({"agent_version": "1.25"}, False), - ({"agent_version": None}, False), - ([], False), + ("1.24.99", False), + (SUPPORTED_VERSION, True), + ("1.26.10", True), + ("2.0.0", True), + ("1.25.0-beta", False), + ("broken", False), + ("1.25", False), + ("2.0", False), + (None, False), ], ) - async def test_agent_version_controls_real_request_detail( + async def test_advertised_version_controls_later_detail( self, link: BandLink, activity_peer: ActivityPeer, - version: object, + detail: AgentActivityDetail, + version: str | None, supported: bool, ) -> None: activity_peer.version = version - detail = AgentActivityDetail(kind="tool_call", name="Bash") assert await link.report_activity("room", True, activity=detail) - assert await link.report_activity("other", True, activity=detail) - assert len(activity_peer.version_requests) == 1 - expected = {"working": True} - if supported: - expected["activity"] = {"kind": "tool_call", "name": "Bash"} - assert activity_peer.reports == [expected, expected] - assert "authorization" not in activity_peer.version_requests[0].headers + assert await link.report_activity("room", True, activity=detail) + later = ( + {"working": True, "activity": DETAIL} if supported else {"working": True} + ) + assert activity_peer.reports == [{"working": True}, later] + + async def test_default_saas_url_learns_detail_from_activity_responses( + self, link: BandLink, activity_peer: ActivityPeer, detail: AgentActivityDetail + ) -> None: + assert link.rest_url == DEFAULT_REST_URL + assert await link.report_activity("room", True, activity=detail) + assert await link.report_activity("room", True, activity=detail) + assert activity_peer.reports[-1] == {"working": True, "activity": DETAIL} + assert all( + request.url.path.endswith("/activity") for request in activity_peer.requests + ) + + async def test_learned_support_is_shared_across_links( + self, + link: BandLink, + second_link: BandLink, + activity_peer: ActivityPeer, + detail: AgentActivityDetail, + ) -> None: + assert await link.report_activity("first", True) + assert await second_link.report_activity("second", True, activity=detail) + assert activity_peer.reports[-1] == {"working": True, "activity": DETAIL} @pytest.mark.parametrize("kind", ["reasoning", "memory", "permission"]) async def test_non_tool_detail_omits_name_and_false_omits_detail( self, link: BandLink, activity_peer: ActivityPeer, kind: str ) -> None: detail = AgentActivityDetail(kind=kind) + assert await link.report_activity("room", True) assert await link.report_activity("room", True, activity=detail) assert await link.report_activity("room", False, activity=detail) - assert activity_peer.reports == [ + assert activity_peer.reports[1:] == [ {"working": True, "activity": {"kind": kind}}, {"working": False}, ] - async def test_boolean_callers_do_not_discover( - self, link: BandLink, activity_peer: ActivityPeer + async def test_unadvertised_version_is_logged_for_diagnosis( + self, + link: BandLink, + activity_peer: ActivityPeer, + caplog: pytest.LogCaptureFixture, ) -> None: - assert await link.report_activity("room", True) - assert await link.report_activity("room", False) - assert not activity_peer.version_requests - assert activity_peer.reports == [{"working": True}, {"working": False}] + activity_peer.version = "1.24.0" + with caplog.at_level(logging.DEBUG, logger="band.platform.activity"): + assert await link.report_activity("room", True) + [record] = caplog.records + assert record.levelno == logging.DEBUG + assert "'1.24.0'" in record.getMessage() diff --git a/tests/platform/test_link.py b/tests/platform/test_link.py index d6401a71d..8a17eeb79 100644 --- a/tests/platform/test_link.py +++ b/tests/platform/test_link.py @@ -3,16 +3,9 @@ from __future__ import annotations import asyncio -import logging from unittest.mock import ANY, AsyncMock, MagicMock, create_autospec, patch import pytest -from band_rest import ( - AsyncRestClient, - NotFoundError, - UnauthorizedError, - UnprocessableEntityError, -) from band_rest.core.api_error import ApiError from band_rest.types.chat_message_metadata import ChatMessageMetadata from band_sdk_core import AgentTopicStatus, DeadReason, SessionState, chat_room_topic @@ -1346,135 +1339,3 @@ async def test_returns_empty_list_on_failure(self): messages = await link.get_stale_processing_messages("room-1") assert messages == [] - - -class TestReportActivity: - """Tests for BandLink.report_activity (boolean working-state reporting).""" - - @pytest.mark.asyncio - async def test_reports_working_true(self): - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock() - - result = await link.report_activity("room-1", True) - - assert result is True - call = link.rest.agent_api_activity.report_agent_chat_activity - call.assert_awaited_once() - assert call.call_args.kwargs["chat_id"] == "room-1" - assert call.call_args.kwargs["working"] is True - - @pytest.mark.asyncio - async def test_passes_per_post_timeout_and_no_retries(self): - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock() - - await link.report_activity("room-1", True, timeout_seconds=2) - - opts = link.rest.agent_api_activity.report_agent_chat_activity.call_args.kwargs[ - "request_options" - ] - assert opts["timeout_in_seconds"] == 2 - assert opts["max_retries"] == 0 - - @pytest.mark.asyncio - async def test_reports_working_false(self): - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock() - - result = await link.report_activity("room-1", False) - - assert result is True - call = link.rest.agent_api_activity.report_agent_chat_activity - assert call.call_args.kwargs["working"] is False - - @pytest.mark.asyncio - async def test_returns_false_on_not_found(self): - - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock( - side_effect=NotFoundError(headers={}, body="no active execution") - ) - - result = await link.report_activity("room-1", True) - - assert result is False - - @pytest.mark.asyncio - async def test_returns_false_on_unauthorized(self): - - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock( - side_effect=UnauthorizedError(headers={}, body="bad key") - ) - - result = await link.report_activity("room-1", True) - - assert result is False - - @pytest.mark.asyncio - async def test_returns_false_on_unprocessable_entity(self): - - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock( - side_effect=UnprocessableEntityError(headers={}, body="bad uuid") - ) - - result = await link.report_activity("room-1", True) - - assert result is False - - @pytest.mark.asyncio - async def test_returns_false_on_network_error(self): - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock( - side_effect=Exception("network down") - ) - - result = await link.report_activity("room-1", True) - - assert result is False - - def test_real_client_exposes_activity_method(self): - """Guard: the real REST client must actually expose the activity method. - - The AsyncMock-based tests above auto-fabricate attributes, so they would - stay green even if `agent_api_activity.report_agent_chat_activity` - disappeared or was renamed in a band_rest bump. This test pins the - real wire contract: instantiate the real client and assert the method - exists and is callable. - """ - - client = AsyncRestClient(api_key="test-key", base_url="https://test.com") - method = getattr(client.agent_api_activity, "report_agent_chat_activity", None) - assert callable(method) - - @pytest.mark.asyncio - async def test_repeated_failures_warn_once_then_recover(self, caplog): - - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock( - side_effect=Exception("down") - ) - - with caplog.at_level(logging.WARNING, logger="band.platform.message_lifecycle"): - assert await link.report_activity("room-1", True) is False - assert await link.report_activity("room-1", True) is False - - warnings = [r for r in caplog.records if r.levelno == logging.WARNING] - assert len(warnings) == 1 # debounced: only the first failure warns - - # Recovery logs once at INFO and re-arms the warning. - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock() - with caplog.at_level(logging.INFO, logger="band.platform.message_lifecycle"): - caplog.clear() - assert await link.report_activity("room-1", True) is True - assert any("recovered" in r.message.lower() for r in caplog.records) diff --git a/tests/runtime/activityownership/conftest.py b/tests/runtime/activityownership/conftest.py index 7fa9fec6e..4b2e227bd 100644 --- a/tests/runtime/activityownership/conftest.py +++ b/tests/runtime/activityownership/conftest.py @@ -4,10 +4,7 @@ from unittest.mock import AsyncMock import pytest -from pytest_httpx import HTTPXMock -import band.platform.message_lifecycle as lifecycle -from band.platform.activity import ActivitySupport from band.platform.event import PlatformEvent from band.platform.link import BandLink from band.runtime.execution import ExecutionContext @@ -22,16 +19,8 @@ @pytest.fixture -async def activity_platform( - monkeypatch: pytest.MonkeyPatch, httpx_mock: HTTPXMock -) -> AsyncIterator[ActivityPlatform]: - monkeypatch.setattr(lifecycle, "activity_support", ActivitySupport()) - httpx_mock.add_response( - url="https://app.band.ai/api/version", - json={"agent_version": "1.25.0"}, - is_optional=True, - ) - yield ActivityPlatform() +def activity_platform() -> ActivityPlatform: + return ActivityPlatform() @pytest.fixture diff --git a/tests/runtime/conftest.py b/tests/runtime/conftest.py index 0a26403e8..4b723a5f0 100644 --- a/tests/runtime/conftest.py +++ b/tests/runtime/conftest.py @@ -34,9 +34,10 @@ import band.platform.message_lifecycle as lifecycle from band.core.types import PlatformConnection, PlatformMessage -from band.platform.activity import ActivitySupport +from band.platform.activity import API_VERSION_HEADER, ActivitySupport from band.runtime.presence import RoomPresence from tests.conftest import make_participant_mock +from tests.platform.activity.kit import SUPPORTED_VERSION from tests.runtime.helpers import AGENT_ID @@ -265,11 +266,7 @@ def link(mock_rest_client: MagicMock) -> MagicMock: @pytest.fixture(autouse=True) def runtime_activity_support(monkeypatch: pytest.MonkeyPatch) -> None: - """Keep deployment discovery independent of runtime scheduling regressions.""" + """Start every runtime test on a platform that already advertised detail.""" support = ActivitySupport() - - async def discover(rest_url: str, budget: float) -> bool: - return True - - monkeypatch.setattr(support, "_discover", discover) + support.observe({API_VERSION_HEADER: SUPPORTED_VERSION}) monkeypatch.setattr(lifecycle, "activity_support", support) From eac1fddc031bb1a144203e01cabc774a7369ca14 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Tue, 20 Oct 2026 17:25:56 +0300 Subject: [PATCH 37/47] fix: Forward activity detail through the baseline ControlLink Every working report now passes activity=; the baseline ControlLink override lacked the keyword, so its TypeError was swallowed and STOP-echo smokes never saw working:true. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DFmR3J53UqbsYDMmTUd3Rb --- src/band/runtime/execution.py | 2 +- tests/e2e/baseline/toolkit/control.py | 9 ++++++++- tests/platform/activity/test_control_link.py | 21 ++++++++++++++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 tests/platform/activity/test_control_link.py diff --git a/src/band/runtime/execution.py b/src/band/runtime/execution.py index 193b171b2..e8297ed52 100644 --- a/src/band/runtime/execution.py +++ b/src/band/runtime/execution.py @@ -1892,7 +1892,7 @@ async def _report_working_state(self, working: bool) -> bool: return await self.link.report_activity( self.room_id, working, - **({"activity": detail} if detail is not None else {}), + activity=detail, timeout_seconds=self.config.working_request_timeout_seconds, on_stopped=partial( self._observe_activity_stop, diff --git a/tests/e2e/baseline/toolkit/control.py b/tests/e2e/baseline/toolkit/control.py index 70e5c1764..2b5087d92 100644 --- a/tests/e2e/baseline/toolkit/control.py +++ b/tests/e2e/baseline/toolkit/control.py @@ -7,6 +7,8 @@ from collections.abc import AsyncGenerator, Callable from contextlib import asynccontextmanager +from band_rest.types.agent_activity_detail import AgentActivityDetail + import band.runtime from band.client.streaming import ( AgentControlPayload, @@ -195,6 +197,7 @@ async def report_activity( room_id: str, working: bool, *, + activity: AgentActivityDetail | None = None, timeout_seconds: int = 2, on_stopped: Callable[[], None] | None = None, ) -> bool: @@ -205,7 +208,11 @@ def observe_stop() -> None: on_stopped() accepted = await super().report_activity( - room_id, working, timeout_seconds=timeout_seconds, on_stopped=observe_stop + room_id, + working, + activity=activity, + timeout_seconds=timeout_seconds, + on_stopped=observe_stop, ) if accepted and not working and room_id == self.control_room_id: self.control.activity_idle.set() diff --git a/tests/platform/activity/test_control_link.py b/tests/platform/activity/test_control_link.py new file mode 100644 index 000000000..008781d86 --- /dev/null +++ b/tests/platform/activity/test_control_link.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import pytest +from band_rest import AgentActivityDetail + +from band.platform.link import BandLink +from tests.e2e.baseline.toolkit.control import ControlLink, ControlRuntime +from tests.platform.activity.kit import DETAIL, ActivityPeer + + +@pytest.mark.usefixtures("learned") +class TestControlLinkActivity: + async def test_baseline_control_link_forwards_activity_detail( + self, link: BandLink, activity_peer: ActivityPeer, detail: AgentActivityDetail + ) -> None: + control_link = ControlLink(agent_id="agent", api_key="test-key") + control_link.rest = link.rest + control_link.control = ControlRuntime() + control_link.control_room_id = "room" + assert await control_link.report_activity("room", True, activity=detail) + assert activity_peer.reports == [{"working": True, "activity": DETAIL}] From 9a1430b685456d02d36edcd0e21a1a257af86439 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Tue, 20 Oct 2026 17:25:56 +0300 Subject: [PATCH 38/47] refactor: Keep one detail reset and one ownership recheck in the reporter The activity-clear seam only ran for already-aborted scopes, which never own activity, so it never cleared anything; aborts reset detail through the false report. Detail resets share one helper and constant, tool_call naming has one rule, and observations are checked for ownership only under the report lock, so a queued stale observation is provably dropped. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DFmR3J53UqbsYDMmTUd3Rb --- src/band/runtime/execution.py | 23 -------- src/band/runtime/working_state.py | 41 +++++++------- .../runtime/activityownership/test_detail.py | 53 ++++++++++++++++--- tests/runtime/workingstate/test_detail.py | 36 +++++++++++++ 4 files changed, 101 insertions(+), 52 deletions(-) diff --git a/src/band/runtime/execution.py b/src/band/runtime/execution.py index e8297ed52..b91312fc1 100644 --- a/src/band/runtime/execution.py +++ b/src/band/runtime/execution.py @@ -446,10 +446,6 @@ def __init__( # The first stop's graceful result; a retry only settles retained work. self._stop_graceful = True - self._on_activity_clear: Callable[[], Awaitable[None]] | None = ( - self._clear_scope_activity - ) - @property def thread_id(self) -> str: """LangGraph thread_id = room_id.""" @@ -884,17 +880,6 @@ async def _stop_retained_tasks(self, deadline: float) -> None: "task(s) still running" ) - async def _clear_activity(self) -> None: - """Invoke the optional activity-clear seam after an aborted cycle.""" - if self._on_activity_clear is None: - return - try: - await self._on_activity_clear() - except Exception: - logger.exception( - "ExecutionContext %s: activity-clear hook failed", self.room_id - ) - # --- Participant management --- def add_participant(self, participant: dict[str, Any]) -> bool: @@ -1983,13 +1968,6 @@ async def observe_activity( waiting=waiting, ) - async def _clear_scope_activity(self) -> None: - scope = self.current_scope - if scope is not None: - await self._working_reporter.clear_detail( - current=partial(self.owns_activity, scope) - ) - def register_turn_task(self, scope: TurnScope, owner: TurnTask) -> None: """Bind completion to the same owner that delivers worker cancellation.""" scope.bind(owner) @@ -2200,7 +2178,6 @@ async def _abort_cycle(self, kind: ControlMode, msg_id: str | None) -> bool: # poisoned into permanently_failed before it's ever actually attempted. if msg_id: self._retry_tracker.discard_attempt(msg_id) - await self._clear_activity() if kind is ControlMode.INTERRUPT and msg_id: # Consume the message so the idle /next resync does not re-return it # (excludes-only-processed) and re-fire the cycle the user just diff --git a/src/band/runtime/working_state.py b/src/band/runtime/working_state.py index 318a9aac6..c60fb4403 100644 --- a/src/band/runtime/working_state.py +++ b/src/band/runtime/working_state.py @@ -20,6 +20,7 @@ UNKNOWN_ACTIVITY_TOOL_NAME = "unknown" +REASONING = AgentActivityDetail(kind="reasoning") class ActivityObserver(Protocol): @@ -28,22 +29,25 @@ async def update(self, activity: AgentActivityDetail) -> None: ... async def set_manual_wait(self, waiting: bool) -> None: ... +def tool_call_activity(name: str | None) -> AgentActivityDetail: + """The platform requires a name on every ``tool_call`` detail.""" + return AgentActivityDetail( + kind="tool_call", name=name or UNKNOWN_ACTIVITY_TOOL_NAME + ) + + def _observer(tools: object) -> ActivityObserver | None: return getattr(tools, "_activity_observer", None) async def report_tool_activity(tools: object, name: str) -> None: if observer := _observer(tools): - await observer.update( - AgentActivityDetail( - kind="tool_call", name=name or UNKNOWN_ACTIVITY_TOOL_NAME - ) - ) + await observer.update(tool_call_activity(name)) async def end_tool_activity(tools: object) -> None: if observer := _observer(tools): - await observer.update(AgentActivityDetail(kind="reasoning")) + await observer.update(REASONING) async def report_memory_activity(tools: object) -> None: @@ -83,7 +87,7 @@ def __init__( self._active = False self._cycle: object | None = None - self._ordinary = AgentActivityDetail(kind="reasoning") + self._ordinary = REASONING self._manual_wait = False self._started_at: float | None = None self._task: asyncio.Task[None] | None = None @@ -112,8 +116,7 @@ async def start(self, *, owner: object | None = None) -> None: self._active = True cycle = owner if owner is not None else object() self._cycle = cycle - self._ordinary = AgentActivityDetail(kind="reasoning") - self._manual_wait = False + self._reset_detail() self._started_at = asyncio.get_running_loop().time() await self._report_locked(True) if self._cycle is cycle and self._active: @@ -128,7 +131,7 @@ async def update( activity: AgentActivityDetail | None = None, waiting: bool | None = None, ) -> None: - if not current() or not self._enabled: + if not self._enabled: return async with self._lock: if not current() or not self._active or self._cap_reached(): @@ -136,10 +139,7 @@ async def update( previous = self.activity if activity is not None: self._ordinary = ( - AgentActivityDetail( - kind="tool_call", - name=activity.name or UNKNOWN_ACTIVITY_TOOL_NAME, - ) + tool_call_activity(activity.name) if activity.kind == "tool_call" else AgentActivityDetail(kind=activity.kind) ) @@ -148,12 +148,6 @@ async def update( if self.activity != previous: await self._report_locked(True) - async def clear_detail(self, *, current: Callable[[], bool]) -> None: - async with self._lock: - if current(): - self._ordinary = AgentActivityDetail(kind="reasoning") - self._manual_wait = False - def defer_false_report(self) -> None: """Start ``working: false`` and return without waiting for it. @@ -198,14 +192,17 @@ async def stop(self) -> None: def _arm_false_report(self) -> asyncio.Task[None] | None: self._active = False self._cycle = None - self._ordinary = AgentActivityDetail(kind="reasoning") - self._manual_wait = False + self._reset_detail() task, self._task = self._task, None if task is not None: task.cancel() self._false_send = asyncio.ensure_future(self._safe_report(False)) return task + def _reset_detail(self) -> None: + self._ordinary = REASONING + self._manual_wait = False + async def _finish_false_send(self) -> None: send = self._false_send if send is None: diff --git a/tests/runtime/activityownership/test_detail.py b/tests/runtime/activityownership/test_detail.py index 83e7bdf77..11fa346c6 100644 --- a/tests/runtime/activityownership/test_detail.py +++ b/tests/runtime/activityownership/test_detail.py @@ -4,7 +4,6 @@ from collections.abc import Callable import pytest -from pytest_httpx import HTTPXMock from band.core.turn_lifecycle import TurnTask, bind_turn_task from band.platform.event import PlatformEvent @@ -95,23 +94,24 @@ async def test_queued_observation_is_rechecked_after_stop_and_replacement( activity_platform: ActivityPlatform, ) -> None: captured: list[AgentTools] = [] + handler_started = asyncio.Event() release = asyncio.Event() async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: captured.append(AgentTools.from_context(ctx)) + handler_started.set() await release.wait() ctx = execution(handler) turn = asyncio.create_task(ctx._process_event(make_message_event(msg_id="old"))) - while not captured: - await asyncio.sleep(0) + await handler_started.wait() activity_platform.hold_next_activity = True change = asyncio.create_task(report_tool_activity(captured[0], "Bash")) await activity_platform.activity_entered.wait() stale = asyncio.create_task(report_memory_activity(captured[0])) ctx.interrupt() activity_platform.activity_release.set() - await asyncio.gather(turn, change, stale, return_exceptions=True) + await asyncio.gather(turn, change, stale) release.set() await ctx._process_event(make_message_event(msg_id="new")) assert activity_platform.activity_bodies[-3:] == [ @@ -124,12 +124,52 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: for body in activity_platform.activity_bodies ) + @pytest.mark.looptime + async def test_waiting_delivery_takes_over_reporting_once_the_owner_finishes( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + worker_release = asyncio.Event() + waiting_tools: list[AgentTools] = [] + waiting_started = asyncio.Event() + waiting_release = asyncio.Event() + workers: list[TurnTask] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + if workers: + waiting_tools.append(tools) + waiting_started.set() + await waiting_release.wait() + return + workers.append( + bind_turn_task(tools, asyncio.create_task(worker_release.wait())) + ) + + ctx = execution(handler) + await ctx._process_event(make_message_event(msg_id="owner")) + waiting = asyncio.create_task( + ctx._process_event(make_message_event(msg_id="waiting")) + ) + await waiting_started.wait() + worker_release.set() + await workers[0].task + await ctx._join_activity_completion() + await report_tool_activity(waiting_tools[0], "Bash") + assert activity_platform.activity_bodies[-3:] == [ + {"working": False}, + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": True, "activity": {"kind": "tool_call", "name": "Bash"}}, + ] + waiting_release.set() + await waiting + @pytest.mark.parametrize("excluded", ["disabled", "hub"]) - async def test_excluded_reporting_ignores_observations_without_discovery( + async def test_excluded_reporting_ignores_observations( self, execution: Callable[..., ExecutionContext], activity_platform: ActivityPlatform, - httpx_mock: HTTPXMock, excluded: str, ) -> None: async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: @@ -147,4 +187,3 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: ctx = execution(handler, **settings) await ctx._process_event(make_message_event()) assert not activity_platform.activity_bodies - assert not httpx_mock.get_requests() diff --git a/tests/runtime/workingstate/test_detail.py b/tests/runtime/workingstate/test_detail.py index 4d776d1fb..c4dd08167 100644 --- a/tests/runtime/workingstate/test_detail.py +++ b/tests/runtime/workingstate/test_detail.py @@ -104,3 +104,39 @@ async def report(working: bool) -> None: finally: await reporter.stop() assert reports == [True, False] + + async def test_observation_queued_behind_a_report_is_rechecked_under_the_lock( + self, + ) -> None: + entered, release = asyncio.Event(), asyncio.Event() + reports: list[str] = [] + + async def report(working: bool) -> None: + if working: + reports.append(reporter.activity.kind) + if reporter.activity.kind == "tool_call": + entered.set() + await release.wait() + + reporter = WorkingStateReporter(report) + owned = True + try: + await reporter.start() + change = asyncio.create_task( + reporter.update( + current=lambda: owned, + activity=AgentActivityDetail(kind="tool_call", name="Bash"), + ) + ) + await entered.wait() + stale = asyncio.create_task( + reporter.update( + current=lambda: owned, activity=AgentActivityDetail(kind="memory") + ) + ) + owned = False + release.set() + await asyncio.gather(change, stale) + finally: + await reporter.stop() + assert reports == ["reasoning", "tool_call"] From 435ae73b98878878e15f046f799a0efef947d6bc Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Tue, 20 Oct 2026 17:25:56 +0300 Subject: [PATCH 39/47] test: Scope the recovery-jitter patch to resync's random reference Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DFmR3J53UqbsYDMmTUd3Rb --- tests/runtime/execution/test_crashrecoverysync.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/runtime/execution/test_crashrecoverysync.py b/tests/runtime/execution/test_crashrecoverysync.py index 22e71bcd9..b779bf812 100644 --- a/tests/runtime/execution/test_crashrecoverysync.py +++ b/tests/runtime/execution/test_crashrecoverysync.py @@ -2,12 +2,14 @@ import asyncio from datetime import UTC, datetime +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, call import pytest from band_sdk_core import ClaimRegistry from band.client.streaming import MessageMetadata +from band.runtime import resync from band.runtime.execution import ( ExecutionContext, ResyncOutcome, @@ -1069,8 +1071,9 @@ async def test_startup_backlog_claim_failure_does_not_process_newer_ws_event( ) monkeypatch.setattr( - "band.runtime.resync.random.uniform", - lambda lower, upper: initial_recovery_delay, + resync, + "random", + SimpleNamespace(uniform=lambda lower, upper: initial_recovery_delay), ) await ctx.on_event( From 90bc52af84cfe5a2a10d46a36dd6951f744da7ce Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Tue, 20 Oct 2026 18:08:55 +0300 Subject: [PATCH 40/47] fix: Log a content-free cause when activity detail is downgraded The downgrade logged the whole 422 body at INFO and called it a rejection before the boolean resend ran. It now logs the status and the first rejected JSON pointer, with the body at DEBUG. The activity POST helpers become module functions instead of class-qualified staticmethods. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DFmR3J53UqbsYDMmTUd3Rb --- src/band/platform/activity.py | 6 +- src/band/platform/message_lifecycle.py | 99 +++++++++++++----------- tests/platform/activity/test_fallback.py | 38 ++++++++- 3 files changed, 90 insertions(+), 53 deletions(-) diff --git a/src/band/platform/activity.py b/src/band/platform/activity.py index aeff03672..b9c9b6e3f 100644 --- a/src/band/platform/activity.py +++ b/src/band/platform/activity.py @@ -50,16 +50,16 @@ def observe(self, headers: Mapping[str, str]) -> None: with self._lock: self._learned = True - def disable(self, *, room_id: str, reason: object) -> None: + def disable(self, *, room_id: str, field: str) -> None: with self._lock: already_disabled = self._disabled self._disabled = True if not already_disabled: logger.info( - "Activity detail rejected in room %s (%s); " + "Activity report with detail returned 422 in room %s (field %s); " "using boolean reporting for this process", room_id, - reason, + field, ) diff --git a/src/band/platform/message_lifecycle.py b/src/band/platform/message_lifecycle.py index 16b990ca2..cb17ad193 100644 --- a/src/band/platform/message_lifecycle.py +++ b/src/band/platform/message_lifecycle.py @@ -116,6 +116,57 @@ def _failure_result(error: Exception) -> SettlementResult: return SettlementResult.UNCONFIRMED +def _first_rejected_field(body: object) -> str: + """A content-free cause: the first JSON pointer a validation error names.""" + envelope = _validated(ErrorEnvelope, body) + if envelope is None: + return "unparsed" + return next(iter(envelope.error.details), "unnamed") + + +async def _post_activity( + rest: AsyncRestClient, + room_id: str, + working: bool, + activity: AgentActivityDetail | None, + timeout_seconds: int, +) -> AgentActivityResponse: + detail = activity if activity_support.supported else None + try: + response = await _send_activity(rest, room_id, working, detail, timeout_seconds) + except ApiError as error: + if detail is None or error.status_code != HTTPStatus.UNPROCESSABLE_ENTITY: + raise + activity_support.disable( + room_id=room_id, field=_first_rejected_field(error.body) + ) + logger.debug("Activity 422 body for room %s: %r", room_id, error.body) + response = await _send_activity(rest, room_id, working, None, timeout_seconds) + activity_support.observe(response.headers) + return response.data.data + + +async def _send_activity( + rest: AsyncRestClient, + room_id: str, + working: bool, + activity: AgentActivityDetail | None, + timeout_seconds: int, +) -> AsyncHttpResponse[ReportAgentChatActivityResponse]: + kwargs: dict[str, Any] = {} + if activity is not None: + kwargs["activity"] = activity + return await rest.agent_api_activity.with_raw_response.report_agent_chat_activity( + chat_id=room_id, + working=working, + request_options={ + **NO_RETRY_REQUEST_OPTIONS, + "timeout_in_seconds": timeout_seconds, + }, + **kwargs, + ) + + def _platform_message(item: ChatMessage, room_id: str) -> PlatformMessage: return PlatformMessage( id=item.id, @@ -337,7 +388,7 @@ async def report_activity( """ try: async with asyncio.timeout(timeout_seconds): - response = await self._post_activity( + response = await _post_activity( rest, room_id, working, @@ -370,52 +421,6 @@ async def report_activity( on_stopped() return True - @staticmethod - async def _post_activity( - rest: AsyncRestClient, - room_id: str, - working: bool, - activity: AgentActivityDetail | None, - timeout_seconds: int, - ) -> AgentActivityResponse: - detail = activity if activity_support.supported else None - try: - response = await MessageLifecycle._send_activity( - rest, room_id, working, detail, timeout_seconds - ) - except ApiError as error: - if detail is None or error.status_code != HTTPStatus.UNPROCESSABLE_ENTITY: - raise - activity_support.disable(room_id=room_id, reason=error.body) - response = await MessageLifecycle._send_activity( - rest, room_id, working, None, timeout_seconds - ) - activity_support.observe(response.headers) - return response.data.data - - @staticmethod - async def _send_activity( - rest: AsyncRestClient, - room_id: str, - working: bool, - activity: AgentActivityDetail | None, - timeout_seconds: int, - ) -> AsyncHttpResponse[ReportAgentChatActivityResponse]: - kwargs: dict[str, Any] = {} - if activity is not None: - kwargs["activity"] = activity - return ( - await rest.agent_api_activity.with_raw_response.report_agent_chat_activity( - chat_id=room_id, - working=working, - request_options={ - **NO_RETRY_REQUEST_OPTIONS, - "timeout_in_seconds": timeout_seconds, - }, - **kwargs, - ) - ) - async def get_next_message( self, rest: AsyncRestClient, room_id: str ) -> PlatformMessage | None: diff --git a/tests/platform/activity/test_fallback.py b/tests/platform/activity/test_fallback.py index 648121f56..28313ee62 100644 --- a/tests/platform/activity/test_fallback.py +++ b/tests/platform/activity/test_fallback.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from unittest.mock import Mock import httpx @@ -34,8 +35,8 @@ async def test_422_retries_without_detail_and_latches_across_links( ] @pytest.mark.parametrize( - ("working", "activity"), - [(True, None), (False, AgentActivityDetail(**DETAIL))], + ("working", "with_detail"), + [(True, False), (False, True)], ids=["no-detail", "idle"], ) async def test_422_without_detail_neither_retries_nor_latches( @@ -45,13 +46,44 @@ async def test_422_without_detail_neither_retries_nor_latches( learned: ActivitySupport, detail: AgentActivityDetail, working: bool, - activity: AgentActivityDetail | None, + with_detail: bool, ) -> None: activity_peer.responses.append(httpx.Response(422)) + activity = detail if with_detail else None assert not await link.report_activity("room", working, activity=activity) assert len(activity_peer.reports) == 1 assert learned.supported + async def test_downgrade_logs_the_rejected_field_not_the_body( + self, + link: BandLink, + activity_peer: ActivityPeer, + detail: AgentActivityDetail, + caplog: pytest.LogCaptureFixture, + ) -> None: + activity_peer.responses.append( + httpx.Response( + 422, + json={ + "error": { + "code": "validation_error", + "details": {"/activity/name": ["echoed Bash --secret"]}, + "message": "Invalid request", + "request_id": "request-1", + } + }, + ) + ) + with caplog.at_level(logging.DEBUG, logger="band.platform"): + assert await link.report_activity("room", True, activity=detail) + [record] = [r for r in caplog.records if r.levelno == logging.INFO] + assert "/activity/name" in record.getMessage() + assert "secret" not in record.getMessage() + assert any( + r.levelno == logging.DEBUG and "secret" in r.getMessage() + for r in caplog.records + ) + @pytest.mark.parametrize("status", [400, 403, 500]) async def test_other_errors_do_not_retry_or_disable( self, From 3c665de606db0732812d0a80b20302dbaf80e08b Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Tue, 20 Oct 2026 18:08:55 +0300 Subject: [PATCH 41/47] refactor: Drop the unreachable cycle token and pin reporter guards _arm_false_report cancels the keep-alive before any replacement cycle can take the report lock, so the owner/cycle token never rejected anything. Names-only canonicalization and the kind constants live beside REASONING, detail is dropped from idle reports only in the lifecycle, and the reachable guards (settled scope, closing execution, unknown tool name, inactive reporter, detail reset across cycles) now each fail a test when removed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DFmR3J53UqbsYDMmTUd3Rb --- src/band/runtime/execution.py | 5 +- src/band/runtime/working_state.py | 45 ++++++++--------- tests/runtime/activityownership/kit.py | 15 +++++- .../runtime/activityownership/test_detail.py | 49 ++++++++++++++++++- tests/runtime/workingstate/test_detail.py | 47 ++++++++++++++++++ 5 files changed, 132 insertions(+), 29 deletions(-) diff --git a/src/band/runtime/execution.py b/src/band/runtime/execution.py index b91312fc1..5749404d9 100644 --- a/src/band/runtime/execution.py +++ b/src/band/runtime/execution.py @@ -1873,11 +1873,10 @@ async def _invoke_handler(self, event: PlatformEvent) -> None: async def _report_working_state(self, working: bool) -> bool: """Report the room's boolean working state (wired into the reporter).""" - detail = self._working_reporter.activity if working else None return await self.link.report_activity( self.room_id, working, - activity=detail, + activity=self._working_reporter.activity, timeout_seconds=self.config.working_request_timeout_seconds, on_stopped=partial( self._observe_activity_stop, @@ -1941,7 +1940,7 @@ async def _start_activity(self, scope: TurnScope) -> None: if self._activity_owner is not scope: self._activity_stop_scope = None self._activity_owner = scope - await self._working_reporter.start(owner=scope) + await self._working_reporter.start() def activity_observer(self, scope: TurnScope | None) -> ActivityObserver | None: return ScopeActivityObserver(self, scope) if scope is not None else None diff --git a/src/band/runtime/working_state.py b/src/band/runtime/working_state.py index c60fb4403..3638881e3 100644 --- a/src/band/runtime/working_state.py +++ b/src/band/runtime/working_state.py @@ -21,6 +21,8 @@ UNKNOWN_ACTIVITY_TOOL_NAME = "unknown" REASONING = AgentActivityDetail(kind="reasoning") +MEMORY = AgentActivityDetail(kind="memory") +PERMISSION = AgentActivityDetail(kind="permission") class ActivityObserver(Protocol): @@ -36,6 +38,13 @@ def tool_call_activity(name: str | None) -> AgentActivityDetail: ) +def names_only(activity: AgentActivityDetail) -> AgentActivityDetail: + """Keep only what the room may see: the kind and, for a tool, its name.""" + if activity.kind == "tool_call": + return tool_call_activity(activity.name) + return AgentActivityDetail(kind=activity.kind) + + def _observer(tools: object) -> ActivityObserver | None: return getattr(tools, "_activity_observer", None) @@ -52,7 +61,7 @@ async def end_tool_activity(tools: object) -> None: async def report_memory_activity(tools: object) -> None: if observer := _observer(tools): - await observer.update(AgentActivityDetail(kind="memory")) + await observer.update(MEMORY) async def set_permission_wait(tools: object, waiting: bool) -> None: @@ -86,7 +95,6 @@ def __init__( self._enabled = enabled self._active = False - self._cycle: object | None = None self._ordinary = REASONING self._manual_wait = False self._started_at: float | None = None @@ -100,13 +108,10 @@ def __init__( @property def activity(self) -> AgentActivityDetail: """The immutable effective detail, read by the bool callback under its lock.""" - return ( - AgentActivityDetail(kind="permission") - if self._manual_wait - else self._ordinary - ) + return PERMISSION if self._manual_wait else self._ordinary - async def start(self, *, owner: object | None = None) -> None: + async def start(self) -> None: + """Begin a cycle after the previous false; a re-entrant start is a no-op.""" if not self._enabled: return await self._finish_false_send() @@ -114,14 +119,12 @@ async def start(self, *, owner: object | None = None) -> None: if self._active: return self._active = True - cycle = owner if owner is not None else object() - self._cycle = cycle self._reset_detail() self._started_at = asyncio.get_running_loop().time() await self._report_locked(True) - if self._cycle is cycle and self._active: + if self._active: self._task = asyncio.create_task( - self._keep_alive(cycle), name="working-state-keepalive" + self._keep_alive(), name="working-state-keepalive" ) async def update( @@ -131,6 +134,7 @@ async def update( activity: AgentActivityDetail | None = None, waiting: bool | None = None, ) -> None: + """Apply an observation; ``current`` rechecks ownership under the lock.""" if not self._enabled: return async with self._lock: @@ -138,11 +142,7 @@ async def update( return previous = self.activity if activity is not None: - self._ordinary = ( - tool_call_activity(activity.name) - if activity.kind == "tool_call" - else AgentActivityDetail(kind=activity.kind) - ) + self._ordinary = names_only(activity) if waiting is not None: self._manual_wait = waiting if self.activity != previous: @@ -191,7 +191,6 @@ async def stop(self) -> None: def _arm_false_report(self) -> asyncio.Task[None] | None: self._active = False - self._cycle = None self._reset_detail() task, self._task = self._task, None if task is not None: @@ -223,7 +222,7 @@ async def _finish_false_send(self) -> None: if send.done(): self._false_send = None - async def _keep_alive(self, cycle: object) -> None: + async def _keep_alive(self) -> None: """Re-send ``working: true`` on cadence until stopped or capped.""" while True: await asyncio.sleep(self._keep_alive_seconds) @@ -239,7 +238,7 @@ async def _keep_alive(self, cycle: object) -> None: self._max_working_state_seconds, ) return - await self._safe_report(True, cycle=cycle) + await self._safe_report(True) def _cap_reached(self) -> bool: if self._max_working_state_seconds is None or self._started_at is None: @@ -247,11 +246,9 @@ def _cap_reached(self) -> bool: elapsed = asyncio.get_running_loop().time() - self._started_at return elapsed >= self._max_working_state_seconds - async def _safe_report(self, working: bool, *, cycle: object | None = None) -> None: + async def _safe_report(self, working: bool) -> None: async with self._lock: - if working and ( - not self._active or self._cycle is not cycle or self._cap_reached() - ): + if working and (not self._active or self._cap_reached()): return await self._report_locked(working) diff --git a/tests/runtime/activityownership/kit.py b/tests/runtime/activityownership/kit.py index bfec560b1..941a3822a 100644 --- a/tests/runtime/activityownership/kit.py +++ b/tests/runtime/activityownership/kit.py @@ -2,7 +2,7 @@ import asyncio import json -from typing import Any +from typing import Any, Literal import httpx @@ -36,3 +36,16 @@ def answer(self, request: httpx.Request) -> httpx.Response: 201, json={"data": {"id": "m-1", "success": True, "recipients": []}} ) return super().answer(request) + + +class ObservedIdle(asyncio.Event): + """The idle event a graceful stop waits on, announcing that wait.""" + + def __init__(self) -> None: + super().__init__() + self.set() + self.awaited = asyncio.Event() + + async def wait(self) -> Literal[True]: + self.awaited.set() + return await super().wait() diff --git a/tests/runtime/activityownership/test_detail.py b/tests/runtime/activityownership/test_detail.py index 11fa346c6..564a0298c 100644 --- a/tests/runtime/activityownership/test_detail.py +++ b/tests/runtime/activityownership/test_detail.py @@ -16,7 +16,7 @@ set_permission_wait, ) from tests.conftest import make_message_event -from tests.runtime.activityownership.kit import ActivityPlatform +from tests.runtime.activityownership.kit import ActivityPlatform, ObservedIdle from tests.runtime.helpers import ROOM_ID @@ -88,6 +88,53 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: await end_tool_activity(captured[0]) assert activity_platform.activity_bodies == before + async def test_settled_turn_cannot_restart_reporting( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + captured: list[AgentTools] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + captured.append(AgentTools.from_context(ctx)) + + ctx = execution(handler) + await ctx._process_event(make_message_event()) + await ctx._join_activity_completion() + before = list(activity_platform.activity_bodies) + await report_tool_activity(captured[0], "Bash") + assert before[-1] == {"working": False} + assert activity_platform.activity_bodies == before + + async def test_graceful_stop_drops_observations_from_the_draining_turn( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + captured: list[AgentTools] = [] + handler_started = asyncio.Event() + release = asyncio.Event() + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + captured.append(AgentTools.from_context(ctx)) + handler_started.set() + await release.wait() + + ctx = execution(handler) + idle = ctx._idle_event = ObservedIdle() + await ctx.start() + await ctx.on_event(make_message_event()) + await handler_started.wait() + stopping = asyncio.create_task(ctx.stop(timeout=30)) + await idle.awaited.wait() + await report_tool_activity(captured[0], "Bash") + release.set() + assert await stopping + assert activity_platform.activity_bodies == [ + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": False}, + ] + async def test_queued_observation_is_rechecked_after_stop_and_replacement( self, execution: Callable[..., ExecutionContext], diff --git a/tests/runtime/workingstate/test_detail.py b/tests/runtime/workingstate/test_detail.py index c4dd08167..6cecd58ec 100644 --- a/tests/runtime/workingstate/test_detail.py +++ b/tests/runtime/workingstate/test_detail.py @@ -140,3 +140,50 @@ async def report(working: bool) -> None: finally: await reporter.stop() assert reports == ["reasoning", "tool_call"] + + async def test_empty_tool_name_reports_unknown(self) -> None: + reports: list[dict[str, Any]] = [] + + async def report(working: bool) -> None: + if working: + reports.append(reporter.activity.model_dump(exclude_unset=True)) + + reporter = WorkingStateReporter(report) + try: + await reporter.start() + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail(kind="tool_call", name=""), + ) + finally: + await reporter.stop() + assert reports[-1] == {"kind": "tool_call", "name": "unknown"} + + async def test_observation_outside_a_cycle_reports_nothing(self) -> None: + reports: list[bool] = [] + + async def report(working: bool) -> None: + reports.append(working) + + reporter = WorkingStateReporter(report) + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail(kind="tool_call", name="Bash"), + ) + assert reports == [] + + async def test_permission_wait_does_not_carry_into_the_next_cycle(self) -> None: + reports: list[str | None] = [] + + async def report(working: bool) -> None: + reports.append(reporter.activity.kind if working else None) + + reporter = WorkingStateReporter(report) + try: + await reporter.start() + await reporter.update(current=lambda: True, waiting=True) + await reporter.stop() + await reporter.start() + finally: + await reporter.stop() + assert reports == ["reasoning", "permission", None, "reasoning", None] From 2c4a9f75244ba6aacaedc0c2300598dccbe4dc17 Mon Sep 17 00:00:00 2001 From: Alexander Zaikman Date: Tue, 20 Oct 2026 18:08:55 +0300 Subject: [PATCH 42/47] test: Share one activity-support isolation helper Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01DFmR3J53UqbsYDMmTUd3Rb --- tests/activitysupport.py | 25 ++++++++++++++++++++++++ tests/platform/activity/conftest.py | 16 +++++++-------- tests/platform/activity/kit.py | 4 ++-- tests/platform/activity/test_versions.py | 14 +++---------- tests/runtime/conftest.py | 8 ++------ 5 files changed, 39 insertions(+), 28 deletions(-) create mode 100644 tests/activitysupport.py diff --git a/tests/activitysupport.py b/tests/activitysupport.py new file mode 100644 index 000000000..eeba99419 --- /dev/null +++ b/tests/activitysupport.py @@ -0,0 +1,25 @@ +"""Isolate the process-wide activity-detail negotiation per test.""" + +from __future__ import annotations + +import pytest + +import band.platform.message_lifecycle as lifecycle +from band.platform.activity import ( + API_VERSION_HEADER, + MINIMUM_ACTIVITY_VERSION, + ActivitySupport, +) + +SUPPORTED_VERSION = ".".join(map(str, MINIMUM_ACTIVITY_VERSION)) + + +def install_activity_support( + monkeypatch: pytest.MonkeyPatch, *, learned: bool +) -> ActivitySupport: + """Give the lifecycle a fresh negotiation, optionally already supported.""" + support = ActivitySupport() + if learned: + support.observe({API_VERSION_HEADER: SUPPORTED_VERSION}) + monkeypatch.setattr(lifecycle, "activity_support", support) + return support diff --git a/tests/platform/activity/conftest.py b/tests/platform/activity/conftest.py index 10b7390a4..f14f18e0e 100644 --- a/tests/platform/activity/conftest.py +++ b/tests/platform/activity/conftest.py @@ -5,18 +5,16 @@ import pytest from band_rest import AgentActivityDetail -import band.platform.message_lifecycle as lifecycle -from band.platform.activity import API_VERSION_HEADER, ActivitySupport +from band.platform.activity import ActivitySupport from band.platform.link import BandLink -from tests.platform.activity.kit import DETAIL, SUPPORTED_VERSION, ActivityPeer +from tests.activitysupport import install_activity_support +from tests.platform.activity.kit import DETAIL, ActivityPeer from tests.runtime.helpers import rest_client_over @pytest.fixture(autouse=True) def support(monkeypatch: pytest.MonkeyPatch) -> ActivitySupport: - support = ActivitySupport() - monkeypatch.setattr(lifecycle, "activity_support", support) - return support + return install_activity_support(monkeypatch, learned=False) @pytest.fixture @@ -31,6 +29,7 @@ def detail() -> AgentActivityDetail: @pytest.fixture async def link(activity_peer: ActivityPeer) -> AsyncIterator[BandLink]: + """A link on the SDK's default (SaaS) ``rest_url``.""" async with rest_client_over(activity_peer.respond) as rest: link = BandLink(agent_id="agent", api_key="test-key") link.rest = rest @@ -45,7 +44,6 @@ def second_link(link: BandLink) -> BandLink: @pytest.fixture -def learned(support: ActivitySupport) -> ActivitySupport: +def learned(monkeypatch: pytest.MonkeyPatch) -> ActivitySupport: """A process that already saw a supporting ``X-API-Version``.""" - support.observe({API_VERSION_HEADER: SUPPORTED_VERSION}) - return support + return install_activity_support(monkeypatch, learned=True) diff --git a/tests/platform/activity/kit.py b/tests/platform/activity/kit.py index c48168ca7..9eae53c17 100644 --- a/tests/platform/activity/kit.py +++ b/tests/platform/activity/kit.py @@ -7,9 +7,9 @@ import httpx -from band.platform.activity import API_VERSION_HEADER, MINIMUM_ACTIVITY_VERSION +from band.platform.activity import API_VERSION_HEADER +from tests.activitysupport import SUPPORTED_VERSION -SUPPORTED_VERSION = ".".join(map(str, MINIMUM_ACTIVITY_VERSION)) DETAIL = {"kind": "tool_call", "name": "Bash"} diff --git a/tests/platform/activity/test_versions.py b/tests/platform/activity/test_versions.py index c8bcf135f..4274f23d9 100644 --- a/tests/platform/activity/test_versions.py +++ b/tests/platform/activity/test_versions.py @@ -5,9 +5,9 @@ import pytest from band_rest import AgentActivityDetail -from band.config.settings import DEFAULT_REST_URL from band.platform.link import BandLink -from tests.platform.activity.kit import DETAIL, SUPPORTED_VERSION, ActivityPeer +from tests.activitysupport import SUPPORTED_VERSION +from tests.platform.activity.kit import DETAIL, ActivityPeer class TestActivityVersion: @@ -15,7 +15,7 @@ class TestActivityVersion: ("version", "supported"), [ ("1.24.99", False), - (SUPPORTED_VERSION, True), + pytest.param(SUPPORTED_VERSION, True, id="minimum-version-on-saas-default"), ("1.26.10", True), ("2.0.0", True), ("1.25.0-beta", False), @@ -40,14 +40,6 @@ async def test_advertised_version_controls_later_detail( {"working": True, "activity": DETAIL} if supported else {"working": True} ) assert activity_peer.reports == [{"working": True}, later] - - async def test_default_saas_url_learns_detail_from_activity_responses( - self, link: BandLink, activity_peer: ActivityPeer, detail: AgentActivityDetail - ) -> None: - assert link.rest_url == DEFAULT_REST_URL - assert await link.report_activity("room", True, activity=detail) - assert await link.report_activity("room", True, activity=detail) - assert activity_peer.reports[-1] == {"working": True, "activity": DETAIL} assert all( request.url.path.endswith("/activity") for request in activity_peer.requests ) diff --git a/tests/runtime/conftest.py b/tests/runtime/conftest.py index 4b723a5f0..f7457f7ed 100644 --- a/tests/runtime/conftest.py +++ b/tests/runtime/conftest.py @@ -32,12 +32,10 @@ ListAgentChatParticipantsResponse, ) -import band.platform.message_lifecycle as lifecycle from band.core.types import PlatformConnection, PlatformMessage -from band.platform.activity import API_VERSION_HEADER, ActivitySupport from band.runtime.presence import RoomPresence +from tests.activitysupport import install_activity_support from tests.conftest import make_participant_mock -from tests.platform.activity.kit import SUPPORTED_VERSION from tests.runtime.helpers import AGENT_ID @@ -267,6 +265,4 @@ def link(mock_rest_client: MagicMock) -> MagicMock: @pytest.fixture(autouse=True) def runtime_activity_support(monkeypatch: pytest.MonkeyPatch) -> None: """Start every runtime test on a platform that already advertised detail.""" - support = ActivitySupport() - support.observe({API_VERSION_HEADER: SUPPORTED_VERSION}) - monkeypatch.setattr(lifecycle, "activity_support", support) + install_activity_support(monkeypatch, learned=True) From cd3c6aac81a584bb4a0eb43d113ac7d1c81ab72d Mon Sep 17 00:00:00 2001 From: Room Sim Date: Tue, 20 Oct 2026 21:08:30 +0300 Subject: [PATCH 43/47] sim840 (cherry picked from commit 8f9313e35bc2778881e1b7dde30e381b1e5be709) --- docs/turn-outcome.md | 12 + src/band/platform/activity.py | 66 +++++ src/band/platform/link.py | 3 + src/band/platform/message_lifecycle.py | 86 +++++-- src/band/runtime/execution.py | 59 +++-- src/band/runtime/sessionconfig.py | 7 +- src/band/runtime/tools/agent.py | 3 + src/band/runtime/working_state.py | 159 ++++++++---- tests/activitysupport.py | 25 ++ tests/e2e/baseline/toolkit/control.py | 9 +- tests/platform/activity/conftest.py | 49 ++++ tests/platform/activity/kit.py | 39 +++ tests/platform/activity/test_control_link.py | 21 ++ tests/platform/activity/test_deadlines.py | 32 +++ tests/platform/activity/test_fallback.py | 116 +++++++++ tests/platform/activity/test_reporting.py | 44 ++++ tests/platform/activity/test_versions.py | 82 ++++++ tests/platform/test_link.py | 139 ----------- tests/runtime/activityownership/conftest.py | 12 +- tests/runtime/activityownership/kit.py | 34 +++ .../runtime/activityownership/test_detail.py | 236 ++++++++++++++++++ tests/runtime/conftest.py | 7 + .../execution/test_crashrecoverysync.py | 25 +- tests/runtime/workingstate/test_detail.py | 189 ++++++++++++++ 24 files changed, 1226 insertions(+), 228 deletions(-) create mode 100644 src/band/platform/activity.py create mode 100644 tests/activitysupport.py create mode 100644 tests/platform/activity/conftest.py create mode 100644 tests/platform/activity/kit.py create mode 100644 tests/platform/activity/test_control_link.py create mode 100644 tests/platform/activity/test_deadlines.py create mode 100644 tests/platform/activity/test_fallback.py create mode 100644 tests/platform/activity/test_reporting.py create mode 100644 tests/platform/activity/test_versions.py create mode 100644 tests/runtime/activityownership/test_detail.py create mode 100644 tests/runtime/workingstate/test_detail.py diff --git a/docs/turn-outcome.md b/docs/turn-outcome.md index 6324fa535..a665f07d0 100644 --- a/docs/turn-outcome.md +++ b/docs/turn-outcome.md @@ -424,3 +424,15 @@ once. It does not rewrite the original delivery's settled acknowledgement. Explicit interruption invalidates its activity; leave and shutdown clear the indicator while adapter residency retains responsibility for released workers. The authoritative idle report finishes before a replacement reports working. + +`SessionConfig.enable_working_state` controls both the indicator and activity +detail, independently of chat narration and tool capabilities. Changes and +keep-alives share one serialized reporter. The duration cap stops both kinds +of refresh without cancelling provider work; actual completion still clears. + +Detail compatibility is learned from the `X-API-Version` header the platform +puts on every activity response. Reports stay boolean until a response in the +process advertises agent API 1.25.0 or later; a missing or older version keeps +boolean reporting. A detail-bearing HTTP 422 permanently disables detail across +all links in the process and retries that report once without detail, within +its original deadline. diff --git a/src/band/platform/activity.py b/src/band/platform/activity.py new file mode 100644 index 000000000..b9c9b6e3f --- /dev/null +++ b/src/band/platform/activity.py @@ -0,0 +1,66 @@ +"""Process-wide negotiation for optional room activity detail. + +Every agent API response names its version in ``X-API-Version``. Detail is sent +once any activity response reported a supporting version. A detail rejection on +any link permanently downgrades every link; boolean liveness remains available. +""" + +from __future__ import annotations + +import logging +import re +from collections.abc import Mapping +from threading import Lock + +logger = logging.getLogger(__name__) +# The first agent API that accepts ``activity``; older ones reject it with 422. +MINIMUM_ACTIVITY_VERSION = (1, 25, 0) +API_VERSION_HEADER = "x-api-version" +RELEASE_VERSION = re.compile(r"([0-9]+)\.([0-9]+)\.([0-9]+)") + + +def _supports_detail(version: str | None) -> bool: + match = RELEASE_VERSION.fullmatch(version) if version is not None else None + return ( + match is not None + and tuple(map(int, match.groups())) >= MINIMUM_ACTIVITY_VERSION + ) + + +class ActivitySupport: + """Learn detail support from response headers; a rejection latches it off.""" + + def __init__(self) -> None: + self._lock = Lock() + self._learned = False + self._disabled = False + + @property + def supported(self) -> bool: + with self._lock: + return self._learned and not self._disabled + + def observe(self, headers: Mapping[str, str]) -> None: + version = headers.get(API_VERSION_HEADER) + if not _supports_detail(version): + logger.debug( + "Agent API version %r does not advertise activity detail", version + ) + return + with self._lock: + self._learned = True + + def disable(self, *, room_id: str, field: str) -> None: + with self._lock: + already_disabled = self._disabled + self._disabled = True + if not already_disabled: + logger.info( + "Activity report with detail returned 422 in room %s (field %s); " + "using boolean reporting for this process", + room_id, + field, + ) + + +activity_support = ActivitySupport() diff --git a/src/band/platform/link.py b/src/band/platform/link.py index c512952d9..f85634185 100644 --- a/src/band/platform/link.py +++ b/src/band/platform/link.py @@ -13,6 +13,7 @@ from functools import partial from typing import TYPE_CHECKING +from band_rest.types.agent_activity_detail import AgentActivityDetail from band_sdk_core import AgentTopicKind, SessionState from band.client.rest import AsyncRestClient @@ -647,6 +648,7 @@ async def report_activity( room_id: str, working: bool, *, + activity: AgentActivityDetail | None = None, timeout_seconds: int = 2, on_stopped: Callable[[], None] | None = None, ) -> bool: @@ -654,6 +656,7 @@ async def report_activity( self.rest, room_id, working, + activity=activity, timeout_seconds=timeout_seconds, on_stopped=on_stopped, ) diff --git a/src/band/platform/message_lifecycle.py b/src/band/platform/message_lifecycle.py index 2e48b52fb..cb17ad193 100644 --- a/src/band/platform/message_lifecycle.py +++ b/src/band/platform/message_lifecycle.py @@ -6,6 +6,7 @@ from __future__ import annotations +import asyncio import logging from collections import OrderedDict from collections.abc import Awaitable, Callable @@ -14,9 +15,14 @@ from http import HTTPStatus from typing import Any, TypeVar +from band_rest.agent_api_activity.types.report_agent_chat_activity_response import ( + ReportAgentChatActivityResponse, +) from band_rest.core.api_error import ApiError from band_rest.core.http_response import AsyncHttpResponse from band_rest.core.request_options import RequestOptions +from band_rest.types.agent_activity_detail import AgentActivityDetail +from band_rest.types.agent_activity_response import AgentActivityResponse from band_rest.types.chat_message import ChatMessage from band_rest.types.error import Error as PlatformError from band_rest.types.validation_error import ValidationError as ErrorEnvelope @@ -32,6 +38,7 @@ from band.core.exceptions import RoomExecutionStoppedError from band.core.settlement import SettlementResult from band.core.types import metadata_to_dict +from band.platform.activity import activity_support from band.runtime.messages import PlatformMessage logger = logging.getLogger(__name__) @@ -109,6 +116,57 @@ def _failure_result(error: Exception) -> SettlementResult: return SettlementResult.UNCONFIRMED +def _first_rejected_field(body: object) -> str: + """A content-free cause: the first JSON pointer a validation error names.""" + envelope = _validated(ErrorEnvelope, body) + if envelope is None: + return "unparsed" + return next(iter(envelope.error.details), "unnamed") + + +async def _post_activity( + rest: AsyncRestClient, + room_id: str, + working: bool, + activity: AgentActivityDetail | None, + timeout_seconds: int, +) -> AgentActivityResponse: + detail = activity if activity_support.supported else None + try: + response = await _send_activity(rest, room_id, working, detail, timeout_seconds) + except ApiError as error: + if detail is None or error.status_code != HTTPStatus.UNPROCESSABLE_ENTITY: + raise + activity_support.disable( + room_id=room_id, field=_first_rejected_field(error.body) + ) + logger.debug("Activity 422 body for room %s: %r", room_id, error.body) + response = await _send_activity(rest, room_id, working, None, timeout_seconds) + activity_support.observe(response.headers) + return response.data.data + + +async def _send_activity( + rest: AsyncRestClient, + room_id: str, + working: bool, + activity: AgentActivityDetail | None, + timeout_seconds: int, +) -> AsyncHttpResponse[ReportAgentChatActivityResponse]: + kwargs: dict[str, Any] = {} + if activity is not None: + kwargs["activity"] = activity + return await rest.agent_api_activity.with_raw_response.report_agent_chat_activity( + chat_id=room_id, + working=working, + request_options={ + **NO_RETRY_REQUEST_OPTIONS, + "timeout_in_seconds": timeout_seconds, + }, + **kwargs, + ) + + def _platform_message(item: ChatMessage, room_id: str) -> PlatformMessage: return PlatformMessage( id=item.id, @@ -306,6 +364,7 @@ async def report_activity( room_id: str, working: bool, *, + activity: AgentActivityDetail | None = None, timeout_seconds: int = 2, on_stopped: Callable[[], None] | None = None, ) -> bool: @@ -318,26 +377,25 @@ async def report_activity( the backstop and activity reporting must never break message processing. - ``timeout_seconds`` bounds each POST so a slow/half-open endpoint - can't stall the keep-alive or wedge teardown. Retries are off: a - dropped keep-alive gets re-sent next cadence tick, and a dropped - ``false`` is cleared by the platform TTL either way — retrying would - only add latency. + Detail is attached once a response reported a supporting agent API + version. One deadline bounds the POST and its sole compatibility retry: + a rejected detail permanently downgrades the process to boolean + reports; other failures wait for the next keep-alive or platform TTL. A successful ``working=True`` report echoed as ``False`` identifies a stopped execution. Notify ``on_stopped`` separately from the request's success boolean; transport failures never imply STOP. """ try: - response = await rest.agent_api_activity.report_agent_chat_activity( - chat_id=room_id, - working=working, - request_options={ - **NO_RETRY_REQUEST_OPTIONS, - "timeout_in_seconds": timeout_seconds, - }, - ) - stopped = working and response.data.working is False + async with asyncio.timeout(timeout_seconds): + response = await _post_activity( + rest, + room_id, + working, + activity if working else None, + timeout_seconds, + ) + stopped = working and response.working is False except Exception as e: # noqa: BLE001 -- best-effort event emission must not crash the turn/link if not self._activity_report_failing: self._activity_report_failing = True diff --git a/src/band/runtime/execution.py b/src/band/runtime/execution.py index 971ee0d0f..5749404d9 100644 --- a/src/band/runtime/execution.py +++ b/src/band/runtime/execution.py @@ -19,6 +19,7 @@ import logging from collections import deque from collections.abc import Awaitable, Callable, Iterator +from dataclasses import dataclass from datetime import UTC, datetime from enum import Enum, StrEnum from functools import partial @@ -30,6 +31,7 @@ runtime_checkable, ) +from band_rest.types.agent_activity_detail import AgentActivityDetail from band_sdk_core import ( AgentFailure, ClaimRegistry, @@ -84,7 +86,7 @@ from band.runtime.resync import ResyncSchedule from band.runtime.sessionconfig import SessionConfig from band.runtime.tools.agent import AgentTools -from band.runtime.working_state import WorkingStateReporter +from band.runtime.working_state import ActivityObserver, WorkingStateReporter if TYPE_CHECKING: from band.platform.link import BandLink @@ -282,6 +284,18 @@ async def resume_room(self) -> None: ExecutionHandler = Callable[["ExecutionContext", PlatformEvent], Awaitable[None]] +@dataclass(frozen=True) +class ScopeActivityObserver: + context: ExecutionContext + scope: TurnScope + + async def update(self, activity: AgentActivityDetail) -> None: + await self.context.observe_activity(self.scope, activity=activity) + + async def set_manual_wait(self, waiting: bool) -> None: + await self.context.observe_activity(self.scope, waiting=waiting) + + class ExecutionContext: """ Default execution: context accumulation model. @@ -432,11 +446,6 @@ def __init__( # The first stop's graceful result; a retry only settles retained work. self._stop_graceful = True - # Optional seam for clearing user-visible activity state ("reasoning…") - # when a cycle is interrupted/stopped. Filled by the activity-signal - # work; a no-op (None) here. - self._on_activity_clear: Callable[[], Awaitable[None]] | None = None - @property def thread_id(self) -> str: """LangGraph thread_id = room_id.""" @@ -871,17 +880,6 @@ async def _stop_retained_tasks(self, deadline: float) -> None: "task(s) still running" ) - async def _clear_activity(self) -> None: - """Invoke the optional activity-clear seam after an aborted cycle.""" - if self._on_activity_clear is None: - return - try: - await self._on_activity_clear() - except Exception: - logger.exception( - "ExecutionContext %s: activity-clear hook failed", self.room_id - ) - # --- Participant management --- def add_participant(self, participant: dict[str, Any]) -> bool: @@ -1878,6 +1876,7 @@ async def _report_working_state(self, working: bool) -> bool: return await self.link.report_activity( self.room_id, working, + activity=self._working_reporter.activity, timeout_seconds=self.config.working_request_timeout_seconds, on_stopped=partial( self._observe_activity_stop, @@ -1943,6 +1942,31 @@ async def _start_activity(self, scope: TurnScope) -> None: self._activity_owner = scope await self._working_reporter.start() + def activity_observer(self, scope: TurnScope | None) -> ActivityObserver | None: + return ScopeActivityObserver(self, scope) if scope is not None else None + + def owns_activity(self, scope: TurnScope) -> bool: + return self._activity_owner is scope and not scope.aborted and not self._closing + + async def observe_activity( + self, + scope: TurnScope, + *, + activity: AgentActivityDetail | None = None, + waiting: bool | None = None, + ) -> None: + if ( + self._activity_owner is None + and scope is self.current_scope + and not scope.settled + ): + await self._acquire_activity(scope) + await self._working_reporter.update( + current=partial(self.owns_activity, scope), + activity=activity, + waiting=waiting, + ) + def register_turn_task(self, scope: TurnScope, owner: TurnTask) -> None: """Bind completion to the same owner that delivers worker cancellation.""" scope.bind(owner) @@ -2153,7 +2177,6 @@ async def _abort_cycle(self, kind: ControlMode, msg_id: str | None) -> bool: # poisoned into permanently_failed before it's ever actually attempted. if msg_id: self._retry_tracker.discard_attempt(msg_id) - await self._clear_activity() if kind is ControlMode.INTERRUPT and msg_id: # Consume the message so the idle /next resync does not re-return it # (excludes-only-processed) and re-fire the cycle the user just diff --git a/src/band/runtime/sessionconfig.py b/src/band/runtime/sessionconfig.py index 9c9b73171..05e5a98e6 100644 --- a/src/band/runtime/sessionconfig.py +++ b/src/band/runtime/sessionconfig.py @@ -43,14 +43,13 @@ class SessionConfig(BaseSettings): # Queue traffic cannot postpone a check; active handlers remain serial. idle_resync_seconds: PositiveFiniteSeconds = 60.0 - # --- Working-state (boolean "is the agent reasoning") reporting --- - # Kill-switch: disable all working-state reporting instantly if needed. + # Indicator and activity detail share this switch, including approval waits. enable_working_state: bool = True # Keep-alive cadence: how often working:true is refreshed while a cycle runs. # Must be < TTL/2 so a single missed/slow ping still lands inside the TTL. working_keep_alive_seconds: float = 3.0 - # Per-POST deadline for each activity report (must be < cadence so a slow - # POST can't pile onto the next keep-alive tick). + # Total deadline for one report's POST and its compatibility fallback. + # Must be < cadence so a slow report cannot pile onto the next tick. working_request_timeout_seconds: int = 2 # Optional upper bound on how long we keep asserting working:true for one # cycle. When exceeded we stop refreshing (platform TTL clears it); we never diff --git a/src/band/runtime/tools/agent.py b/src/band/runtime/tools/agent.py index 88d53634b..823291c83 100644 --- a/src/band/runtime/tools/agent.py +++ b/src/band/runtime/tools/agent.py @@ -74,6 +74,7 @@ validate_tool_arguments, ) from band.runtime.turn import records_turn_effects +from band.runtime.working_state import ActivityObserver if TYPE_CHECKING: from anthropic.types import ToolParam @@ -344,6 +345,7 @@ def __init__( self._agent_id = agent_id self._ctx: ExecutionContext | None = None self._scope: TurnScope | None = None + self._activity_observer: ActivityObserver | None = None self.turn = Turn() @property @@ -384,6 +386,7 @@ def from_context( ) tools._ctx = ctx tools._scope = ctx.current_scope if bind_scope else None + tools._activity_observer = ctx.activity_observer(tools._scope) tools.turn = Turn(posts_missing_reply=ctx.config.report_turn_failures_to_room) return tools diff --git a/src/band/runtime/working_state.py b/src/band/runtime/working_state.py index 4d842c7fe..3638881e3 100644 --- a/src/band/runtime/working_state.py +++ b/src/band/runtime/working_state.py @@ -1,26 +1,8 @@ -"""Boolean working-state keep-alive reporter. - -Owns the per-room "is the agent working" signal for a single reasoning cycle: -sends ``working: true`` when a cycle starts, refreshes it on a keep-alive -cadence while the cycle is active, and sends an authoritative ``working: false`` -when the cycle ends. The platform applies a short TTL, so if refreshes stop for -any reason (crash, disconnect, loop-block) the indicator clears on its own — the -keep-alive is a *liveness* signal, not a correctness signal. - -The reporter is transport-agnostic: it is given an async ``report(working)`` -callback (wired to ``BandLink.report_activity`` by the runtime) and never raises -into the caller's reasoning loop. - -Contract: ``report`` MUST be time-bounded. The reporter awaits it (including the -authoritative false-send during teardown), so an unbounded callback could wedge -cycle teardown and cancellation propagation. ``BandLink.report_activity`` applies -a per-POST timeout to satisfy this. - -Cadence accuracy: the keep-alive sleeps ``keep_alive_seconds`` and *then* reports, -so the effective interval between successful refreshes is -``keep_alive_seconds + report-latency``. The platform-TTL headroom is preserved by -the ``cadence < TTL/2`` guard on ``SessionConfig`` (with the per-POST timeout kept -below the cadence), so even one missed/slow ping stays within the TTL window. +"""Serialized working-state and names-only activity for one provider turn. + +The boolean callback must be time bounded. It reads immutable detail under the +same lock as changes and heartbeats. Terminal false is shielded; platform TTL +backstops failed reports. A duration cap stops refreshes without cancelling work. """ from __future__ import annotations @@ -28,12 +10,65 @@ import asyncio import logging from collections.abc import Awaitable, Callable +from typing import Protocol + +from band_rest.types.agent_activity_detail import AgentActivityDetail logger = logging.getLogger(__name__) ReportFn = Callable[[bool], Awaitable[object]] +UNKNOWN_ACTIVITY_TOOL_NAME = "unknown" +REASONING = AgentActivityDetail(kind="reasoning") +MEMORY = AgentActivityDetail(kind="memory") +PERMISSION = AgentActivityDetail(kind="permission") + + +class ActivityObserver(Protocol): + async def update(self, activity: AgentActivityDetail) -> None: ... + + async def set_manual_wait(self, waiting: bool) -> None: ... + + +def tool_call_activity(name: str | None) -> AgentActivityDetail: + """The platform requires a name on every ``tool_call`` detail.""" + return AgentActivityDetail( + kind="tool_call", name=name or UNKNOWN_ACTIVITY_TOOL_NAME + ) + + +def names_only(activity: AgentActivityDetail) -> AgentActivityDetail: + """Keep only what the room may see: the kind and, for a tool, its name.""" + if activity.kind == "tool_call": + return tool_call_activity(activity.name) + return AgentActivityDetail(kind=activity.kind) + + +def _observer(tools: object) -> ActivityObserver | None: + return getattr(tools, "_activity_observer", None) + + +async def report_tool_activity(tools: object, name: str) -> None: + if observer := _observer(tools): + await observer.update(tool_call_activity(name)) + + +async def end_tool_activity(tools: object) -> None: + if observer := _observer(tools): + await observer.update(REASONING) + + +async def report_memory_activity(tools: object) -> None: + if observer := _observer(tools): + await observer.update(MEMORY) + + +async def set_permission_wait(tools: object, waiting: bool) -> None: + if observer := _observer(tools): + await observer.set_manual_wait(waiting) + + def _retrieve_task_error(task: asyncio.Task[None]) -> None: if not task.cancelled(): task.exception() @@ -60,6 +95,8 @@ def __init__( self._enabled = enabled self._active = False + self._ordinary = REASONING + self._manual_wait = False self._started_at: float | None = None self._task: asyncio.Task[None] | None = None self._false_send: asyncio.Future[None] | None = None @@ -68,19 +105,48 @@ def __init__( # (boolean payload carries no sequence number to disambiguate). self._lock = asyncio.Lock() + @property + def activity(self) -> AgentActivityDetail: + """The immutable effective detail, read by the bool callback under its lock.""" + return PERMISSION if self._manual_wait else self._ordinary + async def start(self) -> None: - """Begin a reasoning cycle: send ``working: true`` and arm keep-alive.""" - if not self._enabled or self._active: - # Re-entrant start while active is a no-op: the signal is already - # alive and the keep-alive task is already running. + """Begin a cycle after the previous false; a re-entrant start is a no-op.""" + if not self._enabled: return await self._finish_false_send() - self._active = True - self._started_at = asyncio.get_running_loop().time() - await self._safe_report(True) - self._task = asyncio.create_task( - self._keep_alive(), name="working-state-keepalive" - ) + async with self._lock: + if self._active: + return + self._active = True + self._reset_detail() + self._started_at = asyncio.get_running_loop().time() + await self._report_locked(True) + if self._active: + self._task = asyncio.create_task( + self._keep_alive(), name="working-state-keepalive" + ) + + async def update( + self, + *, + current: Callable[[], bool], + activity: AgentActivityDetail | None = None, + waiting: bool | None = None, + ) -> None: + """Apply an observation; ``current`` rechecks ownership under the lock.""" + if not self._enabled: + return + async with self._lock: + if not current() or not self._active or self._cap_reached(): + return + previous = self.activity + if activity is not None: + self._ordinary = names_only(activity) + if waiting is not None: + self._manual_wait = waiting + if self.activity != previous: + await self._report_locked(True) def defer_false_report(self) -> None: """Start ``working: false`` and return without waiting for it. @@ -125,12 +191,17 @@ async def stop(self) -> None: def _arm_false_report(self) -> asyncio.Task[None] | None: self._active = False + self._reset_detail() task, self._task = self._task, None if task is not None: task.cancel() self._false_send = asyncio.ensure_future(self._safe_report(False)) return task + def _reset_detail(self) -> None: + self._ordinary = REASONING + self._manual_wait = False + async def _finish_false_send(self) -> None: send = self._false_send if send is None: @@ -176,13 +247,17 @@ def _cap_reached(self) -> bool: return elapsed >= self._max_working_state_seconds async def _safe_report(self, working: bool) -> None: - """Report under the serialization lock; never raise.""" async with self._lock: - try: - await self._report(working) - except asyncio.CancelledError: - raise - except Exception: - logger.exception( - "working-state report failed (working=%s); swallowing", working - ) + if working and (not self._active or self._cap_reached()): + return + await self._report_locked(working) + + async def _report_locked(self, working: bool) -> None: + try: + await self._report(working) + except asyncio.CancelledError: + raise + except Exception: + logger.exception( + "working-state report failed (working=%s); swallowing", working + ) diff --git a/tests/activitysupport.py b/tests/activitysupport.py new file mode 100644 index 000000000..eeba99419 --- /dev/null +++ b/tests/activitysupport.py @@ -0,0 +1,25 @@ +"""Isolate the process-wide activity-detail negotiation per test.""" + +from __future__ import annotations + +import pytest + +import band.platform.message_lifecycle as lifecycle +from band.platform.activity import ( + API_VERSION_HEADER, + MINIMUM_ACTIVITY_VERSION, + ActivitySupport, +) + +SUPPORTED_VERSION = ".".join(map(str, MINIMUM_ACTIVITY_VERSION)) + + +def install_activity_support( + monkeypatch: pytest.MonkeyPatch, *, learned: bool +) -> ActivitySupport: + """Give the lifecycle a fresh negotiation, optionally already supported.""" + support = ActivitySupport() + if learned: + support.observe({API_VERSION_HEADER: SUPPORTED_VERSION}) + monkeypatch.setattr(lifecycle, "activity_support", support) + return support diff --git a/tests/e2e/baseline/toolkit/control.py b/tests/e2e/baseline/toolkit/control.py index 70e5c1764..2b5087d92 100644 --- a/tests/e2e/baseline/toolkit/control.py +++ b/tests/e2e/baseline/toolkit/control.py @@ -7,6 +7,8 @@ from collections.abc import AsyncGenerator, Callable from contextlib import asynccontextmanager +from band_rest.types.agent_activity_detail import AgentActivityDetail + import band.runtime from band.client.streaming import ( AgentControlPayload, @@ -195,6 +197,7 @@ async def report_activity( room_id: str, working: bool, *, + activity: AgentActivityDetail | None = None, timeout_seconds: int = 2, on_stopped: Callable[[], None] | None = None, ) -> bool: @@ -205,7 +208,11 @@ def observe_stop() -> None: on_stopped() accepted = await super().report_activity( - room_id, working, timeout_seconds=timeout_seconds, on_stopped=observe_stop + room_id, + working, + activity=activity, + timeout_seconds=timeout_seconds, + on_stopped=observe_stop, ) if accepted and not working and room_id == self.control_room_id: self.control.activity_idle.set() diff --git a/tests/platform/activity/conftest.py b/tests/platform/activity/conftest.py new file mode 100644 index 000000000..f14f18e0e --- /dev/null +++ b/tests/platform/activity/conftest.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +from collections.abc import AsyncIterator + +import pytest +from band_rest import AgentActivityDetail + +from band.platform.activity import ActivitySupport +from band.platform.link import BandLink +from tests.activitysupport import install_activity_support +from tests.platform.activity.kit import DETAIL, ActivityPeer +from tests.runtime.helpers import rest_client_over + + +@pytest.fixture(autouse=True) +def support(monkeypatch: pytest.MonkeyPatch) -> ActivitySupport: + return install_activity_support(monkeypatch, learned=False) + + +@pytest.fixture +def activity_peer() -> ActivityPeer: + return ActivityPeer() + + +@pytest.fixture +def detail() -> AgentActivityDetail: + return AgentActivityDetail(**DETAIL) + + +@pytest.fixture +async def link(activity_peer: ActivityPeer) -> AsyncIterator[BandLink]: + """A link on the SDK's default (SaaS) ``rest_url``.""" + async with rest_client_over(activity_peer.respond) as rest: + link = BandLink(agent_id="agent", api_key="test-key") + link.rest = rest + yield link + + +@pytest.fixture +def second_link(link: BandLink) -> BandLink: + other = BandLink(agent_id="second-agent", api_key="test-key") + other.rest = link.rest + return other + + +@pytest.fixture +def learned(monkeypatch: pytest.MonkeyPatch) -> ActivitySupport: + """A process that already saw a supporting ``X-API-Version``.""" + return install_activity_support(monkeypatch, learned=True) diff --git a/tests/platform/activity/kit.py b/tests/platform/activity/kit.py new file mode 100644 index 000000000..9eae53c17 --- /dev/null +++ b/tests/platform/activity/kit.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import asyncio +import json +from collections import deque +from typing import Any + +import httpx + +from band.platform.activity import API_VERSION_HEADER +from tests.activitysupport import SUPPORTED_VERSION + +DETAIL = {"kind": "tool_call", "name": "Bash"} + + +class ActivityPeer: + """The activity endpoint, advertising ``version`` on every response.""" + + def __init__(self) -> None: + self.version: str | None = SUPPORTED_VERSION + self.reports: list[dict[str, Any]] = [] + self.requests: list[httpx.Request] = [] + self.responses: deque[httpx.Response] = deque() + self.stall_post_number: int | None = None + + async def respond(self, request: httpx.Request) -> httpx.Response: + body = json.loads(request.content) + self.requests.append(request) + self.reports.append(body) + if len(self.reports) == self.stall_post_number: + await asyncio.Event().wait() + response = ( + self.responses.popleft() + if self.responses + else httpx.Response(200, json={"data": {"working": body["working"]}}) + ) + if self.version is not None: + response.headers[API_VERSION_HEADER] = self.version + return response diff --git a/tests/platform/activity/test_control_link.py b/tests/platform/activity/test_control_link.py new file mode 100644 index 000000000..008781d86 --- /dev/null +++ b/tests/platform/activity/test_control_link.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import pytest +from band_rest import AgentActivityDetail + +from band.platform.link import BandLink +from tests.e2e.baseline.toolkit.control import ControlLink, ControlRuntime +from tests.platform.activity.kit import DETAIL, ActivityPeer + + +@pytest.mark.usefixtures("learned") +class TestControlLinkActivity: + async def test_baseline_control_link_forwards_activity_detail( + self, link: BandLink, activity_peer: ActivityPeer, detail: AgentActivityDetail + ) -> None: + control_link = ControlLink(agent_id="agent", api_key="test-key") + control_link.rest = link.rest + control_link.control = ControlRuntime() + control_link.control_room_id = "room" + assert await control_link.report_activity("room", True, activity=detail) + assert activity_peer.reports == [{"working": True, "activity": DETAIL}] diff --git a/tests/platform/activity/test_deadlines.py b/tests/platform/activity/test_deadlines.py new file mode 100644 index 000000000..330c5e25c --- /dev/null +++ b/tests/platform/activity/test_deadlines.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +import asyncio + +import httpx +import pytest +from band_rest import AgentActivityDetail + +from band.platform.link import BandLink +from tests.platform.activity.kit import DETAIL, ActivityPeer + + +@pytest.mark.looptime +@pytest.mark.usefixtures("learned") +class TestActivityDeadline: + async def test_retry_cannot_extend_total_deadline_and_downgrade_survives_timeout( + self, link: BandLink, activity_peer: ActivityPeer, detail: AgentActivityDetail + ) -> None: + activity_peer.responses.append(httpx.Response(422)) + activity_peer.stall_post_number = 2 + started = asyncio.get_running_loop().time() + assert not await link.report_activity( + "room", True, activity=detail, timeout_seconds=2 + ) + assert asyncio.get_running_loop().time() - started <= 2.01 + assert activity_peer.reports == [ + {"working": True, "activity": DETAIL}, + {"working": True}, + ] + activity_peer.stall_post_number = None + assert await link.report_activity("next", True, activity=detail) + assert activity_peer.reports[-1] == {"working": True} diff --git a/tests/platform/activity/test_fallback.py b/tests/platform/activity/test_fallback.py new file mode 100644 index 000000000..28313ee62 --- /dev/null +++ b/tests/platform/activity/test_fallback.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import logging +from unittest.mock import Mock + +import httpx +import pytest +from band_rest import AgentActivityDetail + +from band.platform.activity import ActivitySupport +from band.platform.link import BandLink +from tests.platform.activity.kit import DETAIL, ActivityPeer + + +@pytest.mark.usefixtures("learned") +class TestActivityFallback: + @pytest.mark.parametrize( + "body", [b'{"errors":{"activity":["unknown"]}}', b"not json", b""] + ) + async def test_422_retries_without_detail_and_latches_across_links( + self, + link: BandLink, + second_link: BandLink, + activity_peer: ActivityPeer, + detail: AgentActivityDetail, + body: bytes, + ) -> None: + activity_peer.responses.append(httpx.Response(422, content=body)) + assert await link.report_activity("first", True, activity=detail) + assert await second_link.report_activity("second", True, activity=detail) + assert activity_peer.reports == [ + {"working": True, "activity": DETAIL}, + {"working": True}, + {"working": True}, + ] + + @pytest.mark.parametrize( + ("working", "with_detail"), + [(True, False), (False, True)], + ids=["no-detail", "idle"], + ) + async def test_422_without_detail_neither_retries_nor_latches( + self, + link: BandLink, + activity_peer: ActivityPeer, + learned: ActivitySupport, + detail: AgentActivityDetail, + working: bool, + with_detail: bool, + ) -> None: + activity_peer.responses.append(httpx.Response(422)) + activity = detail if with_detail else None + assert not await link.report_activity("room", working, activity=activity) + assert len(activity_peer.reports) == 1 + assert learned.supported + + async def test_downgrade_logs_the_rejected_field_not_the_body( + self, + link: BandLink, + activity_peer: ActivityPeer, + detail: AgentActivityDetail, + caplog: pytest.LogCaptureFixture, + ) -> None: + activity_peer.responses.append( + httpx.Response( + 422, + json={ + "error": { + "code": "validation_error", + "details": {"/activity/name": ["echoed Bash --secret"]}, + "message": "Invalid request", + "request_id": "request-1", + } + }, + ) + ) + with caplog.at_level(logging.DEBUG, logger="band.platform"): + assert await link.report_activity("room", True, activity=detail) + [record] = [r for r in caplog.records if r.levelno == logging.INFO] + assert "/activity/name" in record.getMessage() + assert "secret" not in record.getMessage() + assert any( + r.levelno == logging.DEBUG and "secret" in r.getMessage() + for r in caplog.records + ) + + @pytest.mark.parametrize("status", [400, 403, 500]) + async def test_other_errors_do_not_retry_or_disable( + self, + link: BandLink, + activity_peer: ActivityPeer, + detail: AgentActivityDetail, + status: int, + ) -> None: + activity_peer.responses.append( + httpx.Response(status, json={"error": "failure"}) + ) + assert not await link.report_activity("room", True, activity=detail) + assert await link.report_activity("room", True, activity=detail) + assert activity_peer.reports == [{"working": True, "activity": DETAIL}] * 2 + + async def test_fallback_echoed_false_keeps_stop_notification( + self, link: BandLink, activity_peer: ActivityPeer, detail: AgentActivityDetail + ) -> None: + activity_peer.responses.extend( + [ + httpx.Response(422), + httpx.Response(200, json={"data": {"working": False}}), + ] + ) + stopped = Mock() + assert await link.report_activity( + "room", True, activity=detail, on_stopped=stopped + ) + stopped.assert_called_once_with() + assert activity_peer.reports[-1] == {"working": True} diff --git a/tests/platform/activity/test_reporting.py b/tests/platform/activity/test_reporting.py new file mode 100644 index 000000000..bfc8bb51f --- /dev/null +++ b/tests/platform/activity/test_reporting.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import logging + +import httpx +import pytest + +from band.platform.link import BandLink +from tests.platform.activity.kit import ActivityPeer + + +class TestActivityReporting: + @pytest.mark.parametrize("working", [True, False]) + async def test_reports_the_boolean_state_to_the_room( + self, link: BandLink, activity_peer: ActivityPeer, working: bool + ) -> None: + assert await link.report_activity("room-1", working) + assert activity_peer.reports == [{"working": working}] + assert activity_peer.requests[-1].url.path.endswith("/chats/room-1/activity") + + async def test_post_is_bounded_and_never_retried( + self, link: BandLink, activity_peer: ActivityPeer + ) -> None: + activity_peer.responses.append(httpx.Response(500)) + assert not await link.report_activity("room-1", True, timeout_seconds=2) + [request] = activity_peer.requests + assert request.extensions["timeout"]["read"] == 2 + + async def test_repeated_failures_warn_once_then_recover( + self, + link: BandLink, + activity_peer: ActivityPeer, + caplog: pytest.LogCaptureFixture, + ) -> None: + activity_peer.responses.extend([httpx.Response(500), httpx.Response(500)]) + with caplog.at_level(logging.INFO, logger="band.platform.message_lifecycle"): + assert not await link.report_activity("room-1", True) + assert not await link.report_activity("room-1", True) + assert await link.report_activity("room-1", True) + assert [record.levelno for record in caplog.records] == [ + logging.WARNING, + logging.INFO, + ] + assert "recovered" in caplog.records[-1].getMessage() diff --git a/tests/platform/activity/test_versions.py b/tests/platform/activity/test_versions.py new file mode 100644 index 000000000..4274f23d9 --- /dev/null +++ b/tests/platform/activity/test_versions.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import logging + +import pytest +from band_rest import AgentActivityDetail + +from band.platform.link import BandLink +from tests.activitysupport import SUPPORTED_VERSION +from tests.platform.activity.kit import DETAIL, ActivityPeer + + +class TestActivityVersion: + @pytest.mark.parametrize( + ("version", "supported"), + [ + ("1.24.99", False), + pytest.param(SUPPORTED_VERSION, True, id="minimum-version-on-saas-default"), + ("1.26.10", True), + ("2.0.0", True), + ("1.25.0-beta", False), + ("broken", False), + ("1.25", False), + ("2.0", False), + (None, False), + ], + ) + async def test_advertised_version_controls_later_detail( + self, + link: BandLink, + activity_peer: ActivityPeer, + detail: AgentActivityDetail, + version: str | None, + supported: bool, + ) -> None: + activity_peer.version = version + assert await link.report_activity("room", True, activity=detail) + assert await link.report_activity("room", True, activity=detail) + later = ( + {"working": True, "activity": DETAIL} if supported else {"working": True} + ) + assert activity_peer.reports == [{"working": True}, later] + assert all( + request.url.path.endswith("/activity") for request in activity_peer.requests + ) + + async def test_learned_support_is_shared_across_links( + self, + link: BandLink, + second_link: BandLink, + activity_peer: ActivityPeer, + detail: AgentActivityDetail, + ) -> None: + assert await link.report_activity("first", True) + assert await second_link.report_activity("second", True, activity=detail) + assert activity_peer.reports[-1] == {"working": True, "activity": DETAIL} + + @pytest.mark.parametrize("kind", ["reasoning", "memory", "permission"]) + async def test_non_tool_detail_omits_name_and_false_omits_detail( + self, link: BandLink, activity_peer: ActivityPeer, kind: str + ) -> None: + detail = AgentActivityDetail(kind=kind) + assert await link.report_activity("room", True) + assert await link.report_activity("room", True, activity=detail) + assert await link.report_activity("room", False, activity=detail) + assert activity_peer.reports[1:] == [ + {"working": True, "activity": {"kind": kind}}, + {"working": False}, + ] + + async def test_unadvertised_version_is_logged_for_diagnosis( + self, + link: BandLink, + activity_peer: ActivityPeer, + caplog: pytest.LogCaptureFixture, + ) -> None: + activity_peer.version = "1.24.0" + with caplog.at_level(logging.DEBUG, logger="band.platform.activity"): + assert await link.report_activity("room", True) + [record] = caplog.records + assert record.levelno == logging.DEBUG + assert "'1.24.0'" in record.getMessage() diff --git a/tests/platform/test_link.py b/tests/platform/test_link.py index d6401a71d..8a17eeb79 100644 --- a/tests/platform/test_link.py +++ b/tests/platform/test_link.py @@ -3,16 +3,9 @@ from __future__ import annotations import asyncio -import logging from unittest.mock import ANY, AsyncMock, MagicMock, create_autospec, patch import pytest -from band_rest import ( - AsyncRestClient, - NotFoundError, - UnauthorizedError, - UnprocessableEntityError, -) from band_rest.core.api_error import ApiError from band_rest.types.chat_message_metadata import ChatMessageMetadata from band_sdk_core import AgentTopicStatus, DeadReason, SessionState, chat_room_topic @@ -1346,135 +1339,3 @@ async def test_returns_empty_list_on_failure(self): messages = await link.get_stale_processing_messages("room-1") assert messages == [] - - -class TestReportActivity: - """Tests for BandLink.report_activity (boolean working-state reporting).""" - - @pytest.mark.asyncio - async def test_reports_working_true(self): - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock() - - result = await link.report_activity("room-1", True) - - assert result is True - call = link.rest.agent_api_activity.report_agent_chat_activity - call.assert_awaited_once() - assert call.call_args.kwargs["chat_id"] == "room-1" - assert call.call_args.kwargs["working"] is True - - @pytest.mark.asyncio - async def test_passes_per_post_timeout_and_no_retries(self): - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock() - - await link.report_activity("room-1", True, timeout_seconds=2) - - opts = link.rest.agent_api_activity.report_agent_chat_activity.call_args.kwargs[ - "request_options" - ] - assert opts["timeout_in_seconds"] == 2 - assert opts["max_retries"] == 0 - - @pytest.mark.asyncio - async def test_reports_working_false(self): - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock() - - result = await link.report_activity("room-1", False) - - assert result is True - call = link.rest.agent_api_activity.report_agent_chat_activity - assert call.call_args.kwargs["working"] is False - - @pytest.mark.asyncio - async def test_returns_false_on_not_found(self): - - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock( - side_effect=NotFoundError(headers={}, body="no active execution") - ) - - result = await link.report_activity("room-1", True) - - assert result is False - - @pytest.mark.asyncio - async def test_returns_false_on_unauthorized(self): - - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock( - side_effect=UnauthorizedError(headers={}, body="bad key") - ) - - result = await link.report_activity("room-1", True) - - assert result is False - - @pytest.mark.asyncio - async def test_returns_false_on_unprocessable_entity(self): - - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock( - side_effect=UnprocessableEntityError(headers={}, body="bad uuid") - ) - - result = await link.report_activity("room-1", True) - - assert result is False - - @pytest.mark.asyncio - async def test_returns_false_on_network_error(self): - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock( - side_effect=Exception("network down") - ) - - result = await link.report_activity("room-1", True) - - assert result is False - - def test_real_client_exposes_activity_method(self): - """Guard: the real REST client must actually expose the activity method. - - The AsyncMock-based tests above auto-fabricate attributes, so they would - stay green even if `agent_api_activity.report_agent_chat_activity` - disappeared or was renamed in a band_rest bump. This test pins the - real wire contract: instantiate the real client and assert the method - exists and is callable. - """ - - client = AsyncRestClient(api_key="test-key", base_url="https://test.com") - method = getattr(client.agent_api_activity, "report_agent_chat_activity", None) - assert callable(method) - - @pytest.mark.asyncio - async def test_repeated_failures_warn_once_then_recover(self, caplog): - - link = BandLink(agent_id="agent-123", api_key="test-key") - link.rest = MagicMock() - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock( - side_effect=Exception("down") - ) - - with caplog.at_level(logging.WARNING, logger="band.platform.message_lifecycle"): - assert await link.report_activity("room-1", True) is False - assert await link.report_activity("room-1", True) is False - - warnings = [r for r in caplog.records if r.levelno == logging.WARNING] - assert len(warnings) == 1 # debounced: only the first failure warns - - # Recovery logs once at INFO and re-arms the warning. - link.rest.agent_api_activity.report_agent_chat_activity = AsyncMock() - with caplog.at_level(logging.INFO, logger="band.platform.message_lifecycle"): - caplog.clear() - assert await link.report_activity("room-1", True) is True - assert any("recovered" in r.message.lower() for r in caplog.records) diff --git a/tests/runtime/activityownership/conftest.py b/tests/runtime/activityownership/conftest.py index 16bcf6bfc..4b2e227bd 100644 --- a/tests/runtime/activityownership/conftest.py +++ b/tests/runtime/activityownership/conftest.py @@ -14,25 +14,25 @@ from tests.runtime.helpers import ( AGENT_ID, ROOM_ID, - LifecyclePlatform, rest_client_over, ) @pytest.fixture -async def activity_platform() -> AsyncIterator[LifecyclePlatform]: - yield ActivityPlatform() +def activity_platform() -> ActivityPlatform: + return ActivityPlatform() @pytest.fixture async def execution( - activity_platform: LifecyclePlatform, + activity_platform: ActivityPlatform, ) -> AsyncIterator[Callable[..., ExecutionContext]]: contexts: list[ExecutionContext] = [] - async with rest_client_over(activity_platform.answer) as rest: + async with rest_client_over(activity_platform.respond) as rest: def create( handler: Callable[[ExecutionContext, PlatformEvent], Awaitable[None]], + hub_room_id: str | None = None, **settings: object, ) -> ExecutionContext: link = BandLink(agent_id=AGENT_ID, api_key="test-key") @@ -45,6 +45,7 @@ def create( link, handler, agent_id=AGENT_ID, + hub_room_id=hub_room_id, on_platform_stop=runtime._on_platform_stop, config=SessionConfig(enable_context_hydration=False, **settings), ) @@ -53,4 +54,5 @@ def create( yield create for ctx in contexts: + activity_platform.activity_release.set() await ctx.stop() diff --git a/tests/runtime/activityownership/kit.py b/tests/runtime/activityownership/kit.py index d43a5b921..941a3822a 100644 --- a/tests/runtime/activityownership/kit.py +++ b/tests/runtime/activityownership/kit.py @@ -1,13 +1,34 @@ from __future__ import annotations +import asyncio +import json +from typing import Any, Literal + import httpx from tests.runtime.helpers import ALICE, LifecyclePlatform, page class ActivityPlatform(LifecyclePlatform): + def __init__(self) -> None: + super().__init__() + self.activity_bodies: list[dict[str, Any]] = [] + self.hold_next_activity = False + self.activity_entered = asyncio.Event() + self.activity_release = asyncio.Event() + + async def respond(self, request: httpx.Request) -> httpx.Response: + response = self.answer(request) + if self.hold_next_activity and request.url.path.endswith("/activity"): + self.hold_next_activity = False + self.activity_entered.set() + await self.activity_release.wait() + return response + def answer(self, request: httpx.Request) -> httpx.Response: tail = request.url.path.rsplit("/", 1)[-1] + if tail == "activity": + self.activity_bodies.append(json.loads(request.content)) if tail == "participants": return page([{**ALICE, "role": "member", "status": "active"}]) if request.method == "POST" and tail == "messages": @@ -15,3 +36,16 @@ def answer(self, request: httpx.Request) -> httpx.Response: 201, json={"data": {"id": "m-1", "success": True, "recipients": []}} ) return super().answer(request) + + +class ObservedIdle(asyncio.Event): + """The idle event a graceful stop waits on, announcing that wait.""" + + def __init__(self) -> None: + super().__init__() + self.set() + self.awaited = asyncio.Event() + + async def wait(self) -> Literal[True]: + self.awaited.set() + return await super().wait() diff --git a/tests/runtime/activityownership/test_detail.py b/tests/runtime/activityownership/test_detail.py new file mode 100644 index 000000000..564a0298c --- /dev/null +++ b/tests/runtime/activityownership/test_detail.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Callable + +import pytest + +from band.core.turn_lifecycle import TurnTask, bind_turn_task +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from band.runtime.working_state import ( + end_tool_activity, + report_memory_activity, + report_tool_activity, + set_permission_wait, +) +from tests.conftest import make_message_event +from tests.runtime.activityownership.kit import ActivityPlatform, ObservedIdle +from tests.runtime.helpers import ROOM_ID + + +class TestOwnedDetail: + async def test_real_tools_observer_reports_canonical_wire_detail( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + await report_tool_activity(tools, "Bash") + await set_permission_wait(tools, True) + await end_tool_activity(tools) + await report_memory_activity(tools) + await set_permission_wait(tools, False) + await end_tool_activity(tools) + + ctx = execution(handler) + await ctx._process_event(make_message_event()) + assert activity_platform.activity_bodies == [ + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": True, "activity": {"kind": "tool_call", "name": "Bash"}}, + {"working": True, "activity": {"kind": "permission"}}, + {"working": True, "activity": {"kind": "memory"}}, + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": False}, + ] + + @pytest.mark.looptime + async def test_settled_owner_remains_authoritative_and_transient_tools_are_quiet( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + release = asyncio.Event() + captured: list[AgentTools] = [] + workers: list[TurnTask] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + if captured: + await report_tool_activity(tools, "unrelated") + return + captured.append(tools) + workers.append(bind_turn_task(tools, asyncio.create_task(release.wait()))) + await set_permission_wait(tools, True) + + ctx = execution(handler) + await ctx._process_event(make_message_event(msg_id="original")) + await ctx._process_event(make_message_event(msg_id="busy")) + await asyncio.sleep(6.1) + assert ( + activity_platform.activity_bodies[-2:] + == [{"working": True, "activity": {"kind": "permission"}}] * 2 + ) + await report_tool_activity(captured[0], "Bash") + await set_permission_wait(captured[0], False) + assert activity_platform.activity_bodies[-1] == { + "working": True, + "activity": {"kind": "tool_call", "name": "Bash"}, + } + release.set() + await workers[0].task + await asyncio.sleep(0) + await ctx._join_activity_completion() + assert activity_platform.activity_bodies[-1] == {"working": False} + before = list(activity_platform.activity_bodies) + await end_tool_activity(captured[0]) + assert activity_platform.activity_bodies == before + + async def test_settled_turn_cannot_restart_reporting( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + captured: list[AgentTools] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + captured.append(AgentTools.from_context(ctx)) + + ctx = execution(handler) + await ctx._process_event(make_message_event()) + await ctx._join_activity_completion() + before = list(activity_platform.activity_bodies) + await report_tool_activity(captured[0], "Bash") + assert before[-1] == {"working": False} + assert activity_platform.activity_bodies == before + + async def test_graceful_stop_drops_observations_from_the_draining_turn( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + captured: list[AgentTools] = [] + handler_started = asyncio.Event() + release = asyncio.Event() + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + captured.append(AgentTools.from_context(ctx)) + handler_started.set() + await release.wait() + + ctx = execution(handler) + idle = ctx._idle_event = ObservedIdle() + await ctx.start() + await ctx.on_event(make_message_event()) + await handler_started.wait() + stopping = asyncio.create_task(ctx.stop(timeout=30)) + await idle.awaited.wait() + await report_tool_activity(captured[0], "Bash") + release.set() + assert await stopping + assert activity_platform.activity_bodies == [ + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": False}, + ] + + async def test_queued_observation_is_rechecked_after_stop_and_replacement( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + captured: list[AgentTools] = [] + handler_started = asyncio.Event() + release = asyncio.Event() + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + captured.append(AgentTools.from_context(ctx)) + handler_started.set() + await release.wait() + + ctx = execution(handler) + turn = asyncio.create_task(ctx._process_event(make_message_event(msg_id="old"))) + await handler_started.wait() + activity_platform.hold_next_activity = True + change = asyncio.create_task(report_tool_activity(captured[0], "Bash")) + await activity_platform.activity_entered.wait() + stale = asyncio.create_task(report_memory_activity(captured[0])) + ctx.interrupt() + activity_platform.activity_release.set() + await asyncio.gather(turn, change, stale) + release.set() + await ctx._process_event(make_message_event(msg_id="new")) + assert activity_platform.activity_bodies[-3:] == [ + {"working": False}, + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": False}, + ] + assert not any( + body.get("activity", {}).get("kind") == "memory" + for body in activity_platform.activity_bodies + ) + + @pytest.mark.looptime + async def test_waiting_delivery_takes_over_reporting_once_the_owner_finishes( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + worker_release = asyncio.Event() + waiting_tools: list[AgentTools] = [] + waiting_started = asyncio.Event() + waiting_release = asyncio.Event() + workers: list[TurnTask] = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + if workers: + waiting_tools.append(tools) + waiting_started.set() + await waiting_release.wait() + return + workers.append( + bind_turn_task(tools, asyncio.create_task(worker_release.wait())) + ) + + ctx = execution(handler) + await ctx._process_event(make_message_event(msg_id="owner")) + waiting = asyncio.create_task( + ctx._process_event(make_message_event(msg_id="waiting")) + ) + await waiting_started.wait() + worker_release.set() + await workers[0].task + await ctx._join_activity_completion() + await report_tool_activity(waiting_tools[0], "Bash") + assert activity_platform.activity_bodies[-3:] == [ + {"working": False}, + {"working": True, "activity": {"kind": "reasoning"}}, + {"working": True, "activity": {"kind": "tool_call", "name": "Bash"}}, + ] + waiting_release.set() + await waiting + + @pytest.mark.parametrize("excluded", ["disabled", "hub"]) + async def test_excluded_reporting_ignores_observations( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + excluded: str, + ) -> None: + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + await report_tool_activity(tools, "Bash") + await set_permission_wait(tools, True) + await report_memory_activity(tools) + await end_tool_activity(tools) + + settings = ( + {"enable_working_state": False} + if excluded == "disabled" + else {"hub_room_id": ROOM_ID} + ) + ctx = execution(handler, **settings) + await ctx._process_event(make_message_event()) + assert not activity_platform.activity_bodies diff --git a/tests/runtime/conftest.py b/tests/runtime/conftest.py index 46f8a06ed..f7457f7ed 100644 --- a/tests/runtime/conftest.py +++ b/tests/runtime/conftest.py @@ -34,6 +34,7 @@ from band.core.types import PlatformConnection, PlatformMessage from band.runtime.presence import RoomPresence +from tests.activitysupport import install_activity_support from tests.conftest import make_participant_mock from tests.runtime.helpers import AGENT_ID @@ -259,3 +260,9 @@ def link(mock_rest_client: MagicMock) -> MagicMock: link.get_stale_processing_messages = AsyncMock(return_value=[]) link.report_activity = AsyncMock(return_value=True) return link + + +@pytest.fixture(autouse=True) +def runtime_activity_support(monkeypatch: pytest.MonkeyPatch) -> None: + """Start every runtime test on a platform that already advertised detail.""" + install_activity_support(monkeypatch, learned=True) diff --git a/tests/runtime/execution/test_crashrecoverysync.py b/tests/runtime/execution/test_crashrecoverysync.py index c2d6bc739..b779bf812 100644 --- a/tests/runtime/execution/test_crashrecoverysync.py +++ b/tests/runtime/execution/test_crashrecoverysync.py @@ -2,12 +2,14 @@ import asyncio from datetime import UTC, datetime -from unittest.mock import AsyncMock, MagicMock +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, call import pytest from band_sdk_core import ClaimRegistry from band.client.streaming import MessageMetadata +from band.runtime import resync from band.runtime.execution import ( ExecutionContext, ResyncOutcome, @@ -1035,9 +1037,14 @@ async def test_refused_claims_never_spend_the_retry_budget( mock_handler.assert_awaited_once() assert not ctx._retry_tracker.is_permanently_failed(msg.id) + @pytest.mark.parametrize("initial_recovery_delay", [0.0, 10.0]) async def test_startup_backlog_claim_failure_does_not_process_newer_ws_event( - self, mock_link_with_next, mock_handler - ): + self, + mock_link_with_next, + mock_handler, + monkeypatch: pytest.MonkeyPatch, + initial_recovery_delay: float, + ) -> None: """Startup sync should not switch to WebSocket after an unclaimable backlog message.""" backlog_msg = PlatformMessage( @@ -1063,6 +1070,12 @@ async def test_startup_backlog_claim_failure_does_not_process_newer_ws_event( ), ) + monkeypatch.setattr( + resync, + "random", + SimpleNamespace(uniform=lambda lower, upper: initial_recovery_delay), + ) + await ctx.on_event( make_message_event(room_id="room-123", msg_id="msg-newer-ws") ) @@ -1071,8 +1084,10 @@ async def test_startup_backlog_claim_failure_does_not_process_newer_ws_event( lambda: mock_link_with_next.mark_processing.await_count >= 1 ) - mock_link_with_next.mark_processing.assert_awaited_once_with( - "room-123", "msg-older-claim-fails" + assert ( + mock_link_with_next.mark_processing.await_args_list + == [call("room-123", "msg-older-claim-fails")] + * mock_link_with_next.mark_processing.await_count ) mock_handler.assert_not_called() assert ctx._sync_complete is False diff --git a/tests/runtime/workingstate/test_detail.py b/tests/runtime/workingstate/test_detail.py new file mode 100644 index 000000000..6cecd58ec --- /dev/null +++ b/tests/runtime/workingstate/test_detail.py @@ -0,0 +1,189 @@ +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest +from band_rest import AgentActivityDetail + +from band.runtime.working_state import WorkingStateReporter + + +@pytest.mark.looptime +class TestActivityDetail: + async def test_changes_repeat_current_detail_and_keep_boolean_callback( + self, + ) -> None: + reports: list[tuple[bool, dict[str, Any] | None]] = [] + + async def report(working: bool) -> None: + reports.append( + ( + working, + reporter.activity.model_dump(exclude_unset=True) + if working + else None, + ) + ) + + reporter = WorkingStateReporter(report) + try: + await reporter.start() + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail( + kind="tool_call", name="Bash", arguments="private" + ), + ) + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail(kind="tool_call", name="Bash"), + ) + await asyncio.sleep(3.1) + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail( + kind="reasoning", name="private", output="private" + ), + ) + finally: + await reporter.stop() + assert reports == [ + (True, {"kind": "reasoning"}), + (True, {"kind": "tool_call", "name": "Bash"}), + (True, {"kind": "tool_call", "name": "Bash"}), + (True, {"kind": "reasoning"}), + (False, None), + ] + + async def test_manual_wait_overrides_latest_progress_without_restoring_stale_state( + self, + ) -> None: + reports: list[str] = [] + + async def report(working: bool) -> None: + if working: + reports.append(reporter.activity.kind) + + reporter = WorkingStateReporter(report) + try: + await reporter.start() + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail(kind="tool_call", name="Read"), + ) + await reporter.update(current=lambda: True, waiting=True) + await reporter.update( + current=lambda: True, activity=AgentActivityDetail(kind="reasoning") + ) + await reporter.update(current=lambda: True, waiting=True) + await reporter.update( + current=lambda: True, activity=AgentActivityDetail(kind="memory") + ) + await reporter.update(current=lambda: True, waiting=False) + finally: + await reporter.stop() + assert reports == ["reasoning", "tool_call", "permission", "memory"] + + async def test_cap_prevents_detail_from_reviving_the_indicator(self) -> None: + reports: list[bool] = [] + + async def report(working: bool) -> None: + reports.append(working) + + reporter = WorkingStateReporter(report, max_working_state_seconds=2) + try: + await reporter.start() + await asyncio.sleep(2.1) + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail(kind="tool_call", name="Bash"), + ) + await asyncio.sleep(3) + assert reports == [True] + finally: + await reporter.stop() + assert reports == [True, False] + + async def test_observation_queued_behind_a_report_is_rechecked_under_the_lock( + self, + ) -> None: + entered, release = asyncio.Event(), asyncio.Event() + reports: list[str] = [] + + async def report(working: bool) -> None: + if working: + reports.append(reporter.activity.kind) + if reporter.activity.kind == "tool_call": + entered.set() + await release.wait() + + reporter = WorkingStateReporter(report) + owned = True + try: + await reporter.start() + change = asyncio.create_task( + reporter.update( + current=lambda: owned, + activity=AgentActivityDetail(kind="tool_call", name="Bash"), + ) + ) + await entered.wait() + stale = asyncio.create_task( + reporter.update( + current=lambda: owned, activity=AgentActivityDetail(kind="memory") + ) + ) + owned = False + release.set() + await asyncio.gather(change, stale) + finally: + await reporter.stop() + assert reports == ["reasoning", "tool_call"] + + async def test_empty_tool_name_reports_unknown(self) -> None: + reports: list[dict[str, Any]] = [] + + async def report(working: bool) -> None: + if working: + reports.append(reporter.activity.model_dump(exclude_unset=True)) + + reporter = WorkingStateReporter(report) + try: + await reporter.start() + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail(kind="tool_call", name=""), + ) + finally: + await reporter.stop() + assert reports[-1] == {"kind": "tool_call", "name": "unknown"} + + async def test_observation_outside_a_cycle_reports_nothing(self) -> None: + reports: list[bool] = [] + + async def report(working: bool) -> None: + reports.append(working) + + reporter = WorkingStateReporter(report) + await reporter.update( + current=lambda: True, + activity=AgentActivityDetail(kind="tool_call", name="Bash"), + ) + assert reports == [] + + async def test_permission_wait_does_not_carry_into_the_next_cycle(self) -> None: + reports: list[str | None] = [] + + async def report(working: bool) -> None: + reports.append(reporter.activity.kind if working else None) + + reporter = WorkingStateReporter(report) + try: + await reporter.start() + await reporter.update(current=lambda: True, waiting=True) + await reporter.stop() + await reporter.start() + finally: + await reporter.stop() + assert reports == ["reasoning", "permission", None, "reasoning", None] From ef5d6b950632d3be8724c4d3e85777b7a9cd844a Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Tue, 20 Oct 2026 23:18:05 +0300 Subject: [PATCH 44/47] fix(activity): clip over-long tool names before the platform rejects them A name longer than 1024 characters is a 422 on the activity endpoint, and the compatibility path reads a detail-bearing 422 as an old platform: one oversized tool name would latch activity detail off for the whole process. Clip at the serialization chokepoint instead, one constant, so a long name costs characters and nothing else. Guarded by tests/platform/activity/test_truncation.py (over-limit clipped, exact-limit untouched). --- src/band/platform/message_lifecycle.py | 10 ++++++ tests/platform/activity/test_truncation.py | 42 ++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 tests/platform/activity/test_truncation.py diff --git a/src/band/platform/message_lifecycle.py b/src/band/platform/message_lifecycle.py index cb17ad193..2450cfe35 100644 --- a/src/band/platform/message_lifecycle.py +++ b/src/band/platform/message_lifecycle.py @@ -54,6 +54,10 @@ PROCESSING_ATTEMPT_POINTER = "/processing_attempt" NOT_FOUND_ERROR_CODE = "not_found" +# A tool name over this length is a 422 on the activity endpoint, which the +# compatibility path would read as an old platform; clip first, never reject. +MAX_ACTIVITY_NAME_CHARACTERS = 1024 + def _accepted_response( response: AsyncHttpResponse[ResponseData | None], @@ -155,6 +159,12 @@ async def _send_activity( ) -> AsyncHttpResponse[ReportAgentChatActivityResponse]: kwargs: dict[str, Any] = {} if activity is not None: + if activity.name and len(activity.name) > MAX_ACTIVITY_NAME_CHARACTERS: + # I-08: clip before the POST, never let a long name read as an + # old platform (a detail-bearing 422 latches detail off). + activity = activity.model_copy( + update={"name": activity.name[:MAX_ACTIVITY_NAME_CHARACTERS]} + ) kwargs["activity"] = activity return await rest.agent_api_activity.with_raw_response.report_agent_chat_activity( chat_id=room_id, diff --git a/tests/platform/activity/test_truncation.py b/tests/platform/activity/test_truncation.py new file mode 100644 index 000000000..93d83d290 --- /dev/null +++ b/tests/platform/activity/test_truncation.py @@ -0,0 +1,42 @@ +"""I-08: a tool name too long for the platform is truncated, never rejected.""" + +from __future__ import annotations + +import pytest +from band_rest import AgentActivityDetail + +from band.platform.link import BandLink +from tests.platform.activity.kit import ActivityPeer + +LIMIT = 1024 + + +@pytest.mark.usefixtures("learned") +class TestNameTruncation: + async def test_over_long_name_is_clipped_before_the_post( + self, + link: BandLink, + activity_peer: ActivityPeer, + ) -> None: + long_name = "a" * (LIMIT + 16) + sent = await link.report_activity( + "room", + True, + activity=AgentActivityDetail(kind="tool_call", name=long_name), + ) + assert sent + assert activity_peer.reports == [ + { + "working": True, + "activity": {"kind": "tool_call", "name": "a" * LIMIT}, + } + ] + + async def test_exact_limit_name_is_untouched( + self, + link: BandLink, + activity_peer: ActivityPeer, + ) -> None: + detail = AgentActivityDetail(kind="tool_call", name="b" * LIMIT) + assert await link.report_activity("room", True, activity=detail) + assert activity_peer.reports[0]["activity"]["name"] == "b" * LIMIT From a476c3a263620b3ae0eed22bd26aadc0e1139995 Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Wed, 21 Oct 2026 02:27:45 +0300 Subject: [PATCH 45/47] test: Accept room event posts in the activity peer The lifecycle peer rejects narration posts with the stopped-execution answer, so the first reported tool call read as a platform stop and tore down the turn machinery around a parked approval. A running room echoes 201 like the posting peer before it; stopped peers keep rejecting. --- tests/runtime/activityownership/kit.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/runtime/activityownership/kit.py b/tests/runtime/activityownership/kit.py index 7e2aa3db5..3661bc2a4 100644 --- a/tests/runtime/activityownership/kit.py +++ b/tests/runtime/activityownership/kit.py @@ -55,6 +55,21 @@ def answer(self, request: httpx.Request) -> httpx.Response: tail = request.url.path.rsplit("/", 1)[-1] if tail == "events" and request.method == "POST": self.event_bodies.append(json.loads(request.content)) + if not self.stopped: + # A running room accepts narration; falling through to the + # lifecycle base would answer 403, which the platform client + # reads as a stopped execution and tears the turn machinery + # down around the first reported tool call. + return httpx.Response( + 201, + json={ + "data": { + "id": "e-1", + "message_type": "event", + "success": True, + } + }, + ) if tail == "activity": body = json.loads(request.content) self.activity_bodies.append(body) From d697f4325318e777e0582a4f1946d91c7cee720a Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Wed, 21 Oct 2026 02:50:37 +0300 Subject: [PATCH 46/47] style: drop merge-seam blank lines from the reporter files --- src/band/runtime/execution.py | 1 - src/band/runtime/working_state.py | 1 - 2 files changed, 2 deletions(-) diff --git a/src/band/runtime/execution.py b/src/band/runtime/execution.py index a615d6bb4..cfb1de69e 100644 --- a/src/band/runtime/execution.py +++ b/src/band/runtime/execution.py @@ -1976,7 +1976,6 @@ async def observe_activity( waiting=waiting, ) - def register_turn_task(self, scope: TurnScope, owner: TurnTask) -> None: """Bind completion to the same owner that delivers worker cancellation.""" scope.bind(owner) diff --git a/src/band/runtime/working_state.py b/src/band/runtime/working_state.py index 9fe6959a9..07114594b 100644 --- a/src/band/runtime/working_state.py +++ b/src/band/runtime/working_state.py @@ -175,7 +175,6 @@ async def update( if self.activity != previous: await self._report_locked(True) - def defer_false_report(self) -> None: """Start ``working: false`` and return without waiting for it. From d764959b57b8553d4b5f9027d1547028c2bd303e Mon Sep 17 00:00:00 2001 From: bandzalkin Date: Wed, 21 Oct 2026 08:12:47 +0300 Subject: [PATCH 47/47] fix(activity): consult the tool-name gate at report time The mask wrapper was attached only when the flag was already off, so an adapter that binds its observer before the emit sync (codex's item/started path) kept reporting named tool_calls after the choice was turned off. The observer now always wraps and reads the live flag on every report; the hook site gates on the OMP branch remain as the second guard. Red-first test: observer bound with names on, choice flipped off mid-turn, tool report must arrive nameless. Co-Authored-By: Claude --- src/band/core/activity.py | 13 +++++++++---- src/band/runtime/execution.py | 12 +++++++----- tests/runtime/activity/test_namegate.py | 21 +++++++++++++++++++++ 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/src/band/core/activity.py b/src/band/core/activity.py index 74bf0f090..5da2a76c0 100644 --- a/src/band/core/activity.py +++ b/src/band/core/activity.py @@ -8,7 +8,7 @@ from __future__ import annotations -from collections.abc import Collection +from collections.abc import Callable, Collection from typing import TYPE_CHECKING from band_rest.types.agent_activity_detail import AgentActivityDetail @@ -23,14 +23,19 @@ class ToolNamesObserver: """Hide tool names while leaving every other activity detail intact. Permission waits and memory activity name no content, so they report - whatever the tool-name choice is. + whatever the tool-name choice is. ``hidden`` is consulted at report time, + never bound at construction: an observer attached before the adapter's + emit sync must still follow the turn's live choice. """ - def __init__(self, inner: ActivityObserver) -> None: + def __init__( + self, inner: ActivityObserver, hidden: Callable[[], bool] = lambda: True + ) -> None: self._inner = inner + self._hidden = hidden async def update(self, activity: AgentActivityDetail) -> None: - if activity.kind == "tool_call": + if activity.kind == "tool_call" and self._hidden(): await self._inner.update(AgentActivityDetail(kind="reasoning")) return await self._inner.update(activity) diff --git a/src/band/runtime/execution.py b/src/band/runtime/execution.py index 23c57dc30..94f9a6f3b 100644 --- a/src/band/runtime/execution.py +++ b/src/band/runtime/execution.py @@ -1954,17 +1954,19 @@ def set_activity_tool_names(self, enabled: bool) -> None: """Gate tool names in room activity to match tool-call narration. Permission waits and memory activity are unaffected: they name no - content. Observers created after a change pick it up immediately. + content. The gate is consulted at report time, so an observer bound + before this change (a continuation's adapter pumping tool events while + the runner re-reads the choice) still follows the live choice. """ self._activity_tool_names = enabled def activity_observer(self, scope: TurnScope | None) -> ActivityObserver | None: if scope is None or not scope.activity_allowed: return None - observer: ActivityObserver = ScopeActivityObserver(self, scope) - if not self._activity_tool_names: - observer = ToolNamesObserver(observer) - return observer + return ToolNamesObserver( + ScopeActivityObserver(self, scope), + hidden=lambda: not self._activity_tool_names, + ) def owns_activity(self, scope: TurnScope) -> bool: return self._activity_owner is scope and not scope.aborted and not self._closing diff --git a/tests/runtime/activity/test_namegate.py b/tests/runtime/activity/test_namegate.py index 3051bcf30..19676192e 100644 --- a/tests/runtime/activity/test_namegate.py +++ b/tests/runtime/activity/test_namegate.py @@ -16,6 +16,7 @@ report_memory_activity, report_tool_activity, set_permission_wait, + tool_call_activity, ) from tests.conftest import make_message_event from tests.runtime.activity.kit import ActivityPeer, activity_context @@ -95,6 +96,26 @@ async def test_permission_and_memory_survive_the_name_gate(self) -> None: assert "memory" in kinds assert "permission" in kinds + async def test_flip_after_binding_hides_names_in_the_running_turn( + self, + ) -> None: + # The codex ordering: the observer is bound to the turn before the + # adapter's emit sync flips the flag. The gate must read the live + # choice at report time, not the choice at construction time. + peer = DetailPeer() + bound: list = [] + + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + bound.append(ctx.activity_observer(ctx.current_scope)) + ctx.set_activity_tool_names(False) + await bound[0].update(tool_call_activity("Bash")) + + async with activity_context(peer, handler, AsyncMock()) as ctx: + await ctx._process_event(make_message_event(msg_id="trigger")) + tool_frames = [d for d in peer.details if d and d.get("kind") == "tool_call"] + assert not tool_frames + assert {"kind": "reasoning"} in peer.details + async def test_sync_resolves_names_from_the_emit_choice(self) -> None: async with activity_context( ActivityPeer(), _reporting_handler(), AsyncMock()