diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c4022c6e..3894a9c18 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -187,6 +187,8 @@ jobs: tests/integrations/test_crewai_flow_real_sdk.py \ tests/integrations/test_crewai_real_tools.py \ tests/integrations/test_crewai_completion.py \ + tests/integrations/crewai/activity/test_execution.py \ + tests/integrations/crewai/activity/test_send_event.py \ tests/test_capability_gating_e2e.py \ tests/framework_conformance/test_files_image_passthrough_matrix.py \ tests/framework_conformance/ -k crewai @@ -325,6 +327,8 @@ jobs: tests/integrations/parlant/test_customtools.py \ tests/integrations/parlant/test_filetools.py \ tests/integrations/parlant/test_guard.py \ + tests/integrations/parlant/activity/test_execution.py \ + tests/integrations/parlant/activity/test_send_event.py \ tests/integrations/parlant/test_preambletask.py \ tests/integrations/parlant/test_selection.py \ tests/integrations/parlant/test_server.py \ diff --git a/docs/turn-outcome.md b/docs/turn-outcome.md index 6be76ef6c..cc9054e25 100644 --- a/docs/turn-outcome.md +++ b/docs/turn-outcome.md @@ -437,6 +437,16 @@ change wakes the reporter's keep-alive, which sends the latest detail, so changes made while a report is in flight coalesce. The duration cap stops both kinds of refresh without cancelling provider work; actual completion still clears. +Every Band tool call shows as a `tool_call` with its name, or `memory` while it +saves a memory, then returns to reasoning when it ends, even on failure or +cancellation. `band.runtime.toolactivity` owns that mapping: wrap any other tool +call in its `tool_activity`. Only the outermost call reports, so a Band tool +that calls other Band tools shows as itself throughout. `send_event` reports +only where a model invoked it, since adapters also post their own events +through it. A tool's end lands on the turn it started on, even if its tools are +rebound to a newer turn meanwhile, and a failed report never changes the tool's +outcome. + Detail compatibility is learned from the `X-API-Version` header the platform puts on every activity response. Reports stay boolean until a response in the process advertises agent API 1.25.0 or later; a missing or older version keeps diff --git a/src/band/adapters/strands.py b/src/band/adapters/strands.py index 809b6eba8..5ebe9e9cd 100644 --- a/src/band/adapters/strands.py +++ b/src/band/adapters/strands.py @@ -66,6 +66,7 @@ reject_conflicting_tool_names, ) from band.runtime.prompts import render_system_prompt, with_system_updates +from band.runtime.toolactivity import tool_activity from band.runtime.tools import ( ALL_TOOL_NAMES, ToolCallOutcome, @@ -354,7 +355,8 @@ async def _execute(self, arguments: dict[str, Any]) -> ToolCallOutcome: Callable[..., Awaitable[object]], getattr(self._tools, self._definition.method_name), ) - value = await method(**validated) + async with tool_activity(self._tools, self._definition.name): + value = await method(**validated) except Exception as error: logger.exception("Platform tool %s failed", self.tool_name) message = f"Error executing {self.tool_name}: {error}" diff --git a/src/band/integrations/claude_sdk/dedup_tools.py b/src/band/integrations/claude_sdk/dedup_tools.py index 70f9716f4..70ca238a7 100644 --- a/src/band/integrations/claude_sdk/dedup_tools.py +++ b/src/band/integrations/claude_sdk/dedup_tools.py @@ -42,6 +42,8 @@ from collections import OrderedDict from typing import TYPE_CHECKING, Any +from band.runtime.activity import ActivityObserver, activity_observer_of + if TYPE_CHECKING: from band.core.protocols import AgentToolsProtocol @@ -199,6 +201,10 @@ async def send_message( # --- transparent passthrough -------------------------------------- + @property + def _activity_observer(self) -> ActivityObserver | None: + return activity_observer_of(self._inner) + def __getattr__(self, name: str) -> Any: # ``__getattr__`` only runs for attributes not found via the # normal MRO. ``send_message`` is defined above, so it never diff --git a/src/band/integrations/crewai/catalog.py b/src/band/integrations/crewai/catalog.py index e33df22f1..573ec6850 100644 --- a/src/band/integrations/crewai/catalog.py +++ b/src/band/integrations/crewai/catalog.py @@ -20,6 +20,7 @@ from band.core.protocols import AgentToolsProtocol from band.core.task_types import TaskAssignmentStatus, TaskLifecycleState, TaskListState from band.integrations.crewai.reporting import CrewAIToolReporter +from band.runtime.toolactivity import tool_activity from band.runtime.tools import ( BandTool, file_content_placeholder, @@ -288,9 +289,10 @@ async def _send_event(call: Invocation, **kwargs: Any) -> Any: # bypassing args_schema, and _execute_tool turns the ValueError into the # error result the caller expects. args = validate_tool_arguments(BandTool.SEND_EVENT, SEND_EVENT_ARGS_SCHEMA, kwargs) - await call.tools.send_event( - args["content"], args["message_type"], metadata=args.get("metadata") - ) + async with tool_activity(call.tools, BandTool.SEND_EVENT): + await call.tools.send_event( + args["content"], args["message_type"], metadata=args.get("metadata") + ) return succeeded("Event sent") diff --git a/src/band/integrations/crewai/tools.py b/src/band/integrations/crewai/tools.py index 104e1a573..37387f503 100644 --- a/src/band/integrations/crewai/tools.py +++ b/src/band/integrations/crewai/tools.py @@ -53,6 +53,7 @@ execute_custom_tool, get_custom_tool_name, ) +from band.runtime.toolactivity import tool_activity from band.runtime.tools import ( EVENT_TOOL_NAMES, BandTool, @@ -219,9 +220,10 @@ class CustomCrewAITool(BaseTool): def _run(self, *_args: Any, **kwargs: Any) -> Any: async def execute(tools: AgentToolsProtocol) -> str: await reporter.report_call(tools, tool_name, kwargs) - result = await execute_custom_tool( - tool=definition, arguments=kwargs, turn=tools.turn - ) + async with tool_activity(tools, tool_name): + result = await execute_custom_tool( + tool=definition, arguments=kwargs, turn=tools.turn + ) await reporter.report_result(tools, tool_name, result) return json.dumps({"status": "success", "result": result}, default=str) diff --git a/src/band/integrations/mcp/engine.py b/src/band/integrations/mcp/engine.py index c89c6ea4f..2bd34d752 100644 --- a/src/band/integrations/mcp/engine.py +++ b/src/band/integrations/mcp/engine.py @@ -49,6 +49,7 @@ execute_custom_tool, get_custom_tool_name, ) +from band.runtime.toolactivity import tool_activity from band.runtime.tools import ( CHAT_ID_FIELD_NAME, CHAT_ID_MAX_LENGTH, @@ -219,7 +220,8 @@ async def dispatch_tool( """ method = resolve_tool_method(tools, definition) try: - return await method(**arguments) + async with tool_activity(tools, definition.name): + return await method(**arguments) except (ValueError, BandToolError) as error: raise enrich_send_message_error(definition, tools, error) from error diff --git a/src/band/integrations/parlant/bandtools/chat.py b/src/band/integrations/parlant/bandtools/chat.py index 6886419d2..311d87f94 100644 --- a/src/band/integrations/parlant/bandtools/chat.py +++ b/src/band/integrations/parlant/bandtools/chat.py @@ -15,7 +15,8 @@ split_mentions, ) from band.integrations.parlant.sessiontools import require_session_tools -from band.runtime.tools import ParticipantAddStatus, serialize_tool_result +from band.runtime.toolactivity import tool_activity +from band.runtime.tools import BandTool, ParticipantAddStatus, serialize_tool_result EVENT_MESSAGE_TYPES: tuple[str, ...] = get_args(EventMessageType) @@ -59,7 +60,8 @@ async def band_send_event( if message_type not in EVENT_MESSAGE_TYPES: return invalid_choice("message_type", message_type, EVENT_MESSAGE_TYPES) - await tools.send_event(content, message_type, None) + async with tool_activity(tools, BandTool.SEND_EVENT): + await tools.send_event(content, message_type, None) return ToolResult(data=f"Event ({message_type}) sent successfully") diff --git a/src/band/integrations/parlant/customtools.py b/src/band/integrations/parlant/customtools.py index 0f5c1f8d9..fc4095d67 100644 --- a/src/band/integrations/parlant/customtools.py +++ b/src/band/integrations/parlant/customtools.py @@ -16,6 +16,7 @@ require_session_tools, ) from band.runtime.custom_tools import CustomToolDef, execute_custom_tool +from band.runtime.toolactivity import tool_activity from band.runtime.tools import tool_result_text # Parlant delivers every argument as a string, which only lax validation @@ -30,12 +31,13 @@ def build_custom_tool(tool_def: CustomToolDef) -> Any: async def run(context: ToolContext, **arguments: Any) -> ToolResult: tools = require_session_tools(context) - result = await execute_custom_tool( - tool_def, - parse_delivered(arguments, described.fields), - turn=tools.turn, - strict=STRICT_VALIDATION, - ) + async with tool_activity(tools, described.name): + result = await execute_custom_tool( + tool_def, + parse_delivered(arguments, described.fields), + turn=tools.turn, + strict=STRICT_VALIDATION, + ) return ToolResult(data=tool_result_text(result)) run.__name__ = described.name diff --git a/src/band/integrations/pydantic_ai/tools.py b/src/band/integrations/pydantic_ai/tools.py index a9d724b3b..0fa1ff068 100644 --- a/src/band/integrations/pydantic_ai/tools.py +++ b/src/band/integrations/pydantic_ai/tools.py @@ -25,6 +25,7 @@ from band.core.protocols import AgentToolsProtocol from band.core.tool_filter import sanitize_tool_schema from band.core.types import AdapterFeatures, MessageType +from band.runtime.toolactivity import tool_activity from band.runtime.tools import ( BandTool, ToolDefinition, @@ -202,7 +203,8 @@ async def dispatch(ctx: RunContext[AgentToolsProtocol], **arguments: Any) -> Any # something to hand the model as a tool error it could act on. method = _resolve_method(ctx.deps, definition) try: - result = await method(**kwargs) + async with tool_activity(ctx.deps, definition.name): + result = await method(**kwargs) except Exception as error: # noqa: BLE001 -- tool calls may raise any exception type; must surface to the LLM as an error string, not crash the turn return await _dispatch_failed(ctx, definition, error, notify_room=True) try: diff --git a/src/band/integrations/slack/adapter.py b/src/band/integrations/slack/adapter.py index 2842bf067..6c85fd66e 100644 --- a/src/band/integrations/slack/adapter.py +++ b/src/band/integrations/slack/adapter.py @@ -61,6 +61,7 @@ dispose_adapter, dispose_framework, ) +from band.runtime.toolactivity import tool_activity from band.runtime.tools import AgentTools, ToolCallOutcome, TurnEffect from band.runtime.tools.schema import is_failed_tool_output @@ -162,6 +163,7 @@ def __init__( # adapter judging ``wrap`` sees what the brain did through this tee. self._ctx = wrap._ctx self._scope = wrap._scope + self._activity_observer = wrap._activity_observer self.turn = wrap.turn self._slack = slack self._binding = binding @@ -214,11 +216,12 @@ async def slack_send_message(self, content: str) -> dict[str, Any]: failed (e.g. the channel was deleted, the bot was kicked). """ try: - await self._slack.chat_postMessage( - channel=self._binding.channel, - text=content, - thread_ts=self._binding.thread_ts, - ) + async with tool_activity(self, SLACK_SEND_MESSAGE_TOOL_NAME): + await self._slack.chat_postMessage( + channel=self._binding.channel, + text=content, + thread_ts=self._binding.thread_ts, + ) except Exception as exc: logger.exception( "slack_send_message failed (channel=%s thread_ts=%s)", diff --git a/src/band/runtime/activity.py b/src/band/runtime/activity.py index 62de696b3..bffd8e92d 100644 --- a/src/band/runtime/activity.py +++ b/src/band/runtime/activity.py @@ -1,9 +1,10 @@ """Names-only room activity detail, the per-turn state that holds it, and the -producers a turn reports it through.""" +observer a turn reports it through.""" from __future__ import annotations from dataclasses import dataclass +from inspect import getattr_static from typing import Protocol from band_rest.types.agent_activity_detail import AgentActivityDetail @@ -56,25 +57,16 @@ def apply( self.waiting = waiting -def _observer(tools: object) -> ActivityObserver | None: +def activity_observer_of(tools: object) -> ActivityObserver | None: + """The turn observer bound to ``tools``, if it declares one.""" + # Only a declared attribute counts: a dynamic proxy (``__getattr__``, a + # mock) would conjure an observer it never had. The second, dynamic read + # resolves a declared property such as a wrapper's pass-through. + if getattr_static(tools, "_activity_observer", None) is None: + return None return getattr(tools, "_activity_observer", None) -async def report_tool_activity(tools: object, name: str) -> None: - if observer := _observer(tools): - await observer.update(tool_call_activity(name)) - - -async def end_tool_activity(tools: object) -> None: - if observer := _observer(tools): - await observer.update(REASONING) - - -async def report_memory_activity(tools: object) -> None: - if observer := _observer(tools): - await observer.update(MEMORY) - - async def set_permission_wait(tools: object, waiting: bool) -> None: - if observer := _observer(tools): + if observer := activity_observer_of(tools): await observer.set_manual_wait(waiting) diff --git a/src/band/runtime/toolactivity.py b/src/band/runtime/toolactivity.py new file mode 100644 index 000000000..1d784469e --- /dev/null +++ b/src/band/runtime/toolactivity.py @@ -0,0 +1,113 @@ +"""What a tool call shows as its turn's activity, and the one place it is reported.""" + +from __future__ import annotations + +import functools +import logging +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from contextvars import ContextVar +from typing import Any + +from band_rest.types.agent_activity_detail import AgentActivityDetail + +from band.runtime.activity import ( + MEMORY, + REASONING, + ActivityObserver, + activity_observer_of, + tool_call_activity, +) +from band.runtime.tools.effects import turn_effect +from band.runtime.tools.registry import ( + MEMORY_TOOL_NAMES, + ToolsClass, + agent_tool_methods, +) +from band.runtime.tools.types import BandTool + +logger = logging.getLogger(__name__) + +MEMORY_SAVE_TOOLS = frozenset( + name for name in MEMORY_TOOL_NAMES if turn_effect(name).did_work +) + +# Set while a tool call is shown, so a Band tool another tool calls stays hidden. +_showing_tool: ContextVar[bool] = ContextVar("band_showing_tool", default=False) + + +def tool_detail(name: str) -> AgentActivityDetail: + """What the room sees while ``name`` runs.""" + return MEMORY if name in MEMORY_SAVE_TOOLS else tool_call_activity(name) + + +def method_reports_activity(name: str) -> bool: + """Whether a tools method reports itself; adapters post their own events + through ``send_event``, so only a model's call to it reports.""" + return name != BandTool.SEND_EVENT + + +async def _show(observer: ActivityObserver, detail: AgentActivityDetail) -> None: + """Activity is best-effort: a failed report never changes the tool's outcome.""" + try: + await observer.update(detail) + except Exception: + logger.warning("Could not report %s activity", detail.kind, exc_info=True) + + +@asynccontextmanager +async def tool_activity(tools: object, name: str) -> AsyncIterator[None]: + """Show ``name`` as the turn's activity until the tool ends. + + Only the outermost call reports, so a tool that calls other Band tools + shows as itself throughout. The observer is resolved once so the end lands + on the turn the tool started on, even if ``tools`` is rebound meanwhile. + """ + observer = activity_observer_of(tools) + if observer is None or _showing_tool.get(): + yield + return + token = _showing_tool.set(True) + try: + await _show(observer, tool_detail(name)) + yield + finally: + _showing_tool.reset(token) + await _show(observer, REASONING) + + +async def _show_on(tools: object, detail: AgentActivityDetail) -> None: + if observer := activity_observer_of(tools): + await _show(observer, detail) + + +async def report_tool_activity(tools: object, name: str) -> None: + """Show ``name`` for a tool whose start and end arrive as separate events.""" + await _show_on(tools, tool_detail(name)) + + +async def end_tool_activity(tools: object) -> None: + await _show_on(tools, REASONING) + + +async def report_memory_activity(tools: object) -> None: + await _show_on(tools, MEMORY) + + +def _observing( + tool_name: str, method: Callable[..., Awaitable[Any]] +) -> Callable[..., Awaitable[Any]]: + @functools.wraps(method) + async def observe(self: Any, *args: Any, **kwargs: Any) -> Any: + async with tool_activity(self, tool_name): + return await method(self, *args, **kwargs) + + return observe + + +def reports_tool_activity(cls: ToolsClass) -> ToolsClass: + """Report each agent tool method that ``method_reports_activity`` selects.""" + for definition, method in agent_tool_methods(cls): + if method_reports_activity(definition.name): + setattr(cls, definition.method_name, _observing(definition.name, method)) + return cls diff --git a/src/band/runtime/tools/agent.py b/src/band/runtime/tools/agent.py index 9067440a0..2edfe1fde 100644 --- a/src/band/runtime/tools/agent.py +++ b/src/band/runtime/tools/agent.py @@ -63,6 +63,7 @@ from band.runtime.context_serialization import context_item_to_dict from band.runtime.cycle import TurnScope from band.runtime.participants import log_roster_call, participant_snapshot +from band.runtime.toolactivity import reports_tool_activity, tool_activity from band.runtime.tools.registry import ( TOOL_DEFINITIONS, TOOL_MODELS, @@ -286,6 +287,7 @@ def cache_parameters(self) -> Any: ... @records_turn_effects +@reports_tool_activity class AgentTools(AgentToolsProtocol): """ Room-bound tools for LLM platform interaction. @@ -2042,7 +2044,8 @@ async def execute_tool_call_structured( try: method = getattr(self, definition.method_name) - result = await method(**arguments) + async with tool_activity(self, tool_name): + result = await method(**arguments) return ToolCallOutcome(value=serialize_tool_result(result), ok=True) except (BandToolError, RoomExecutionStoppedError): # Stopped execution must escape ordinary tool-error conversion. diff --git a/src/band/runtime/tools/registry.py b/src/band/runtime/tools/registry.py index 28dc3df16..feafb3d14 100644 --- a/src/band/runtime/tools/registry.py +++ b/src/band/runtime/tools/registry.py @@ -10,8 +10,8 @@ from __future__ import annotations import base64 -from collections.abc import Collection, Mapping -from typing import Any, Literal, TypedDict, cast +from collections.abc import Awaitable, Callable, Collection, Iterator, Mapping +from typing import Any, Literal, TypedDict, TypeVar, cast from pydantic import BaseModel @@ -920,6 +920,20 @@ def decode_image_blocks(result: Mapping[str, Any]) -> list[tuple[bytes, str]]: return [decode_image_block(block) for block in result["content"]] +ToolsClass = TypeVar("ToolsClass", bound=type) +"""A tools class a decorator rewrites in place and returns.""" + + +def agent_tool_methods( + cls: type, +) -> Iterator[tuple[ToolDefinition, Callable[..., Awaitable[Any]]]]: + """Each agent tool ``cls`` itself defines, with its method.""" + for definition in TOOL_DEFINITIONS.values(): + method = cls.__dict__.get(definition.method_name) + if definition.surface == Surface.AGENT and method is not None: + yield definition, method + + def selected_tool_definitions( features: AdapterFeatures, *, is_hub_room: bool = False ) -> list[ToolDefinition]: diff --git a/src/band/runtime/turn.py b/src/band/runtime/turn.py index c99551e48..649383cca 100644 --- a/src/band/runtime/turn.py +++ b/src/band/runtime/turn.py @@ -4,14 +4,12 @@ import functools from collections.abc import Awaitable, Callable -from typing import Any, TypeVar +from typing import Any from band.core.turn import Turn from band.runtime.tools.effects import turn_effect -from band.runtime.tools.registry import TOOL_DEFINITIONS -from band.runtime.tools.types import Surface, TurnEffect - -_ToolsClass = TypeVar("_ToolsClass", bound=type) +from band.runtime.tools.registry import ToolsClass, agent_tool_methods +from band.runtime.tools.types import TurnEffect def record_tool_result(turn: Turn, tool_name: str, result: Any) -> None: @@ -37,14 +35,12 @@ async def record_on_success(self: Any, *args: Any, **kwargs: Any) -> Any: return record_on_success -def records_turn_effects(cls: _ToolsClass) -> _ToolsClass: +def records_turn_effects(cls: ToolsClass) -> ToolsClass: """Record every agent tool method's effect on the instance's ``turn``. Wrapping the methods themselves counts every call path: registry dispatch, a framework calling the method directly, and ``deliver_reply``. """ - for definition in TOOL_DEFINITIONS.values(): - method = cls.__dict__.get(definition.method_name) - if definition.surface == Surface.AGENT and method is not None: - setattr(cls, definition.method_name, _recording(definition.name, method)) + for definition, method in agent_tool_methods(cls): + setattr(cls, definition.method_name, _recording(definition.name, method)) return cls diff --git a/src/band/testing/activity.py b/src/band/testing/activity.py new file mode 100644 index 000000000..b48a93922 --- /dev/null +++ b/src/band/testing/activity.py @@ -0,0 +1,45 @@ +"""Record what a turn's activity observer is told, without reporting it.""" + +from __future__ import annotations + +from typing import Any, Protocol, TypeVar + +from band_rest.types.agent_activity_detail import AgentActivityDetail + +from band.runtime.activity import ActivityObserver + + +class ObservableTools(Protocol): + _activity_observer: ActivityObserver | None + + +_Tools = TypeVar("_Tools", bound=ObservableTools) + +SHOWN_REASONING: dict[str, Any] = {"kind": "reasoning"} +SHOWN_MEMORY: dict[str, Any] = {"kind": "memory"} + + +def shown_tool(name: str) -> dict[str, Any]: + """A recorded ``tool_call`` observation for ``name``.""" + return {"kind": "tool_call", "name": name} + + +class ActivityRecorder: + """An ``ActivityObserver`` that keeps every observation in order.""" + + def __init__(self) -> None: + self.events: list[dict[str, Any]] = [] + + def observing(self, tools: _Tools) -> _Tools: + """Bind this recorder as the turn observer of ``tools``.""" + tools._activity_observer = self + return tools + + async def update(self, activity: AgentActivityDetail) -> None: + self.events.append(activity.model_dump(exclude_none=True)) + + async def set_manual_wait(self, waiting: bool) -> None: + self.events.append({"waiting": waiting}) + + def assert_completed_tool(self, name: str) -> None: + assert self.events == [shown_tool(name), SHOWN_REASONING] diff --git a/src/band/testing/fake_tools.py b/src/band/testing/fake_tools.py index 35248e458..09b95e59e 100644 --- a/src/band/testing/fake_tools.py +++ b/src/band/testing/fake_tools.py @@ -66,6 +66,7 @@ ContactRequestStatus, MessageType, ) +from band.runtime.activity import ActivityObserver from band.runtime.context_serialization import context_item_to_dict from band.runtime.tools import ( DEFAULT_FILE_CAPTION, @@ -326,6 +327,7 @@ def __init__( self._held_messages: list[HeldMessage] = [] self._observers: list[tuple[Callable[[], bool], asyncio.Future[None]]] = [] self.turn = Turn() + self._activity_observer: ActivityObserver | None = None @property def agent_id(self) -> str | None: diff --git a/tests/adapters/activity/__init__.py b/tests/adapters/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/adapters/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/adapters/activity/kit.py b/tests/adapters/activity/kit.py new file mode 100644 index 000000000..c45e3d549 --- /dev/null +++ b/tests/adapters/activity/kit.py @@ -0,0 +1,56 @@ +"""Model entry points that drive one Band tool call through a real framework boundary.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import Any + +from mcp.shared.memory import create_connected_server_and_client_session +from pydantic_ai import Agent +from pydantic_ai.messages import ModelMessage, ModelResponse, TextPart, ToolCallPart +from pydantic_ai.models.function import AgentInfo, FunctionModel + +from band.core.protocols import AgentToolsProtocol +from band.core.types import AdapterFeatures +from band.integrations.pydantic_ai.tools import build_band_pydantic_ai_tools +from band.runtime.tools import AgentTools +from band.runtime.tools.registry import TOOL_DEFINITIONS +from tests.runtime.helpers import ROOM_ID +from tests.runtime.toolactivity.kit import band_tool_server + +ModelCall = Callable[[AgentTools, str, dict[str, Any]], Awaitable[Any]] + + +async def call_over_mcp(tools: AgentTools, name: str, arguments: dict[str, Any]) -> Any: + server = band_tool_server(tools, TOOL_DEFINITIONS[name]) + async with create_connected_server_and_client_session(server) as session: + result = await session.call_tool(name, {**arguments, "chat_id": ROOM_ID}) + return result.content + + +async def call_over_pydantic_ai( + tools: AgentTools, name: str, arguments: dict[str, Any] +) -> Any: + """Run a real ``Agent`` whose model calls ``name`` once, then answers.""" + + def model(messages: list[ModelMessage], _info: AgentInfo) -> ModelResponse: + if len(messages) == 1: + return ModelResponse(parts=[ToolCallPart(tool_name=name, args=arguments)]) + return ModelResponse(parts=[TextPart("done")]) + + agent = Agent( + FunctionModel(model), + deps_type=AgentToolsProtocol, + tools=build_band_pydantic_ai_tools(AdapterFeatures()), + ) + result = await agent.run("go", deps=tools) + return result.all_messages() + + +async def call_through_execute_tool_call( + tools: AgentTools, name: str, arguments: dict[str, Any] +) -> Any: + return await tools.execute_tool_call(name, arguments) + + +MODEL_CALLS = [call_over_mcp, call_over_pydantic_ai, call_through_execute_tool_call] diff --git a/tests/adapters/activity/test_failed_model_tool.py b/tests/adapters/activity/test_failed_model_tool.py new file mode 100644 index 000000000..a53f2a77e --- /dev/null +++ b/tests/adapters/activity/test_failed_model_tool.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from band.runtime.tools import AgentTools +from band.runtime.tools.types import BandTool +from band.testing.activity import ActivityRecorder +from tests.adapters.activity.kit import MODEL_CALLS, ModelCall +from tests.runtime.helpers import ROOM_ID + + +class TestFailedModelToolActivity: + @pytest.mark.parametrize("call", MODEL_CALLS) + async def test_a_failing_call_returns_the_room_to_reasoning( + self, mock_rest_client: MagicMock, call: ModelCall + ) -> None: + listing = mock_rest_client.agent_api_participants.list_agent_chat_participants + listing.side_effect = RuntimeError("room unavailable") + recorder = ActivityRecorder() + tools = recorder.observing(AgentTools(ROOM_ID, mock_rest_client)) + + answer = await call(tools, BandTool.GET_PARTICIPANTS, {}) + + assert "room unavailable" in str(answer) + recorder.assert_completed_tool(BandTool.GET_PARTICIPANTS) diff --git a/tests/adapters/activity/test_model_event.py b/tests/adapters/activity/test_model_event.py new file mode 100644 index 000000000..2690ef98a --- /dev/null +++ b/tests/adapters/activity/test_model_event.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from band.runtime.tools import AgentTools +from band.runtime.tools.types import BandTool +from band.testing.activity import ActivityRecorder +from tests.adapters.activity.kit import MODEL_CALLS, ModelCall +from tests.runtime.helpers import ROOM_ID + +EVENT = {"content": "model telemetry", "message_type": "thought"} + + +class TestModelToolActivity: + """A model's tool call shows exactly once, whichever framework dispatched it.""" + + @pytest.mark.parametrize("call", MODEL_CALLS) + @pytest.mark.parametrize( + ("name", "arguments"), + [(BandTool.SEND_EVENT, EVENT), (BandTool.GET_PARTICIPANTS, {})], + ids=["send_event", "get_participants"], + ) + async def test_the_call_shows_once_while_adapter_telemetry_stays_quiet( + self, + mock_rest_client: MagicMock, + call: ModelCall, + name: BandTool, + arguments: dict[str, Any], + ) -> None: + recorder = ActivityRecorder() + tools = recorder.observing(AgentTools(ROOM_ID, mock_rest_client)) + await tools.send_event("adapter telemetry", "thought") + assert recorder.events == [] + + await call(tools, name, arguments) + + recorder.assert_completed_tool(name) diff --git a/tests/adapters/strands/kit.py b/tests/adapters/strands/kit.py index 6a74a87e9..fae67b13d 100644 --- a/tests/adapters/strands/kit.py +++ b/tests/adapters/strands/kit.py @@ -72,7 +72,7 @@ async def _run_message( ) -async def _run_turn(adapter: StrandsAdapter, tools: FakeAgentTools) -> None: +async def run_judged_turn(adapter: StrandsAdapter, tools: FakeAgentTools) -> None: """Run one turn through ``on_event``, where the shared turn verdict lives.""" await adapter.on_event(turn_input(tools, msg=_make_msg(ROOM))) diff --git a/tests/adapters/strands/test_model_event_activity.py b/tests/adapters/strands/test_model_event_activity.py new file mode 100644 index 000000000..a91cd71ae --- /dev/null +++ b/tests/adapters/strands/test_model_event_activity.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from collections.abc import Awaitable, Callable + +import pytest + +pytest.importorskip("strands", reason="strands extra not installed") + +from band.adapters.strands import StrandsAdapter +from band.runtime.tools import BandTool +from band.testing import FakeAgentTools, TextTurn, ToolTurn +from band.testing.activity import SHOWN_REASONING, ActivityRecorder, shown_tool +from tests.adapters.strands.kit import run_judged_turn + + +class TestStrandsToolActivity: + async def test_model_tool_calls_show_in_order_while_adapter_telemetry_stays_quiet( + self, + scripted: Callable[..., Awaitable[StrandsAdapter]], + tools: FakeAgentTools, + ) -> None: + recorder = ActivityRecorder() + recorder.observing(tools) + adapter = await scripted( + ToolTurn( + BandTool.SEND_EVENT, {"content": "model", "message_type": "thought"} + ), + ToolTurn( + BandTool.SEND_MESSAGE, {"content": "done", "mentions": ["@tester"]} + ), + TextTurn("finished"), + ) + + await run_judged_turn(adapter, tools) + + assert recorder.events == [ + shown_tool(BandTool.SEND_EVENT), + SHOWN_REASONING, + shown_tool(BandTool.SEND_MESSAGE), + SHOWN_REASONING, + ] + assert "model" in [event["content"] for event in tools.events_sent] diff --git a/tests/adapters/strands/test_turn_failure.py b/tests/adapters/strands/test_turn_failure.py index c4377956d..11e662db4 100644 --- a/tests/adapters/strands/test_turn_failure.py +++ b/tests/adapters/strands/test_turn_failure.py @@ -49,8 +49,8 @@ SEND_TURN, _make_msg, _run_message, - _run_turn, _tool_results, + run_judged_turn, ) from tests.responsespeer import ResponsesPeer from tests.strandskit import outline @@ -102,7 +102,7 @@ async def test_bedrock_decline_preserves_followup_turn( ) await adapter.on_started("Bot", "A bot") first = FakeAgentTools(room_id=ROOM) - await _run_turn(adapter, first) + await run_judged_turn(adapter, first) second = FakeAgentTools(room_id=ROOM) await adapter.on_event( replace( @@ -146,7 +146,7 @@ async def test_responses_decline_preserves_followup_tool_output( ) await adapter.on_started("Bot", "A bot") first = FakeAgentTools(room_id=ROOM) - await _run_turn(adapter, first) + await run_judged_turn(adapter, first) second = FakeAgentTools(room_id=ROOM) await adapter.on_event( replace( @@ -195,7 +195,7 @@ async def test_responses_decline_ends_only_a_successful_turn( ) await adapter.on_started("Bot", "A bot") - await _run_turn(adapter, tools) + await run_judged_turn(adapter, tools) assert len(peer.requests) == (2 if fails else 1) assert tools.turn.complete @@ -232,9 +232,9 @@ async def test_responses_tool_events_usage_and_failure( if fails: with pytest.raises(EventLoopException, match="provider down"): - await _run_turn(adapter, tools) + await run_judged_turn(adapter, tools) else: - await _run_turn(adapter, tools) + await run_judged_turn(adapter, tools) tools.assert_message_sent(content="hi", count=1) (call,) = events_of_type(tools, MessageType.TOOL_CALL) diff --git a/tests/adapters/strands/test_turn_productivity.py b/tests/adapters/strands/test_turn_productivity.py index fb49b5c69..9a3b9a48b 100644 --- a/tests/adapters/strands/test_turn_productivity.py +++ b/tests/adapters/strands/test_turn_productivity.py @@ -46,9 +46,9 @@ ROOM, SEND_TURN, _run_message, - _run_turn, _tool_results, failure_diagnostics, + run_judged_turn, ) from tests.framework_conformance.turnprobes import CUSTOM_TOOL_DECLARATIONS @@ -73,7 +73,7 @@ async def test_missing_reply_logs_reads_and_final_output_without_events( ) with pytest.raises(TurnResultAlreadyReported): - await _run_turn(adapter, tools) + await run_judged_turn(adapter, tools) (diagnostic,) = failure_diagnostics(caplog) assert diagnostic["final_text"].strip() == final_text @@ -112,7 +112,7 @@ async def send_message( ) with pytest.raises(TurnResultAlreadyReported): - await _run_turn(adapter, tools) + await run_judged_turn(adapter, tools) (diagnostic,) = failure_diagnostics(caplog) (call,) = diagnostic["tool_calls"] @@ -134,7 +134,7 @@ async def test_completed_turn_does_not_log_diagnostics( ) -> None: adapter = await scripted(SEND_TURN) with caplog.at_level(logging.WARNING): - await _run_turn(adapter, tools) + await run_judged_turn(adapter, tools) assert failure_diagnostics(caplog) == [] @pytest.mark.asyncio @@ -192,7 +192,7 @@ async def inspect(args: InspectInput) -> dict[str, Any]: ) with pytest.raises(TurnResultAlreadyReported): - await _run_turn(adapter, tools) + await run_judged_turn(adapter, tools) (diagnostic,) = failure_diagnostics(caplog) (call,) = diagnostic["tool_calls"] @@ -214,10 +214,10 @@ async def test_failure_diagnostics_only_include_the_failing_turn( ToolTurn("band_lookup_peers"), TextTurn("checked"), ) - await _run_turn(adapter, tools) + await run_judged_turn(adapter, tools) with pytest.raises(TurnResultAlreadyReported): - await _run_turn(adapter, FakeAgentTools(room_id=ROOM)) + await run_judged_turn(adapter, FakeAgentTools(room_id=ROOM)) (diagnostic,) = failure_diagnostics(caplog) (call,) = diagnostic["tool_calls"] @@ -239,7 +239,7 @@ async def test_failure_diagnostics_keep_repeated_calls( emit=set(), ) with pytest.raises(TurnResultAlreadyReported): - await _run_turn(adapter, tools) + await run_judged_turn(adapter, tools) (diagnostic,) = failure_diagnostics(caplog) first, second = diagnostic["tool_calls"] assert first["args"] == {"limit": 1} @@ -261,7 +261,7 @@ async def test_provider_failure_keeps_the_partial_tool_trace( emit=set(), ) with pytest.raises(EventLoopException, match="provider down"): - await _run_turn(adapter, tools) + await run_judged_turn(adapter, tools) (diagnostic,) = failure_diagnostics(caplog) (call,) = diagnostic["tool_calls"] assert call["name"] == "band_list_tasks" @@ -281,7 +281,7 @@ async def send_message(self, content, mentions=None): adapter = await scripted(SEND_TURN) with pytest.raises(TurnResultAlreadyReported): - await _run_turn(adapter, tools) + await run_judged_turn(adapter, tools) assert tools.messages_sent == [] assert failure_reports(tools) == [MISSING_REPLY_FAILURE] @@ -326,7 +326,7 @@ async def test_read_only_tool_alone_does_not_end_the_turn(self, tools, scripted) adapter = await scripted(ToolTurn("band_lookup_peers", {})) with pytest.raises(TurnResultAlreadyReported): - await _run_turn(adapter, tools) + await run_judged_turn(adapter, tools) assert _tool_results(adapter) # the lookup did run and succeed assert tools.messages_sent == [] @@ -347,7 +347,7 @@ def native_finish(note: str) -> str: ToolTurn("native_finish", {"note": "go"}), additional_tools=[native_finish] ) - await _run_turn(adapter, tools) + await run_judged_turn(adapter, tools) assert failure_reports(tools) == [] @@ -374,7 +374,7 @@ def native_finish(note: str) -> Any: ) with pytest.raises(TurnResultAlreadyReported): - await _run_turn(adapter, tools) + await run_judged_turn(adapter, tools) assert failure_reports(tools) == [MISSING_REPLY_FAILURE] @@ -394,7 +394,7 @@ def native_peek(note: str) -> str: additional_tools=[native_peek], ) - await _run_turn(adapter, tools) + await run_judged_turn(adapter, tools) tools.assert_message_sent(content="hi", count=1) assert tools.turn.complete @@ -412,7 +412,7 @@ def native_peek(note: str) -> str: ) with pytest.raises(TurnResultAlreadyReported): - await _run_turn(adapter, tools) + await run_judged_turn(adapter, tools) assert failure_reports(tools) == [MISSING_REPLY_FAILURE] diff --git a/tests/agent/test_startup_race_condition.py b/tests/agent/test_startup_race_condition.py index 36e01ff91..a493ea408 100644 --- a/tests/agent/test_startup_race_condition.py +++ b/tests/agent/test_startup_race_condition.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import UTC, datetime -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock import pytest @@ -14,7 +14,8 @@ ) from band.core.simple_adapter import SimpleAdapter from band.platform.event import MessageEvent -from band.runtime.context import ConversationContext +from band.runtime.execution import ExecutionContext +from band.runtime.sessionconfig import SessionConfig from band.testing.platform import platform_connection_stub @@ -26,7 +27,9 @@ class TestStartupRaceCondition: """ @pytest.mark.asyncio - async def test_adapter_on_started_before_first_message(self): + async def test_adapter_on_started_before_first_message( + self, mock_rest_client: MagicMock + ) -> None: """System prompt must be set before any message processing.""" # Track the order of calls @@ -93,20 +96,13 @@ async def start( self._on_execute = on_execute # Create a proper MessageEvent that passes through preprocessor - mock_ctx = MagicMock() - mock_ctx.is_llm_initialized = False - mock_ctx.mark_llm_initialized = MagicMock() - mock_ctx.get_context = AsyncMock( - return_value=ConversationContext( - room_id="room-123", - messages=[], - participants=[], - hydrated_at=datetime.now(UTC), - ) + mock_ctx = ExecutionContext( + room_id="room-123", + link=MagicMock(rest=mock_rest_client), + on_execute=on_execute, + agent_id="agent-123", + config=SessionConfig(enable_context_hydration=False), ) - mock_ctx.participants = [] - mock_ctx.participants_changed = MagicMock(return_value=False) - mock_ctx.room_id = "room-123" # Create a proper MessageEvent (not a MagicMock) message_event = MessageEvent( @@ -139,8 +135,4 @@ async def run_forever(self, *, install_signal_handlers: bool = True): # This will fail if on_message is called before on_started await agent.start() - # Verify on_started was called first - assert len(call_order) >= 1, "No calls were made" - assert call_order[0] == "on_started", ( - f"Expected on_started to be called first, but got: {call_order}" - ) + assert call_order == ["on_started", "on_message"] diff --git a/tests/framework_conformance/test_crewai_job_coverage.py b/tests/framework_conformance/test_crewai_job_coverage.py index 89f5314fc..e54c57981 100644 --- a/tests/framework_conformance/test_crewai_job_coverage.py +++ b/tests/framework_conformance/test_crewai_job_coverage.py @@ -75,6 +75,8 @@ def test_detection_finds_the_tests_that_only_the_crewai_venv_can_run(self) -> No Path("tests/integrations/test_crewai_flow_real_sdk.py"), Path("tests/integrations/test_crewai_real_tools.py"), Path("tests/integrations/test_crewai_completion.py"), + Path("tests/integrations/crewai/activity/test_execution.py"), + Path("tests/integrations/crewai/activity/test_send_event.py"), Path("tests/test_capability_gating_e2e.py"), Path("tests/framework_conformance/test_files_image_passthrough_matrix.py"), } diff --git a/tests/framework_conformance/test_parlant_job_coverage.py b/tests/framework_conformance/test_parlant_job_coverage.py index b0bf342e3..b1aee87d3 100644 --- a/tests/framework_conformance/test_parlant_job_coverage.py +++ b/tests/framework_conformance/test_parlant_job_coverage.py @@ -66,6 +66,8 @@ def test_detection_finds_the_tests_that_only_the_parlant_venv_can_run(self) -> N Path("tests/integrations/parlant/test_customtools.py"), Path("tests/integrations/parlant/test_filetools.py"), Path("tests/integrations/parlant/test_guard.py"), + Path("tests/integrations/parlant/activity/test_execution.py"), + Path("tests/integrations/parlant/activity/test_send_event.py"), Path("tests/integrations/parlant/test_preambletask.py"), Path("tests/integrations/parlant/test_selection.py"), Path("tests/integrations/parlant/test_server.py"), diff --git a/tests/integrations/activitykit.py b/tests/integrations/activitykit.py new file mode 100644 index 000000000..17e847712 --- /dev/null +++ b/tests/integrations/activitykit.py @@ -0,0 +1,11 @@ +"""A custom tool shared by the framework activity suites.""" + +from __future__ import annotations + +from pydantic import BaseModel + + +class LookupInput(BaseModel): + """Look up a private value.""" + + note: str diff --git a/tests/integrations/crewai/__init__.py b/tests/integrations/crewai/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/integrations/crewai/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/integrations/crewai/activity/__init__.py b/tests/integrations/crewai/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/integrations/crewai/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/integrations/crewai/activity/test_execution.py b/tests/integrations/crewai/activity/test_execution.py new file mode 100644 index 000000000..e6df594f0 --- /dev/null +++ b/tests/integrations/crewai/activity/test_execution.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import json +from typing import Any + +import pytest + +from band.integrations.crewai.tools import ( + CrewAIToolContext, + NoopReporter, + build_band_crewai_tools, +) +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder, shown_tool +from tests.integrations.activitykit import LookupInput + +pytest.importorskip("crewai") + + +class TestCrewAIActivity: + @pytest.mark.parametrize("failed", [False, True]) + def test_real_base_tool_reports_without_narration_and_ends_on_error( + self, failed: bool + ) -> None: + recorder = ActivityRecorder() + tools = recorder.observing(FakeAgentTools()) + shown_while_running: list[dict[str, Any]] = [] + received: list[str] = [] + + async def lookup(args: LookupInput) -> str: + shown_while_running.extend(recorder.events) + received.append(args.note) + if failed: + raise ValueError("private-error") + return "private-result" + + built = build_band_crewai_tools( + get_context=lambda: CrewAIToolContext(room_id="room", tools=tools), + reporter=NoopReporter(), + custom_tools=[(LookupInput, lookup)], + ) + lookup_tool = next(tool for tool in built if tool.name == "lookup") + result = json.loads(lookup_tool.run(note="private-argument")) + assert result["status"] == ("error" if failed else "success") + assert received == ["private-argument"] + assert shown_while_running == [shown_tool("lookup")] + recorder.assert_completed_tool("lookup") + assert not tools.events_sent diff --git a/tests/integrations/crewai/activity/test_send_event.py b/tests/integrations/crewai/activity/test_send_event.py new file mode 100644 index 000000000..963fcf121 --- /dev/null +++ b/tests/integrations/crewai/activity/test_send_event.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import json + +import pytest + +from band.integrations.crewai.tools import ( + CrewAIToolContext, + NoopReporter, + build_band_crewai_tools, +) +from band.runtime.tools import BandTool +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder + +pytest.importorskip("crewai") + + +class TestCrewAIModelEventActivity: + def test_a_model_sent_event_reports_once_and_is_sent_once(self) -> None: + recorder = ActivityRecorder() + tools = recorder.observing(FakeAgentTools()) + built = build_band_crewai_tools( + get_context=lambda: CrewAIToolContext(room_id="room", tools=tools), + reporter=NoopReporter(), + ) + send_event = next(tool for tool in built if tool.name == BandTool.SEND_EVENT) + + result = json.loads(send_event.run(content="model", message_type="thought")) + + assert result["status"] == "success" + recorder.assert_completed_tool(BandTool.SEND_EVENT) + assert [event["content"] for event in tools.events_sent] == ["model"] diff --git a/tests/integrations/parlant/activity/__init__.py b/tests/integrations/parlant/activity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/integrations/parlant/activity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/integrations/parlant/activity/test_execution.py b/tests/integrations/parlant/activity/test_execution.py new file mode 100644 index 000000000..d307f52e8 --- /dev/null +++ b/tests/integrations/parlant/activity/test_execution.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import asyncio + +import pytest + +pytest.importorskip("parlant.sdk") + +from parlant.core.tools import ToolContext + +from band.integrations.parlant.customtools import build_custom_tool +from band.integrations.parlant.sessiontools import set_session_tools +from band.integrations.parlant.tools import create_parlant_tools +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder, shown_tool +from tests.integrations.activitykit import LookupInput +from tests.integrations.parlant.helpers import ToolServer + + +class TestParlantActivity: + @pytest.mark.parametrize("failed", [False, True]) + async def test_real_server_reports_only_calling_session_and_always_ends( + self, plugin_server: ToolServer, failed: bool + ) -> None: + recorder, quiet = ActivityRecorder(), ActivityRecorder() + tools = recorder.observing(FakeAgentTools()) + other = quiet.observing(FakeAgentTools()) + set_session_tools("calling", tools) + set_session_tools("other", other) + entered, release = asyncio.Event(), asyncio.Event() + + async def lookup(args: LookupInput) -> str: + assert args.note == "private-argument" + entered.set() + await release.wait() + if failed: + raise ValueError("private-error") + return "private-result" + + await plugin_server.enable( + create_parlant_tools(custom_tools=[(LookupInput, lookup)]) + ) + call = asyncio.create_task( + plugin_server.call( + "lookup", session_id="calling", arguments={"note": "private-argument"} + ) + ) + try: + await entered.wait() + assert recorder.events == [shown_tool("lookup")] + assert not quiet.events + finally: + release.set() + result = await call + assert result == ( + "Error running lookup: private-error" if failed else "private-result" + ) + recorder.assert_completed_tool("lookup") + assert not tools.events_sent + + async def test_custom_handler_cancellation_ends_activity(self) -> None: + # The plugin server finishes a handler whose client disconnected, so + # only a direct call can deliver a cancellation to it. + recorder = ActivityRecorder() + set_session_tools("cancelled", recorder.observing(FakeAgentTools())) + entered = asyncio.Event() + + async def lookup(args: LookupInput) -> str: + entered.set() + await asyncio.Event().wait() + return "unreachable" + + entry = build_custom_tool((LookupInput, lookup)) + task = asyncio.create_task( + entry.function( + ToolContext( + agent_id="agent", session_id="cancelled", customer_id="user" + ), + note="private-argument", + ) + ) + await entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + recorder.assert_completed_tool("lookup") diff --git a/tests/integrations/parlant/activity/test_send_event.py b/tests/integrations/parlant/activity/test_send_event.py new file mode 100644 index 000000000..fd6a649f2 --- /dev/null +++ b/tests/integrations/parlant/activity/test_send_event.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +import pytest + +pytest.importorskip("parlant.sdk") + +from band.integrations.parlant.sessiontools import set_session_tools +from band.integrations.parlant.tools import create_parlant_tools +from band.runtime.tools import BandTool +from band.testing import FakeAgentTools +from band.testing.activity import ActivityRecorder +from tests.integrations.parlant.helpers import ToolServer + + +class TestParlantModelEventActivity: + async def test_a_model_sent_event_reports_once_and_is_sent_once( + self, plugin_server: ToolServer + ) -> None: + recorder = ActivityRecorder() + tools = recorder.observing(FakeAgentTools()) + set_session_tools("calling", tools) + await plugin_server.enable(create_parlant_tools()) + + await plugin_server.call( + BandTool.SEND_EVENT, + session_id="calling", + arguments={"content": "model", "message_type": "thought"}, + ) + + recorder.assert_completed_tool(BandTool.SEND_EVENT) + assert [event["content"] for event in tools.events_sent] == ["model"] diff --git a/tests/integrations/slack/test_activity.py b/tests/integrations/slack/test_activity.py new file mode 100644 index 000000000..51cc7a052 --- /dev/null +++ b/tests/integrations/slack/test_activity.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +from band.integrations.slack.adapter import ( + SLACK_SEND_MESSAGE_TOOL_NAME, + SlackTeeingTools, +) +from band.integrations.slack.types import SlackRoomBinding +from band.runtime.tools import AgentTools, BandTool +from band.testing.activity import SHOWN_REASONING, ActivityRecorder, shown_tool +from tests.runtime.helpers import ROOM_ID + + +class TestSlackTeeActivity: + async def test_tools_called_through_the_tee_report_on_the_wrapped_turn( + self, mock_rest_client: MagicMock + ) -> None: + recorder = ActivityRecorder() + band = recorder.observing(AgentTools(ROOM_ID, mock_rest_client)) + slack = AsyncMock() + tee = SlackTeeingTools( + wrap=band, + slack=slack, + binding=SlackRoomBinding(app_slug="dev", channel="C", thread_ts="1.0"), + ) + + await tee.execute_tool_call(BandTool.GET_PARTICIPANTS, {}) + await tee.execute_tool_call( + BandTool.SEND_EVENT, {"content": "model", "message_type": "thought"} + ) + await tee.execute_tool_call(SLACK_SEND_MESSAGE_TOOL_NAME, {"content": "hi"}) + + assert recorder.events == [ + shown_tool(BandTool.GET_PARTICIPANTS), + SHOWN_REASONING, + shown_tool(BandTool.SEND_EVENT), + SHOWN_REASONING, + shown_tool(SLACK_SEND_MESSAGE_TOOL_NAME), + SHOWN_REASONING, + ] + assert slack.chat_postMessage.await_args.kwargs["text"] == "hi" diff --git a/tests/runtime/activityownership/kit.py b/tests/runtime/activityownership/kit.py index 8ebb39513..41455d737 100644 --- a/tests/runtime/activityownership/kit.py +++ b/tests/runtime/activityownership/kit.py @@ -11,14 +11,14 @@ from band.core.turn_lifecycle import TurnTask, bind_turn_task from band.platform.activity import API_VERSION_HEADER from band.platform.event import PlatformEvent -from band.runtime.activity import ( +from band.runtime.activity import set_permission_wait +from band.runtime.execution import ExecutionContext +from band.runtime.sessionconfig import SessionConfig +from band.runtime.toolactivity import ( end_tool_activity, report_memory_activity, report_tool_activity, - set_permission_wait, ) -from band.runtime.execution import ExecutionContext -from band.runtime.sessionconfig import SessionConfig from band.runtime.tools.agent import AgentTools from tests.activitysupport import SUPPORTED_VERSION from tests.runtime.helpers import ALICE, LifecyclePlatform, is_message_post, page diff --git a/tests/runtime/activityownership/test_band_tool_wire.py b/tests/runtime/activityownership/test_band_tool_wire.py new file mode 100644 index 000000000..742506b24 --- /dev/null +++ b/tests/runtime/activityownership/test_band_tool_wire.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator, Callable + +import pytest + +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools import BandTool +from band.runtime.tools.agent import AgentTools +from tests.activitysupport import IDLE_BODY, REASONING_BODY, tool_body +from tests.conftest import make_message_event +from tests.runtime.activityownership.kit import HeldReplyPlatform +from tests.runtime.helpers import IDLE, KEEP_ALIVE + +REPLY = "Found you" + + +@pytest.mark.looptime +class TestBandToolWireActivity: + """A model's Band tool call reaches the platform as the turn's detail.""" + + @pytest.fixture + async def activity_platform(self) -> AsyncIterator[HeldReplyPlatform]: + yield HeldReplyPlatform(held_reply=REPLY) + + async def test_a_model_reply_shows_on_the_wire_until_it_is_posted( + self, + execution: Callable[..., ExecutionContext], + activity_platform: HeldReplyPlatform, + ) -> None: + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + tools = AgentTools.from_context(ctx) + outcome = await tools.execute_tool_call_structured( + BandTool.SEND_MESSAGE, {"content": REPLY, "mentions": ["alice"]} + ) + assert outcome.ok, outcome.error_message + + ctx = execution(handler) + turn = asyncio.create_task(ctx._process_event(make_message_event())) + await activity_platform.reply_entered.wait() + shown = await activity_platform.body_at(1) + activity_platform.release.set() + await turn + + assert shown == tool_body(BandTool.SEND_MESSAGE) + assert activity_platform.room_messages == [REPLY] + assert activity_platform.activity_bodies == [ + REASONING_BODY, + tool_body(BandTool.SEND_MESSAGE), + REASONING_BODY, + IDLE_BODY, + ] + assert activity_platform.reports_without_refreshes == [KEEP_ALIVE, IDLE] diff --git a/tests/runtime/activityownership/test_cap.py b/tests/runtime/activityownership/test_cap.py index 70e8b5c26..0b5d1a0ef 100644 --- a/tests/runtime/activityownership/test_cap.py +++ b/tests/runtime/activityownership/test_cap.py @@ -6,8 +6,9 @@ import pytest from band.platform.event import PlatformEvent -from band.runtime.activity import report_tool_activity, set_permission_wait +from band.runtime.activity import set_permission_wait from band.runtime.execution import ExecutionContext +from band.runtime.toolactivity import report_tool_activity from band.runtime.tools.agent import AgentTools from tests.activitysupport import IDLE_BODY, REASONING_BODY, tool_body from tests.conftest import make_message_event diff --git a/tests/runtime/activityownership/test_control.py b/tests/runtime/activityownership/test_control.py index d925bc5a4..8257b81bb 100644 --- a/tests/runtime/activityownership/test_control.py +++ b/tests/runtime/activityownership/test_control.py @@ -7,8 +7,8 @@ from band.core.turn_lifecycle import TurnTask, bind_turn_task from band.platform.event import PlatformEvent -from band.runtime.activity import report_memory_activity, report_tool_activity from band.runtime.execution import ExecutionContext +from band.runtime.toolactivity import report_memory_activity, report_tool_activity from band.runtime.tools.agent import AgentTools from tests.activitysupport import IDLE_BODY, REASONING_BODY, tool_body from tests.conftest import make_message_event diff --git a/tests/runtime/activityownership/test_delivery.py b/tests/runtime/activityownership/test_delivery.py index 1e7945693..fc1460a8d 100644 --- a/tests/runtime/activityownership/test_delivery.py +++ b/tests/runtime/activityownership/test_delivery.py @@ -6,8 +6,8 @@ import pytest from band.platform.event import PlatformEvent -from band.runtime.activity import report_tool_activity from band.runtime.execution import ExecutionContext +from band.runtime.toolactivity import report_tool_activity from band.runtime.tools.agent import AgentTools from tests.activitysupport import tool_body from tests.conftest import make_message_event diff --git a/tests/runtime/activityownership/test_detail.py b/tests/runtime/activityownership/test_detail.py index 129ac2234..f7cee4acb 100644 --- a/tests/runtime/activityownership/test_detail.py +++ b/tests/runtime/activityownership/test_detail.py @@ -7,13 +7,13 @@ from band.core.turn_lifecycle import TurnTask, bind_turn_task from band.platform.event import PlatformEvent -from band.runtime.activity import ( +from band.runtime.activity import set_permission_wait +from band.runtime.execution import ExecutionContext +from band.runtime.toolactivity import ( end_tool_activity, report_memory_activity, report_tool_activity, - set_permission_wait, ) -from band.runtime.execution import ExecutionContext from band.runtime.tools.agent import AgentTools from tests.activitysupport import ( IDLE_BODY, @@ -54,6 +54,8 @@ async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: PERMISSION_BODY, tool_body("Edit"), MEMORY_BODY, + # The reply is a Band tool call too; it ends back at reasoning. + REASONING_BODY, IDLE_BODY, ] diff --git a/tests/runtime/activityownership/test_exclusion.py b/tests/runtime/activityownership/test_exclusion.py index ffcd7ccf1..6e1c9c7f9 100644 --- a/tests/runtime/activityownership/test_exclusion.py +++ b/tests/runtime/activityownership/test_exclusion.py @@ -5,13 +5,13 @@ import pytest from band.platform.event import PlatformEvent -from band.runtime.activity import ( +from band.runtime.activity import set_permission_wait +from band.runtime.execution import ExecutionContext +from band.runtime.toolactivity import ( end_tool_activity, report_memory_activity, report_tool_activity, - set_permission_wait, ) -from band.runtime.execution import ExecutionContext from band.runtime.tools.agent import AgentTools from tests.conftest import make_message_event from tests.runtime.activityownership.kit import ActivityPlatform, past_a_keep_alive diff --git a/tests/runtime/activityownership/test_housekeeping_tools.py b/tests/runtime/activityownership/test_housekeeping_tools.py new file mode 100644 index 000000000..96cbf1aaf --- /dev/null +++ b/tests/runtime/activityownership/test_housekeeping_tools.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from collections.abc import Callable + +from band.platform.event import PlatformEvent +from band.runtime.execution import ExecutionContext +from band.runtime.tools.agent import AgentTools +from tests.conftest import make_participant_added_event +from tests.runtime.activityownership.kit import ActivityPlatform + + +class TestHousekeepingActivity: + async def test_band_tools_in_a_housekeeping_turn_never_start_reporting( + self, + execution: Callable[..., ExecutionContext], + activity_platform: ActivityPlatform, + ) -> None: + async def handler(ctx: ExecutionContext, event: PlatformEvent) -> None: + await AgentTools.from_context(ctx).get_participants() + + ctx = execution(handler) + await ctx._process_event(make_participant_added_event()) + await ctx._join_activity_completion() + assert activity_platform.activity_bodies == [] diff --git a/tests/runtime/activityownership/test_negotiation.py b/tests/runtime/activityownership/test_negotiation.py index 8194956a5..9958166ee 100644 --- a/tests/runtime/activityownership/test_negotiation.py +++ b/tests/runtime/activityownership/test_negotiation.py @@ -6,8 +6,8 @@ import pytest from band.platform.event import PlatformEvent -from band.runtime.activity import report_tool_activity from band.runtime.execution import ExecutionContext +from band.runtime.toolactivity import report_tool_activity from band.runtime.tools.agent import AgentTools from tests.activitysupport import ( BOOLEAN_BODY, diff --git a/tests/runtime/activityownership/test_nonowner.py b/tests/runtime/activityownership/test_nonowner.py index e13b7bc67..c12df5a5f 100644 --- a/tests/runtime/activityownership/test_nonowner.py +++ b/tests/runtime/activityownership/test_nonowner.py @@ -7,8 +7,9 @@ from band.core.turn_lifecycle import TurnTask, bind_turn_task from band.platform.event import PlatformEvent -from band.runtime.activity import report_tool_activity, set_permission_wait +from band.runtime.activity import set_permission_wait from band.runtime.execution import ExecutionContext +from band.runtime.toolactivity import report_tool_activity from band.runtime.tools.agent import AgentTools from tests.activitysupport import IDLE_BODY, REASONING_BODY, tool_body from tests.conftest import make_message_event diff --git a/tests/runtime/activityownership/test_opencode_admission.py b/tests/runtime/activityownership/test_opencode_admission.py index 08d147f46..6cffc95cf 100644 --- a/tests/runtime/activityownership/test_opencode_admission.py +++ b/tests/runtime/activityownership/test_opencode_admission.py @@ -72,12 +72,12 @@ async def test_a_message_during_an_approved_turns_reply_is_deferred_until_it_end assert activity_platform.marked("processed") == ["original"] assert activity_platform.marked("failed") == [] assert len(client.prompt_calls) == 1 - assert activity_platform.activity_reports == [KEEP_ALIVE] + assert activity_platform.reports_without_refreshes == [KEEP_ALIVE] activity_platform.release.set() await turn_finished(first_turn) await ctx._join_activity_completion() - assert activity_platform.activity_reports == [KEEP_ALIVE, IDLE] + assert activity_platform.reports_without_refreshes == [KEEP_ALIVE, IDLE] await ctx._reconcile() await ctx._join_activity_completion() @@ -88,7 +88,7 @@ async def test_a_message_during_an_approved_turns_reply_is_deferred_until_it_end FIRST_ANSWER, SECOND_ANSWER, ] - assert activity_platform.activity_reports == [ + assert activity_platform.reports_without_refreshes == [ KEEP_ALIVE, IDLE, KEEP_ALIVE, diff --git a/tests/runtime/helpers.py b/tests/runtime/helpers.py index 5d6112fac..a530b72e5 100644 --- a/tests/runtime/helpers.py +++ b/tests/runtime/helpers.py @@ -184,6 +184,17 @@ def __init__(self, *, stopped: bool = False, page_size: int | None = None) -> No self.actionable_cursors: list[str | None] = [] self.activity_reports: list[ActivityReport] = [] + @property + def reports_without_refreshes(self) -> list[ActivityReport]: + """The room's working-state reports with each run of keep-alive + refreshes collapsed; a repeated idle still shows twice.""" + reports = self.activity_reports + return [ + report + for index, report in enumerate(reports) + if not (report == KEEP_ALIVE and index and reports[index - 1] == KEEP_ALIVE) + ] + def add_message(self, message_id: str, *, content: str = "hello") -> None: self.messages.append( { diff --git a/tests/runtime/toolactivity/__init__.py b/tests/runtime/toolactivity/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/runtime/toolactivity/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/runtime/toolactivity/conftest.py b/tests/runtime/toolactivity/conftest.py new file mode 100644 index 000000000..ea8321b65 --- /dev/null +++ b/tests/runtime/toolactivity/conftest.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from band.runtime.tools import AgentTools +from band.testing.activity import ActivityRecorder +from tests.runtime.helpers import ROOM_ID + + +@pytest.fixture +def recorder() -> ActivityRecorder: + return ActivityRecorder() + + +@pytest.fixture +def tools(mock_rest_client: MagicMock, recorder: ActivityRecorder) -> AgentTools: + """Real Band tools whose turn observer is the recorder.""" + return recorder.observing(AgentTools(ROOM_ID, mock_rest_client)) diff --git a/tests/runtime/toolactivity/kit.py b/tests/runtime/toolactivity/kit.py new file mode 100644 index 000000000..ee384a3b2 --- /dev/null +++ b/tests/runtime/toolactivity/kit.py @@ -0,0 +1,72 @@ +"""Real tool boundaries the activity suites drive Band tools through.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +from band_rest.types.agent_activity_detail import AgentActivityDetail +from mcp.server.fastmcp import FastMCP + +from band.integrations.mcp.engine import ( + EmbeddedResolver, + EngineSpec, + build_engine, + build_tool_registration, + extend_with_chat_id, +) +from band.runtime.tools.types import ToolDefinition +from band.testing.activity import ActivityRecorder + + +def band_tool_server( + tools: Any, definition: ToolDefinition, *, request_ids: list[Any] | None = None +) -> FastMCP: + """An MCP engine serving ``definition`` over ``tools``, noting each request id.""" + + def resolve(room_id: str) -> Any: + if request_ids is not None: + request_ids.append(server.get_context().request_context.request_id) + return tools + + registration = build_tool_registration( + definition, + extend_with_chat_id(definition.input_model, None), + resolver=EmbeddedResolver(get_tools=resolve), + strip_chat_id=True, + ) + server = build_engine(EngineSpec(name="activity", tools=(registration,))) + return server + + +class UnreachableObserver(ActivityRecorder): + """Records each observation, then fails to deliver it.""" + + async def update(self, activity: AgentActivityDetail) -> None: + await super().update(activity) + raise RuntimeError("activity endpoint down") + + +class CancelledObserver(ActivityRecorder): + """A report that is cancelled before it lands.""" + + async def update(self, activity: AgentActivityDetail) -> None: + raise asyncio.CancelledError + + +class ConjuringProxy: + """Answers every attribute dynamically, as a mock or proxy would, and + conjures a recording observer it never declared.""" + + def __init__(self, conjured: ActivityRecorder) -> None: + self.conjured = conjured + self.events_sent: list[str] = [] + + def __getattr__(self, name: str) -> Any: + if name == "_activity_observer": + return self.conjured + raise AttributeError(name) + + async def send_event(self, content: str, message_type: str, **_: Any) -> str: + self.events_sent.append(content) + return "evt-1" diff --git a/tests/runtime/toolactivity/test_band_tools.py b/tests/runtime/toolactivity/test_band_tools.py new file mode 100644 index 000000000..637111096 --- /dev/null +++ b/tests/runtime/toolactivity/test_band_tools.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from band.runtime.tools import AgentTools +from band.runtime.tools.registry import TOOL_DEFINITIONS +from band.runtime.tools.types import BandTool, Surface +from band.testing.activity import ActivityRecorder +from tests.runtime.helpers import ROOM_ID + +USER_ONE = {"id": "user-1", "name": "User One", "type": "User", "handle": "user-one"} +EMPTY_PAGE = {"data": []} + + +class TestBandToolActivity: + @pytest.mark.parametrize( + ("name", "arguments"), + [ + (BandTool.GET_PARTICIPANTS, {}), + (BandTool.SEND_MESSAGE, {"content": "hi", "mentions": ["user-one"]}), + (BandTool.LIST_CONTACTS, {}), + (BandTool.LIST_ROOM_FILES, {}), + (BandTool.LIST_TASKS, {}), + ], + ids=["participants", "chat", "contacts", "files", "tasks"], + ) + async def test_a_band_tool_shows_its_name_until_it_ends( + self, + recorder: ActivityRecorder, + mock_rest_client: MagicMock, + name: BandTool, + arguments: dict[str, Any], + ) -> None: + mock_rest_client.agent_api_contacts.list_agent_contacts.return_value = ( + EMPTY_PAGE + ) + mock_rest_client.agent_api_chat_tasks.list_chat_tasks.return_value = EMPTY_PAGE + tools = recorder.observing( + AgentTools(ROOM_ID, mock_rest_client, participants=[USER_ONE]) + ) + outcome = await tools.execute_tool_call_structured(name, arguments) + assert outcome.ok, outcome.error_message + recorder.assert_completed_tool(name) + + def test_every_agent_tool_is_defined_where_its_decorators_wrap_it(self) -> None: + # The class decorators wrap only methods AgentTools itself defines; one + # inherited from a base class would run unreported. + inherited = { + definition.name + for definition in TOOL_DEFINITIONS.values() + if definition.surface == Surface.AGENT + and definition.method_name not in vars(AgentTools) + } + assert inherited == set() + + async def test_a_failing_band_tool_still_returns_to_reasoning( + self, + tools: AgentTools, + recorder: ActivityRecorder, + mock_rest_client: MagicMock, + ) -> None: + participants = mock_rest_client.agent_api_participants + participants.list_agent_chat_participants.side_effect = RuntimeError("down") + with pytest.raises(RuntimeError): + await tools.get_participants() + recorder.assert_completed_tool(BandTool.GET_PARTICIPANTS) + + async def test_adapter_telemetry_is_quiet( + self, tools: AgentTools, recorder: ActivityRecorder + ) -> None: + await tools.send_event("private telemetry", "thought") + assert recorder.events == [] + + async def test_a_model_sent_event_shows_as_a_tool_call( + self, tools: AgentTools, recorder: ActivityRecorder + ) -> None: + await tools.execute_tool_call( + BandTool.SEND_EVENT, + {"content": "model telemetry", "message_type": "thought"}, + ) + recorder.assert_completed_tool(BandTool.SEND_EVENT) diff --git a/tests/runtime/toolactivity/test_cancelled_tool.py b/tests/runtime/toolactivity/test_cancelled_tool.py new file mode 100644 index 000000000..a4f374cf3 --- /dev/null +++ b/tests/runtime/toolactivity/test_cancelled_tool.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import asyncio +from typing import Any +from unittest.mock import MagicMock + +from mcp import types +from mcp.shared.memory import create_connected_server_and_client_session + +from band.runtime.tools import AgentTools +from band.runtime.tools.registry import TOOL_DEFINITIONS +from band.runtime.tools.types import BandTool +from band.testing.activity import ActivityRecorder +from tests.runtime.helpers import ROOM_ID +from tests.runtime.toolactivity.kit import band_tool_server + + +def cancel_request(request_id: Any) -> types.ClientNotification: + return types.ClientNotification( + types.CancelledNotification( + params=types.CancelledNotificationParams(requestId=request_id) + ) + ) + + +class TestCancelledToolActivity: + async def test_a_cancelled_mcp_tool_call_returns_to_reasoning( + self, + tools: AgentTools, + recorder: ActivityRecorder, + mock_rest_client: MagicMock, + ) -> None: + entered, cancelled = asyncio.Event(), asyncio.Event() + + async def held(*_args: Any, **_kwargs: Any) -> Any: + entered.set() + try: + await asyncio.Event().wait() + finally: + cancelled.set() + + participants = mock_rest_client.agent_api_participants + participants.list_agent_chat_participants.side_effect = held + request_ids: list[Any] = [] + server = band_tool_server( + tools, TOOL_DEFINITIONS[BandTool.GET_PARTICIPANTS], request_ids=request_ids + ) + async with create_connected_server_and_client_session(server) as session: + request = asyncio.create_task( + session.call_tool(BandTool.GET_PARTICIPANTS, {"chat_id": ROOM_ID}) + ) + try: + await entered.wait() + await session.send_notification(cancel_request(request_ids[-1])) + await cancelled.wait() + finally: + request.cancel() + await asyncio.gather(request, return_exceptions=True) + + recorder.assert_completed_tool(BandTool.GET_PARTICIPANTS) diff --git a/tests/runtime/toolactivity/test_failed_report.py b/tests/runtime/toolactivity/test_failed_report.py new file mode 100644 index 000000000..e5b1e76e7 --- /dev/null +++ b/tests/runtime/toolactivity/test_failed_report.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import asyncio +import logging +from unittest.mock import MagicMock + +import pytest + +from band.runtime.tools import AgentTools +from band.runtime.tools.types import BandTool +from tests.runtime.helpers import ROOM_ID +from tests.runtime.toolactivity.kit import CancelledObserver, UnreachableObserver + + +@pytest.fixture +def unreachable() -> UnreachableObserver: + return UnreachableObserver() + + +@pytest.fixture +def tools(mock_rest_client: MagicMock, unreachable: UnreachableObserver) -> AgentTools: + return unreachable.observing(AgentTools(ROOM_ID, mock_rest_client)) + + +class TestFailedActivityReport: + """Activity is best-effort: a failed report never changes the tool's outcome.""" + + async def test_the_tool_still_runs_and_returns_its_result( + self, + tools: AgentTools, + unreachable: UnreachableObserver, + mock_rest_client: MagicMock, + caplog: pytest.LogCaptureFixture, + ) -> None: + listing = mock_rest_client.agent_api_participants.list_agent_chat_participants + with caplog.at_level(logging.WARNING, logger="band.runtime.toolactivity"): + participants = await tools.get_participants() + + assert participants == listing.return_value.data + unreachable.assert_completed_tool(BandTool.GET_PARTICIPANTS) + assert [record.getMessage() for record in caplog.records] == [ + "Could not report tool_call activity", + "Could not report reasoning activity", + ] + + async def test_the_tool_keeps_its_own_failure( + self, tools: AgentTools, mock_rest_client: MagicMock + ) -> None: + listing = mock_rest_client.agent_api_participants.list_agent_chat_participants + listing.side_effect = ValueError("room gone") + with pytest.raises(ValueError, match="room gone"): + await tools.get_participants() + + async def test_a_cancelled_report_still_cancels_the_tool( + self, mock_rest_client: MagicMock + ) -> None: + tools = CancelledObserver().observing(AgentTools(ROOM_ID, mock_rest_client)) + with pytest.raises(asyncio.CancelledError): + await tools.get_participants() + listing = mock_rest_client.agent_api_participants.list_agent_chat_participants + listing.assert_not_awaited() diff --git a/tests/runtime/toolactivity/test_memory_tools.py b/tests/runtime/toolactivity/test_memory_tools.py new file mode 100644 index 000000000..b9fba28ac --- /dev/null +++ b/tests/runtime/toolactivity/test_memory_tools.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from band.runtime.tools import AgentTools +from band.runtime.tools.types import BandTool +from band.testing.activity import SHOWN_MEMORY, SHOWN_REASONING, ActivityRecorder + +MEMORY_ID = "00000000-0000-0000-0000-000000000001" + +ToolCall = Callable[[AgentTools], Awaitable[Any]] + + +def store(tools: AgentTools) -> Awaitable[Any]: + return tools.store_memory( + content="remember this", + system="working", + type="semantic", + segment="user", + thought="useful later", + scope="organization", + ) + + +SAVES = [ + (store, "create_agent_memory"), + (lambda tools: tools.supersede_memory(MEMORY_ID), "supersede_agent_memory"), + (lambda tools: tools.archive_memory(MEMORY_ID), "archive_agent_memory"), +] +SAVE_IDS = ["store", "supersede", "archive"] + + +class TestMemoryToolActivity: + @pytest.mark.parametrize(("call", "rest_method"), SAVES, ids=SAVE_IDS) + async def test_saving_memory_shows_memory_then_reasoning( + self, + tools: AgentTools, + recorder: ActivityRecorder, + mock_rest_client: MagicMock, + call: ToolCall, + rest_method: str, + ) -> None: + await call(tools) + getattr(mock_rest_client.agent_api_memories, rest_method).assert_awaited_once() + assert recorder.events == [SHOWN_MEMORY, SHOWN_REASONING] + + @pytest.mark.parametrize(("call", "rest_method"), SAVES, ids=SAVE_IDS) + async def test_a_failed_save_still_returns_to_reasoning( + self, + tools: AgentTools, + recorder: ActivityRecorder, + mock_rest_client: MagicMock, + call: ToolCall, + rest_method: str, + ) -> None: + save = getattr(mock_rest_client.agent_api_memories, rest_method) + save.side_effect = RuntimeError("memory store down") + with pytest.raises(RuntimeError, match="memory store down"): + await call(tools) + assert recorder.events == [SHOWN_MEMORY, SHOWN_REASONING] + + @pytest.mark.parametrize( + ("name", "call"), + [ + (BandTool.GET_MEMORY, lambda tools: tools.get_memory(MEMORY_ID)), + (BandTool.LIST_MEMORIES, lambda tools: tools.list_memories()), + ], + ids=["get", "list"], + ) + async def test_reading_memory_shows_the_tool_call( + self, + tools: AgentTools, + recorder: ActivityRecorder, + name: BandTool, + call: ToolCall, + ) -> None: + await call(tools) + recorder.assert_completed_tool(name) diff --git a/tests/runtime/toolactivity/test_nested_tools.py b/tests/runtime/toolactivity/test_nested_tools.py new file mode 100644 index 000000000..bb5732056 --- /dev/null +++ b/tests/runtime/toolactivity/test_nested_tools.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from band.runtime.tools import AgentTools +from band.runtime.tools.types import BandTool +from band.testing.activity import SHOWN_REASONING, ActivityRecorder, shown_tool + + +class TestNestedToolActivity: + """A Band tool that calls other Band tools shows only itself.""" + + @pytest.mark.parametrize( + ("name", "call", "rest_method"), + [ + ( + BandTool.ADD_PARTICIPANT, + lambda tools: tools.add_participant("Agent Two"), + "add_agent_chat_participant", + ), + ( + BandTool.REMOVE_PARTICIPANT, + lambda tools: tools.remove_participant("User One"), + "remove_agent_chat_participant", + ), + ], + ids=["add", "remove"], + ) + async def test_the_change_runs_under_the_called_tool( + self, + tools: AgentTools, + recorder: ActivityRecorder, + mock_rest_client: MagicMock, + name: BandTool, + call: Callable[[AgentTools], Awaitable[Any]], + rest_method: str, + ) -> None: + participants = mock_rest_client.agent_api_participants + change = getattr(participants, rest_method) + shown_during_change: list[dict[str, Any]] = [] + answer = change.return_value + + async def record_change(*_args: Any, **_kwargs: Any) -> Any: + shown_during_change.extend(recorder.events) + return answer + + change.side_effect = record_change + await call(tools) + + assert shown_during_change == [shown_tool(name)] + assert recorder.events == [shown_tool(name), SHOWN_REASONING] diff --git a/tests/runtime/toolactivity/test_rebound_tools.py b/tests/runtime/toolactivity/test_rebound_tools.py new file mode 100644 index 000000000..107b47062 --- /dev/null +++ b/tests/runtime/toolactivity/test_rebound_tools.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import asyncio +from typing import Any +from unittest.mock import MagicMock + +from band.runtime.tools import AgentTools +from band.runtime.tools.types import BandTool +from band.testing.activity import ActivityRecorder + + +class TestReboundToolActivity: + async def test_a_tool_ends_on_the_turn_it_started_on( + self, + tools: AgentTools, + recorder: ActivityRecorder, + mock_rest_client: MagicMock, + ) -> None: + entered, release = asyncio.Event(), asyncio.Event() + listing = mock_rest_client.agent_api_participants.list_agent_chat_participants + answer = listing.return_value + + async def held(*_args: Any, **_kwargs: Any) -> Any: + entered.set() + await release.wait() + return answer + + listing.side_effect = held + call = asyncio.create_task(tools.get_participants()) + await entered.wait() + next_turn = ActivityRecorder() + next_turn.observing(tools) + release.set() + await call + + recorder.assert_completed_tool(BandTool.GET_PARTICIPANTS) + assert next_turn.events == [] diff --git a/tests/runtime/toolactivity/test_turn_sequence.py b/tests/runtime/toolactivity/test_turn_sequence.py new file mode 100644 index 000000000..f60914af7 --- /dev/null +++ b/tests/runtime/toolactivity/test_turn_sequence.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import MagicMock + +from band.runtime.tools import AgentTools +from band.runtime.tools.types import BandTool +from band.testing.activity import ( + SHOWN_MEMORY, + SHOWN_REASONING, + ActivityRecorder, + shown_tool, +) +from tests.runtime.helpers import ROOM_ID + +USER_ONE = {"id": "user-1", "name": "User One", "type": "User", "handle": "user-one"} +MEMORY = { + "content": "User One prefers short answers", + "system": "long_term", + "type": "semantic", + "segment": "user", + "thought": "shapes later replies", + "scope": "organization", +} + + +class TestTurnSequenceActivity: + async def test_a_turn_shows_each_tool_in_the_order_the_model_called_it( + self, recorder: ActivityRecorder, mock_rest_client: MagicMock + ) -> None: + memories = mock_rest_client.agent_api_memories + memories.create_agent_memory.return_value = SimpleNamespace( + data={"id": "memory-1"} + ) + tools = recorder.observing( + AgentTools(ROOM_ID, mock_rest_client, participants=[USER_ONE]) + ) + + calls = [ + (BandTool.LOOKUP_PEERS, {}), + (BandTool.SEND_MESSAGE, {"content": "Found one", "mentions": ["user-one"]}), + (BandTool.STORE_MEMORY, MEMORY), + ] + for name, arguments in calls: + outcome = await tools.execute_tool_call_structured(name, arguments) + assert outcome.ok, outcome.error_message + + assert recorder.events == [ + shown_tool(BandTool.LOOKUP_PEERS), + SHOWN_REASONING, + shown_tool(BandTool.SEND_MESSAGE), + SHOWN_REASONING, + SHOWN_MEMORY, + SHOWN_REASONING, + ] diff --git a/tests/runtime/toolactivity/test_undeclared_observer.py b/tests/runtime/toolactivity/test_undeclared_observer.py new file mode 100644 index 000000000..c1f623013 --- /dev/null +++ b/tests/runtime/toolactivity/test_undeclared_observer.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from band.integrations.claude_sdk.dedup_tools import DedupingAgentTools +from band.integrations.mcp.engine import dispatch_tool +from band.runtime.tools import AgentTools +from band.runtime.tools.registry import TOOL_DEFINITIONS +from band.runtime.tools.types import BandTool +from band.testing.activity import ActivityRecorder +from tests.runtime.toolactivity.kit import ConjuringProxy + +SEND_EVENT = TOOL_DEFINITIONS[BandTool.SEND_EVENT] +ARGUMENTS = {"content": "model", "message_type": "thought"} + + +class TestUndeclaredObserver: + async def test_a_proxy_reports_through_the_tools_it_wraps( + self, tools: AgentTools, recorder: ActivityRecorder + ) -> None: + await dispatch_tool(DedupingAgentTools(tools), SEND_EVENT, ARGUMENTS) + recorder.assert_completed_tool(BandTool.SEND_EVENT) + + async def test_tools_without_a_declared_observer_run_unreported(self) -> None: + conjured = ActivityRecorder() + proxy = ConjuringProxy(conjured) + + result = await dispatch_tool(proxy, SEND_EVENT, ARGUMENTS) + + assert result == "evt-1" + assert proxy.events_sent == ["model"] + assert conjured.events == [] diff --git a/tests/runtime/tooltests/test_agent_tools_context_sync_back.py b/tests/runtime/tooltests/test_agent_tools_context_sync_back.py index fa6acd8ee..7d317c3b2 100644 --- a/tests/runtime/tooltests/test_agent_tools_context_sync_back.py +++ b/tests/runtime/tooltests/test_agent_tools_context_sync_back.py @@ -20,64 +20,6 @@ class TestAgentToolsContextSyncBack: On the next turn, from_context() would copy the stale list again. """ - @pytest.mark.asyncio - async def test_add_participant_syncs_back_to_ctx(self, mock_rest_client): - """add_participant() must call ctx.add_participant() with full dict.""" - mock_ctx = MagicMock() - mock_ctx.room_id = "room-456" - mock_ctx.link = MagicMock() - mock_ctx.link.rest = mock_rest_client - mock_ctx.participants = [] - mock_ctx.hub_room_id = None - mock_ctx.add_participant = MagicMock() - - # Override list_agent_chat_participants to return empty (avoid "already_in_room") - mock_rest_client.agent_api_participants.list_agent_chat_participants = ( - AsyncMock(return_value=MagicMock(data=[])) - ) - - tools = AgentTools.from_context(mock_ctx) - assert tools._ctx is mock_ctx - - await tools.add_participant("Agent Two") - - mock_ctx.add_participant.assert_called_once() - added = mock_ctx.add_participant.call_args.args[0] - assert added["id"] == "agent-2" - assert added["name"] == "Agent Two" - assert added["type"] == "Agent" - assert added["handle"] == "agent-two" - assert added["description"] == "Another agent" - - @pytest.mark.asyncio - async def test_remove_participant_syncs_back_to_ctx(self, mock_rest_client): - """remove_participant() must call ctx.remove_participant() with correct ID.""" - participant = { - "id": "user-1", - "name": "User One", - "type": "User", - "handle": "user-one", - } - - mock_ctx = MagicMock() - mock_ctx.room_id = "room-456" - mock_ctx.link = MagicMock() - mock_ctx.link.rest = mock_rest_client - mock_ctx.participants = [participant] - mock_ctx.hub_room_id = None - mock_ctx.remove_participant = MagicMock() - - # Return same participant from REST so snapshot matches - p_mock = make_participant_mock("user-1", "User One", "User", handle="user-one") - mock_rest_client.agent_api_participants.list_agent_chat_participants = ( - AsyncMock(return_value=MagicMock(data=[p_mock])) - ) - - tools = AgentTools.from_context(mock_ctx) - await tools.remove_participant("User One") - - mock_ctx.remove_participant.assert_called_once_with("user-1") - @pytest.mark.asyncio async def test_add_participant_persists_across_recreated_tools( self, mock_rest_client @@ -103,8 +45,17 @@ async def test_add_participant_persists_across_recreated_tools( tools1 = AgentTools.from_context(ctx) await tools1.add_participant("Agent Two") + added = ctx.participants[0] assert len(ctx.participants) == 1 - assert ctx.participants[0]["id"] == "agent-2" + assert { + key: added[key] for key in ("id", "name", "type", "handle", "description") + } == { + "id": "agent-2", + "name": "Agent Two", + "type": "Agent", + "handle": "agent-two", + "description": "Another agent", + } # Turn 2: recreate tools — participant must still be there tools2 = AgentTools.from_context(ctx)