diff --git a/CHANGELOG.md b/CHANGELOG.md index bed2c0e35e..9de5ea9fe5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,18 @@ Composio Airtable", then "Airtable list bases". It now says "Used Airtable", and the action, "List bases", with the app beside it. An Arcade app is named the same way, as the app rather than `arcade-Gmail`. +### An app a Bot cannot use yet asks once per turn + +When a Bot's call to an unconnected app was refused, the Bot often also asked to show that app's +card, and the conversation drew the same Connect card twice. Each app's card is now drawn once per +turn; asking again in a new message can draw it again. + +### The Computer panel shows the Bot's screen when it opens + +Opening the Computer panel beside a conversation could leave it on "Waiting for the Bot's screen…" +while the Bot browsed, because the panel slid in from off-screen and the screen counted as out of +view for as long as it stayed open. The panel now starts showing the screen once it has opened. + ### Plainer words, and one word for each thing OpenBot now says **Bot** everywhere a person reads it, where screens used to mix Bot, agent, coworker diff --git a/app/src/components/channels/chat-transcript.tsx b/app/src/components/channels/chat-transcript.tsx index 6cf6cc805d..b439b5c5d7 100644 --- a/app/src/components/channels/chat-transcript.tsx +++ b/app/src/components/channels/chat-transcript.tsx @@ -66,6 +66,7 @@ import { import { cn } from "@/lib/utils"; import { VOICE_CHAT_ACTIVITY, type VoiceChatEntry } from "@/lib/voice/archive"; import { + dropRepeatedAppCards, groupToolSteps, ToolActivity, useDrawnTool, @@ -1560,7 +1561,10 @@ export function ChatTranscript({ * cost was markdown parsing and chart SVGs, and those are skipped by the memoised children below, * which is where the 25x came from. This runs per render and is not worth guarding. */ - const items = groupToolSteps(toVisibleChatItems(messages), isDrawn); + const items = groupToolSteps( + dropRepeatedAppCards(toVisibleChatItems(messages)), + isDrawn, + ); /* * ONLY WHILE THERE IS NOTHING ELSE TO LOOK AT. Once a reply starts streaming, or a tool line diff --git a/app/src/components/channels/tool-activity.tsx b/app/src/components/channels/tool-activity.tsx index 7e01601da7..92fdc3f71b 100644 --- a/app/src/components/channels/tool-activity.tsx +++ b/app/src/components/channels/tool-activity.tsx @@ -141,6 +141,42 @@ function asksThePerson(result: string | undefined): boolean { return readTeamBotConsent(body) !== null || readConnectPrompt(body) !== null; } +/** The app a tool item draws a card for, or null when it draws none. */ +function cardApp(item: ToolItem): string | null { + if (item.result === undefined) return null; + const answer = asText(item.result); + const prompt = answer.startsWith(REFUSAL_MARKER) + ? readConnectPrompt(answer.slice(REFUSAL_MARKER.length).trim()) + : item.toolCall.function.name === SHOW_APP_ACCESS_TOOL + ? readConnectPrompt(answer) + : null; + return prompt?.serverId ?? null; +} + +/** + * One card per app per turn. + * + * A refused call already draws the app's card, and a Bot told to show the card for an app it cannot + * use often calls `show_app_access` for the same app straight after, which drew the same card again. + * The refused call is kept, because it is what the Bot did; a later `show_app_access` for an app + * whose card that turn already drew is dropped. A person's next message starts a new turn. + */ +export function dropRepeatedAppCards( + items: readonly VisibleChatItem[], +): VisibleChatItem[] { + const drawn = new Set(); + return items.filter((item) => { + if (item.kind === "text" && item.role === "user") drawn.clear(); + if (item.kind !== "tool") return true; + const app = cardApp(item); + if (app === null) return true; + const repeat = + drawn.has(app) && item.toolCall.function.name === SHOW_APP_ACCESS_TOOL; + drawn.add(app); + return !repeat; + }); +} + type StepKind = "search" | "fetch" | "browser" | "connector" | "tool"; export type StepDetails = { diff --git a/app/src/components/computer/preview-visibility.ts b/app/src/components/computer/preview-visibility.ts index f8a480432a..9ff323934f 100644 --- a/app/src/components/computer/preview-visibility.ts +++ b/app/src/components/computer/preview-visibility.ts @@ -34,8 +34,10 @@ export function useElementVisible(): [ setVisible(true); return; } - const observer = new IntersectionObserver(([entry]) => { - setVisible(Boolean(entry?.isIntersecting)); + // The newest entry is the element as it is now. A panel sliding open delivers the frame it began + // off-screen and the frame it arrived in together, and reading the first left it "hidden". + const observer = new IntersectionObserver((entries) => { + setVisible(Boolean(entries.at(-1)?.isIntersecting)); }); observer.observe(node); return () => observer.disconnect(); diff --git a/app/tests/preview-visibility.test.tsx b/app/tests/preview-visibility.test.tsx new file mode 100644 index 0000000000..1c328f7fc8 --- /dev/null +++ b/app/tests/preview-visibility.test.tsx @@ -0,0 +1,60 @@ +import { afterAll, afterEach, beforeAll, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; +import { act, cleanup, renderHook } from "@testing-library/react"; +import { useElementVisible } from "../src/components/computer/preview-visibility"; + +beforeAll(() => GlobalRegistrator.register()); +afterEach(cleanup); +afterAll(() => GlobalRegistrator.unregister()); + +type Callback = (entries: Partial[]) => void; + +/** An observer the test fires by hand, as the browser would. */ +function fakeObserver() { + const callbacks: Callback[] = []; + const original = window.IntersectionObserver; + window.IntersectionObserver = class { + constructor(callback: Callback) { + callbacks.push(callback); + } + observe() {} + unobserve() {} + disconnect() {} + takeRecords() { + return []; + } + } as unknown as typeof IntersectionObserver; + return { + fire: (entries: Partial[]) => + act(() => callbacks.at(-1)?.(entries)), + restore: () => { + window.IntersectionObserver = original; + }, + }; +} + +test("an element that slid into view during one batch counts as visible", () => { + // A panel animating open reports the frame it started off-screen and the frame it arrived in + // together. The newest entry is the element as it is now; the first one is history. + const observer = fakeObserver(); + try { + const { result } = renderHook(() => useElementVisible()); + act(() => result.current[0](document.createElement("figure"))); + observer.fire([{ isIntersecting: false }, { isIntersecting: true }]); + expect(result.current[1]).toBe(true); + } finally { + observer.restore(); + } +}); + +test("an element that left view during one batch counts as hidden", () => { + const observer = fakeObserver(); + try { + const { result } = renderHook(() => useElementVisible()); + act(() => result.current[0](document.createElement("figure"))); + observer.fire([{ isIntersecting: true }, { isIntersecting: false }]); + expect(result.current[1]).toBe(false); + } finally { + observer.restore(); + } +}); diff --git a/app/tests/tool-activity.test.tsx b/app/tests/tool-activity.test.tsx index 123ad4794c..1e987b797f 100644 --- a/app/tests/tool-activity.test.tsx +++ b/app/tests/tool-activity.test.tsx @@ -6,6 +6,7 @@ import { CopilotKitContext } from "@copilotkit/react-core/v2/context"; import type { ReactNode } from "react"; import type { VisibleChatItem } from "../src/components/channels/chat-messages"; import { + dropRepeatedAppCards, groupToolSteps, stepDetails, summarise, @@ -300,3 +301,29 @@ test("while running, the line names the step in progress; opened, a step shows i expect(view.getByText("query")).toBeDefined(); expect(view.getAllByText("top 5 GDP 2025").length).toBeGreaterThan(0); }); + +test("an app's card is drawn once a turn, even when the Bot also asks to show it", () => { + const refusal = JSON.stringify( + 'Refused. [connect-app]{"serverId":"composio-airtable","title":"Airtable","agentId":"scout"} Connect Airtable and ask again.', + ); + const shown = JSON.stringify( + '[connect-app]{"serverId":"composio-airtable","title":"Airtable","agentId":"scout"} Connect Airtable.', + ); + const items: VisibleChatItem[] = [ + { kind: "text", id: "q1", role: "user", text: "List my bases" }, + call("list", "mcp__composio-airtable__AIRTABLE_LIST_BASES", {}, refusal), + call("show", "show_app_access", { app: "airtable" }, shown), + text("a1"), + { kind: "text", id: "q2", role: "user", text: "Try again" }, + call("show2", "show_app_access", { app: "airtable" }, shown), + ]; + // The refused call keeps its card; the second card for the same app in that turn is dropped, and + // a new turn may draw it again. + expect(dropRepeatedAppCards(items).map((item) => item.id)).toEqual([ + "q1", + "list", + "a1", + "q2", + "show2", + ]); +});