From ce1f32ab272e0f189b6b1a2f1077cf99aa8956ba Mon Sep 17 00:00:00 2001 From: Nickk'z Date: Wed, 28 Oct 2026 01:26:07 +0530 Subject: [PATCH] Announce replies, stops and approvals to screen readers The running footer (role="status") and the thread error banner (role="alert") already tell screen reader users when a turn is running or has failed, but nothing announced a finished reply, a user stop, or a newly waiting approval or question. Closes #350. Add one visually hidden polite live region to the conversation that speaks a short sentence for each of those events, once. What to say is a pure function (agent-announcement.ts) picked from a before/after snapshot, so it never streams tokens and is unit tested without a DOM. The region (agent-announcer.tsx) re-reads an identical sentence via a keyed span, and conversation-announcer.tsx maps thread state to it and keys it on the thread root so switching threads resets it. Real failures stay silent here so the error banner remains their sole announcement. Both approval cards already route their error through the shared ApprovalCard, so one role="alert" there covers both lanes. A /lab section drives the announcer so it can be heard with VoiceOver or NVDA. --- frontend/app/lab/announcer-showcase.tsx | 103 ++++++++++++++++ frontend/app/lab/component-lab.tsx | 5 + frontend/components/ai/approval-card.test.tsx | 2 + frontend/components/ai/approval-card.tsx | 7 +- .../chat/agent-announcement.test.ts | 88 ++++++++++++++ .../components/chat/agent-announcement.ts | 64 ++++++++++ .../chat/agent-announcer.interaction.test.tsx | 94 +++++++++++++++ frontend/components/chat/agent-announcer.tsx | 52 ++++++++ ...onversation-announcer.interaction.test.tsx | 113 ++++++++++++++++++ .../chat/conversation-announcer.tsx | 51 ++++++++ frontend/components/chat/conversation.tsx | 9 ++ 11 files changed, 587 insertions(+), 1 deletion(-) create mode 100644 frontend/app/lab/announcer-showcase.tsx create mode 100644 frontend/components/chat/agent-announcement.test.ts create mode 100644 frontend/components/chat/agent-announcement.ts create mode 100644 frontend/components/chat/agent-announcer.interaction.test.tsx create mode 100644 frontend/components/chat/agent-announcer.tsx create mode 100644 frontend/components/chat/conversation-announcer.interaction.test.tsx create mode 100644 frontend/components/chat/conversation-announcer.tsx diff --git a/frontend/app/lab/announcer-showcase.tsx b/frontend/app/lab/announcer-showcase.tsx new file mode 100644 index 00000000..79bc4a3e --- /dev/null +++ b/frontend/app/lab/announcer-showcase.tsx @@ -0,0 +1,103 @@ +"use client"; + +// Lab playground for the announcer (#350): fire each event and see what it +// speaks. Turn on VoiceOver or NVDA to actually hear it. Real failures are +// silent here on purpose - the error banner handles those. + +import { useRef, useState } from "react"; +import { Button } from "@/components/base/buttons/button"; +import { type AgentActivity, agentAnnouncement } from "@/components/chat/agent-announcement"; +import { AgentAnnouncer } from "@/components/chat/agent-announcer"; + +const IDLE_ACTIVITY: AgentActivity = { + running: false, + outcome: null, + approvalId: null, + questionId: null, + pendingApprovalIds: [], +}; + +export function AnnouncerShowcase() { + const [activity, setActivity] = useState({ ...IDLE_ACTIVITY, running: true }); + const [spoken, setSpoken] = useState([]); + const nextApproval = useRef(0); + + const apply = (next: AgentActivity) => { + const message = agentAnnouncement(activity, next); + if (message) setSpoken((log) => [message, ...log].slice(0, 6)); + setActivity(next); + }; + + return ( +
+ +
+ + + + + +
+
+

+ Announced (newest first) - a real failure is intentionally silent here, the error banner + owns it: +

+ {spoken.length > 0 ? ( +
    + {spoken.map((line, index) => ( +
  • + {line} +
    • + ))} +
+ ) : ( +

Nothing announced yet.

+ )} +
+
+ ); +} diff --git a/frontend/app/lab/component-lab.tsx b/frontend/app/lab/component-lab.tsx index c18cb589..e0a086a3 100644 --- a/frontend/app/lab/component-lab.tsx +++ b/frontend/app/lab/component-lab.tsx @@ -55,6 +55,7 @@ import * as Modal from "@/components/base/modal/modal"; import { ARTIFACT_CAPABILITY_ROWS } from "./artifact-capability-matrix"; import { BeautifulUiExtras } from "./beautiful-ui-extras"; import { BeuiAgentShowcase } from "./beui-agent-showcase"; +import { AnnouncerShowcase } from "./announcer-showcase"; import { ComposerRetrievalShowcase } from "./composer-retrieval-showcase"; import { ModelPickerShowcase } from "./model-picker-showcase"; import { ProductTourShowcase } from "./product-tour-showcase"; @@ -506,6 +507,10 @@ export function ComponentLab() { +
+ +
+
{ test("error renders the actionable error line (the submission-error state)", () => { const html = render({ error: "Could not send your decision - try again" }); expect(html).toContain("Could not send your decision - try again"); + // The error line is a live alert so a screen reader hears it at once (#350). + expect(html).toContain('role="alert"'); }); test("note and chip render when supplied; reason slot renders its content", () => { diff --git a/frontend/components/ai/approval-card.tsx b/frontend/components/ai/approval-card.tsx index a25a9fc5..5ead493b 100644 --- a/frontend/components/ai/approval-card.tsx +++ b/frontend/components/ai/approval-card.tsx @@ -156,7 +156,12 @@ export function ApprovalCard({ {body} {note &&

{note}

} - {error &&

{error}

} + {/* Both lanes send their error here, so role="alert" reads it out once (#350). */} + {error && ( +

+ {error} +

+ )} {reason} diff --git a/frontend/components/chat/agent-announcement.test.ts b/frontend/components/chat/agent-announcement.test.ts new file mode 100644 index 00000000..985540d1 --- /dev/null +++ b/frontend/components/chat/agent-announcement.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, test } from "bun:test"; +import { type AgentActivity, agentAnnouncement } from "./agent-announcement"; + +const IDLE: AgentActivity = { + running: false, + outcome: null, + approvalId: null, + questionId: null, + pendingApprovalIds: [], +}; + +const running = (over: Partial = {}): AgentActivity => ({ + ...IDLE, + running: true, + ...over, +}); + +describe("agentAnnouncement", () => { + test("the first snapshot establishes a baseline and announces nothing", () => { + // Opening a thread whose agent already replied must not read a stale event. + expect(agentAnnouncement(null, { ...IDLE, outcome: "replied" })).toBeNull(); + expect(agentAnnouncement(null, { ...IDLE, approvalId: "a1" })).toBeNull(); + }); + + test("a finished reply is announced once, on the running to settled edge", () => { + const before = running(); + const after: AgentActivity = { ...IDLE, outcome: "replied" }; + expect(agentAnnouncement(before, after)).toBe("The agent replied."); + // The same settled snapshot repeated is not a new event. + expect(agentAnnouncement(after, after)).toBeNull(); + }); + + test("a user stop is announced, distinct from a reply", () => { + expect(agentAnnouncement(running(), { ...IDLE, outcome: "stopped" })).toBe( + "The agent stopped.", + ); + }); + + test("a real failure is left to the error banner (never announced here)", () => { + // A real failure comes in as outcome null, so we say nothing here. + expect(agentAnnouncement(running(), { ...IDLE, outcome: null })).toBeNull(); + }); + + test("a new native approval is announced when it appears", () => { + expect(agentAnnouncement(running(), running({ approvalId: "ap1" }))).toBe( + "The agent needs your approval.", + ); + }); + + test("a resolved approval is not re-announced", () => { + const waiting = running({ approvalId: "ap1" }); + expect(agentAnnouncement(waiting, running())).toBeNull(); + }); + + test("a different approval replacing the last one counts as a new event", () => { + const first = running({ approvalId: "ap1" }); + expect(agentAnnouncement(first, running({ approvalId: "ap2" }))).toBe( + "The agent needs your approval.", + ); + }); + + test("a newly pending gateway approval is announced", () => { + const before = running({ pendingApprovalIds: ["g1"] }); + expect(agentAnnouncement(before, running({ pendingApprovalIds: ["g1", "g2"] }))).toBe( + "The agent needs your approval.", + ); + // A pending approval already present at baseline is not a fresh event. + expect(agentAnnouncement(before, before)).toBeNull(); + }); + + test("a new native question is announced", () => { + expect(agentAnnouncement(running(), running({ questionId: "q1" }))).toBe( + "The agent has a question.", + ); + }); + + test("a request the person must act on wins over a settled outcome", () => { + // Both edges in one transition: the blocking request is what to hear. + const before = running(); + const after: AgentActivity = { ...IDLE, outcome: "replied", approvalId: "ap1" }; + expect(agentAnnouncement(before, after)).toBe("The agent needs your approval."); + }); + + test("nothing new produces no announcement", () => { + expect(agentAnnouncement(running(), running())).toBeNull(); + expect(agentAnnouncement(IDLE, IDLE)).toBeNull(); + }); +}); diff --git a/frontend/components/chat/agent-announcement.ts b/frontend/components/chat/agent-announcement.ts new file mode 100644 index 00000000..8a49d960 --- /dev/null +++ b/frontend/components/chat/agent-announcement.ts @@ -0,0 +1,64 @@ +// #350: picks what a screen reader should hear - a reply finishing, a user +// stop, or a new approval/question. Pure so we can unit-test it without a DOM. +// Running and real failures are left out on purpose: the running footer and the +// error banner already announce those. + +/** completed = "replied", a user cancel = "stopped". A real failure is left to + * the error banner, so it isn't here. */ +export type TurnOutcome = "replied" | "stopped"; + +export interface AgentActivity { + readonly running: boolean; + /** Newest settled turn's outcome, or null while running or on a real failure. */ + readonly outcome: TurnOutcome | null; + /** Waiting native approval / question ids, null when none waits. */ + readonly approvalId: string | null; + readonly questionId: string | null; + /** Gateway (tool) approvals currently waiting. */ + readonly pendingApprovalIds: readonly string[]; +} + +const MESSAGES = { + replied: "The agent replied.", + stopped: "The agent stopped.", + approval: "The agent needs your approval.", + question: "The agent has a question.", +} as const; + +/** A request id that wasn't there before - so we don't re-announce one that's + * just sitting there or getting resolved. */ +function appeared(prevId: string | null, nextId: string | null): boolean { + return nextId !== null && nextId !== prevId; +} + +function gatewayAppeared(prev: readonly string[], next: readonly string[]): boolean { + const before = new Set(prev); + return next.some((id) => !before.has(id)); +} + +/** + * What to announce going from `prev` to `next`, or null if nothing's new. First + * snapshot (prev === null) stays quiet so opening an old thread doesn't replay + * its last event. Something the user has to act on wins over a settled turn. + */ +export function agentAnnouncement(prev: AgentActivity | null, next: AgentActivity): string | null { + if (!prev) return null; + + // Waiting on the user comes first - that's what's blocking them now. + if ( + appeared(prev.approvalId, next.approvalId) || + gatewayAppeared(prev.pendingApprovalIds, next.pendingApprovalIds) + ) { + return MESSAGES.approval; + } + if (appeared(prev.questionId, next.questionId)) { + return MESSAGES.question; + } + + // Turn just settled. + if (next.outcome !== null && next.outcome !== prev.outcome) { + return MESSAGES[next.outcome]; + } + + return null; +} diff --git a/frontend/components/chat/agent-announcer.interaction.test.tsx b/frontend/components/chat/agent-announcer.interaction.test.tsx new file mode 100644 index 00000000..412aa27e --- /dev/null +++ b/frontend/components/chat/agent-announcer.interaction.test.tsx @@ -0,0 +1,94 @@ +// Mounts the real AgentAnnouncer in happy-dom and drives the props through the +// transitions a thread goes through, checking the live region off the DOM. The +// pure rule is unit-tested separately; this covers the region itself - polite +// status, one sentence per event, and re-reading an identical one. +import { afterEach, describe, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; + +if (typeof window === "undefined") GlobalRegistrator.register(); + +(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +const { act } = await import("react"); +const { createRoot } = await import("react-dom/client"); +const { AgentAnnouncer } = await import("./agent-announcer"); + +import type { ComponentProps } from "react"; + +type Props = ComponentProps; + +const IDLE: Props = { + running: false, + outcome: null, + approvalId: null, + questionId: null, + pendingApprovalIds: [], +}; + +let root: ReturnType | null = null; +let container: HTMLElement | null = null; + +function mount(props: Props) { + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + act(() => root?.render()); +} + +function rerender(props: Props) { + act(() => root?.render()); +} + +function region(): HTMLElement { + const el = container?.querySelector('[role="status"]') as HTMLElement | null; + if (!el) throw new Error("live region did not render"); + return el; +} + +afterEach(() => { + if (root) act(() => root?.unmount()); + container?.remove(); + root = null; + container = null; +}); + +describe("AgentAnnouncer", () => { + test("renders a visually hidden polite, atomic status region", () => { + mount(IDLE); + const el = region(); + expect(el.getAttribute("aria-live")).toBe("polite"); + expect(el.getAttribute("aria-atomic")).toBe("true"); + expect(el.className).toContain("sr-only"); + // Nothing is announced on the baseline render. + expect(el.textContent).toBe(""); + }); + + test("announces a finished reply once, not the baseline", () => { + mount({ ...IDLE, running: true }); + expect(region().textContent).toBe(""); + rerender({ ...IDLE, outcome: "replied" }); + expect(region().textContent).toBe("The agent replied."); + }); + + test("announces a newly waiting approval, and never streams tokens", () => { + mount({ ...IDLE, running: true }); + rerender({ ...IDLE, running: true, approvalId: "ap1" }); + expect(region().textContent).toBe("The agent needs your approval."); + // Resolving it does not re-announce or leave stale text as a new event. + rerender({ ...IDLE, running: true }); + expect(region().textContent).toBe("The agent needs your approval."); + }); + + test("a second identical approval is announced again (keyed remount)", () => { + mount({ ...IDLE, running: true }); + rerender({ ...IDLE, running: true, approvalId: "ap1" }); + const firstSpan = region().querySelector("span"); + expect(firstSpan?.textContent).toBe("The agent needs your approval."); + rerender({ ...IDLE, running: true }); + rerender({ ...IDLE, running: true, approvalId: "ap2" }); + expect(region().textContent).toBe("The agent needs your approval."); + // New DOM node the second time, so the screen reader reads it again. + const secondSpan = region().querySelector("span"); + expect(secondSpan).not.toBe(firstSpan); + }); +}); diff --git a/frontend/components/chat/agent-announcer.tsx b/frontend/components/chat/agent-announcer.tsx new file mode 100644 index 00000000..066daf79 --- /dev/null +++ b/frontend/components/chat/agent-announcer.tsx @@ -0,0 +1,52 @@ +"use client"; + +// #350: the one polite live region for the thread. Feeds each render's activity +// through agentAnnouncement and speaks whatever's new. Only ever holds a short +// sentence, never streamed text. + +import { useEffect, useRef, useState } from "react"; +import { type AgentActivity, agentAnnouncement, type TurnOutcome } from "./agent-announcement"; + +export function AgentAnnouncer({ + running, + outcome, + approvalId, + questionId, + pendingApprovalIds, +}: { + readonly running: boolean; + readonly outcome: TurnOutcome | null; + readonly approvalId: string | null; + readonly questionId: string | null; + readonly pendingApprovalIds: readonly string[]; +}) { + // Join the ids so the effect dep is a string, not a new array every render. + const pendingKey = pendingApprovalIds.join("\u0000"); + const prevActivityRef = useRef(null); + const seqRef = useRef(0); + const [announcement, setAnnouncement] = useState<{ text: string; seq: number } | null>(null); + + useEffect(() => { + const activity: AgentActivity = { + running, + outcome, + approvalId, + questionId, + pendingApprovalIds: pendingKey ? pendingKey.split("\u0000") : [], + }; + const message = agentAnnouncement(prevActivityRef.current, activity); + prevActivityRef.current = activity; + if (message) { + seqRef.current += 1; + setAnnouncement({ text: message, seq: seqRef.current }); + } + }, [running, outcome, approvalId, questionId, pendingKey]); + + return ( +
+ {/* key changes per event, so two identical sentences in a row still get + read out instead of being seen as "no change". */} + {announcement && {announcement.text}} +
+ ); +} diff --git a/frontend/components/chat/conversation-announcer.interaction.test.tsx b/frontend/components/chat/conversation-announcer.interaction.test.tsx new file mode 100644 index 00000000..fc80390e --- /dev/null +++ b/frontend/components/chat/conversation-announcer.interaction.test.tsx @@ -0,0 +1,113 @@ +// Checks the mapping ConversationAnnouncer adds on top of the pure rule: +// completed turn -> reply, user cancel -> stop, real failure -> silent (the +// error banner handles that one). +import { afterEach, describe, expect, test } from "bun:test"; +import { GlobalRegistrator } from "@happy-dom/global-registrator"; + +if (typeof window === "undefined") GlobalRegistrator.register(); + +(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +const { act } = await import("react"); +const { createRoot } = await import("react-dom/client"); +const { ConversationAnnouncer } = await import("./conversation-announcer"); + +import type { GatewayApproval } from "@/lib/gateway-approvals"; +import type { Turn } from "./conversation"; +import type { ApiRun, RunStatus } from "./types"; + +// The announcer reads only status, summary and the root run id, so the rest of +// the Turn shape is irrelevant here. +function turn(id: string, status: RunStatus, summary: string | null): Turn { + return { + run: { id } as unknown as ApiRun, + steps: [], + status, + summary, + live: status === "running", + liveText: "", + liveReasoning: "", + }; +} + +function gateway(id: string, status: GatewayApproval["status"]): GatewayApproval { + return { + id, + runId: "r1", + toolName: "deploy_service", + arguments: {}, + status, + requestedAt: "2026-01-01T00:00:00Z", + resolvedAt: null, + resolvedBy: null, + }; +} + +let root: ReturnType | null = null; +let container: HTMLElement | null = null; + +type Props = Parameters[0]; + +const RUNNING: Props = { + running: true, + turns: [turn("root", "running", null)], + pendingApproval: null, + pendingQuestion: null, + gatewayApprovals: [], +}; + +function mount(props: Props) { + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + act(() => root?.render()); +} + +function rerender(props: Props) { + act(() => root?.render()); +} + +function text(): string { + return (container?.querySelector('[role="status"]')?.textContent ?? "").trim(); +} + +afterEach(() => { + if (root) act(() => root?.unmount()); + container?.remove(); + root = null; + container = null; +}); + +describe("ConversationAnnouncer mapping", () => { + test("a completed turn is announced as a reply", () => { + mount(RUNNING); + rerender({ ...RUNNING, running: false, turns: [turn("root", "completed", "Done.")] }); + expect(text()).toBe("The agent replied."); + }); + + test("a user stop is announced as a stop, not a reply", () => { + mount(RUNNING); + rerender({ + ...RUNNING, + running: false, + turns: [turn("root", "failed", "Stopped by user before the first tool call.")], + }); + expect(text()).toBe("The agent stopped."); + }); + + test("a real failure stays silent (the error banner owns it)", () => { + mount(RUNNING); + rerender({ + ...RUNNING, + running: false, + turns: [turn("root", "failed", "The model returned an error.")], + }); + expect(text()).toBe(""); + }); + + test("a newly pending gateway approval is announced", () => { + mount(RUNNING); + rerender({ ...RUNNING, gatewayApprovals: [gateway("g1", "pending")] }); + expect(text()).toBe("The agent needs your approval."); + }); +}); diff --git a/frontend/components/chat/conversation-announcer.tsx b/frontend/components/chat/conversation-announcer.tsx new file mode 100644 index 00000000..001746a4 --- /dev/null +++ b/frontend/components/chat/conversation-announcer.tsx @@ -0,0 +1,51 @@ +"use client"; + +// #350: maps thread state into the activity the announcer wants. completed is a +// reply, a user cancel is a stop, a real failure stays null (the error banner +// handles it). Keyed on the thread root so switching threads resets it and we +// don't replay the new thread's last event. + +import { isUserStopSummary } from "@/components/session-ui/thread-error-banner"; +import type { GatewayApproval } from "@/lib/gateway-approvals"; +import type { TurnOutcome } from "./agent-announcement"; +import { AgentAnnouncer } from "./agent-announcer"; +import type { PendingApproval } from "./approval-state"; +import type { Turn } from "./conversation"; +import type { PendingQuestion } from "./question-state"; + +export function ConversationAnnouncer({ + running, + turns, + pendingApproval, + pendingQuestion, + gatewayApprovals, +}: { + readonly running: boolean; + readonly turns: readonly Turn[]; + readonly pendingApproval: PendingApproval | null; + readonly pendingQuestion: PendingQuestion | null; + readonly gatewayApprovals: readonly GatewayApproval[] | undefined; +}) { + const latestTurn = turns.at(-1); + const outcome: TurnOutcome | null = running + ? null + : latestTurn?.status === "completed" + ? "replied" + : latestTurn?.status === "failed" && isUserStopSummary(latestTurn.summary) + ? "stopped" + : null; + const pendingApprovalIds = (gatewayApprovals ?? []) + .filter((approval) => approval.status === "pending") + .map((approval) => approval.id); + + return ( + + ); +} diff --git a/frontend/components/chat/conversation.tsx b/frontend/components/chat/conversation.tsx index b55fae1c..83f29188 100644 --- a/frontend/components/chat/conversation.tsx +++ b/frontend/components/chat/conversation.tsx @@ -28,6 +28,7 @@ import { type HandoffReceipt, HandoffReceipts, } from "@/components/chat/handoff-receipts"; +import { ConversationAnnouncer } from "@/components/chat/conversation-announcer"; import { InboundAttachments } from "@/components/chat/inbound-attachments"; import { NativeApprovalCard } from "@/components/chat/native-approval-card"; import { PermissionModeTag } from "@/components/pro/permission-mode-chip"; @@ -667,6 +668,14 @@ export const Conversation = memo(function Conversation({ return (
+ {/* Screen-reader announcements: reply done, stopped, new approval/question (#350). */} +