From 7cb6c0749415ef6e8b28d1a13e76576c481a6054 Mon Sep 17 00:00:00 2001 From: Chris Scott <99081550+chriswritescode-dev@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:26:31 -0400 Subject: [PATCH 1/4] feat(tts): play TTS for visible messages with a keyboard shortcut --- .../src/components/message/MessageThread.tsx | 2 +- .../components/settings/KeyboardShortcuts.tsx | 2 +- frontend/src/lib/visibleMessageTexts.test.ts | 105 ++++++++++++++++++ frontend/src/lib/visibleMessageTexts.ts | 17 +++ frontend/src/pages/SessionDetail.tsx | 17 ++- shared/src/schemas/settings.ts | 1 + 6 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 frontend/src/lib/visibleMessageTexts.test.ts create mode 100644 frontend/src/lib/visibleMessageTexts.ts diff --git a/frontend/src/components/message/MessageThread.tsx b/frontend/src/components/message/MessageThread.tsx index 3045f167b..021ff1e54 100644 --- a/frontend/src/components/message/MessageThread.tsx +++ b/frontend/src/components/message/MessageThread.tsx @@ -496,7 +496,7 @@ const MessageRow = memo(function MessageRow({ : 0 return ( -
+
diff --git a/frontend/src/components/settings/KeyboardShortcuts.tsx b/frontend/src/components/settings/KeyboardShortcuts.tsx index a68d833d7..bb5f0d9c3 100644 --- a/frontend/src/components/settings/KeyboardShortcuts.tsx +++ b/frontend/src/components/settings/KeyboardShortcuts.tsx @@ -7,7 +7,7 @@ import { formatEventModifiers, formatShortcutEvent, isModifierOnlyEvent, normali import { applyTuiKeybindImport, parseTuiKeybindConfig } from '@/lib/tuiKeybindImport' import { showToast } from '@/lib/toast' -const CONVERSATION_ACTIONS = ['submit', 'abort', 'clearPrompt', 'toggleMode', 'undo', 'redo', 'compact', 'fork', 'timeline', 'exportSession', 'selectModel', 'variantCycle', 'favoriteCycle', 'halfPageUp', 'halfPageDown'] +const CONVERSATION_ACTIONS = ['submit', 'abort', 'clearPrompt', 'toggleMode', 'undo', 'redo', 'compact', 'fork', 'timeline', 'exportSession', 'selectModel', 'variantCycle', 'favoriteCycle', 'halfPageUp', 'halfPageDown', 'speakVisible'] const NAVIGATION_ACTIONS = ['settings', 'sessions', 'newSession', 'closeSession', 'toggleSidebar', 'toggleTerminal', 'toggleSourceControl'] const formatShortcutLabel = (action: string): string => { diff --git a/frontend/src/lib/visibleMessageTexts.test.ts b/frontend/src/lib/visibleMessageTexts.test.ts new file mode 100644 index 000000000..3c5680eeb --- /dev/null +++ b/frontend/src/lib/visibleMessageTexts.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from 'vitest' +import type { + SessionMessageAssistant, + SessionMessageInfo, + SessionMessageUser, +} from '@opencode-manager/shared/opencode' +import { collectVisibleAssistantTexts } from './visibleMessageTexts' + +const assistantMessage = (id: string, texts: string[]): SessionMessageAssistant => ({ + id, + type: 'assistant', + agent: 'test-agent', + model: { providerID: 'test-provider', id: 'test-model' }, + content: texts.map((text) => ({ type: 'text', text })), + time: { created: 1, completed: 2 }, +}) + +const userMessage = (id: string, text: string): SessionMessageUser => ({ + id, + type: 'user', + text, + time: { created: 1 }, +}) + +const rect = (top: number, bottom: number): DOMRect => ({ + top, + bottom, + left: 0, + right: 100, + width: 100, + height: bottom - top, + x: 0, + y: top, + toJSON: () => ({}), +} as DOMRect) + +const stubRect = (element: HTMLElement, top: number, bottom: number) => { + element.getBoundingClientRect = () => rect(top, bottom) +} + +const buildContainer = (): HTMLDivElement => { + const container = document.createElement('div') + stubRect(container, 0, 100) + return container +} + +const appendMessageElement = ( + container: HTMLElement, + messageId: string, + top: number, + bottom: number, +): HTMLElement => { + const element = document.createElement('div') + element.dataset.messageId = messageId + stubRect(element, top, bottom) + container.appendChild(element) + return element +} + +describe('collectVisibleAssistantTexts', () => { + it('returns only intersecting assistant messages in DOM order', () => { + const container = buildContainer() + appendMessageElement(container, 'a1', 10, 30) + appendMessageElement(container, 'u2', 40, 60) + appendMessageElement(container, 'a3', 70, 90) + appendMessageElement(container, 'a4', 200, 220) + appendMessageElement(container, 'a5', 5, 8) + + const messages: SessionMessageInfo[] = [ + assistantMessage('a1', ['one']), + userMessage('u2', 'hello'), + assistantMessage('a3', []), + assistantMessage('a4', ['four']), + assistantMessage('a5', ['five']), + ] + + expect(collectVisibleAssistantTexts(container, messages)).toEqual(['one', 'five']) + }) + + it('ignores user and system messages and assistant messages with no text parts', () => { + const container = buildContainer() + appendMessageElement(container, 'u1', 10, 30) + appendMessageElement(container, 's1', 20, 40) + appendMessageElement(container, 'a-empty', 30, 50) + appendMessageElement(container, 'a-valid', 40, 60) + + const messages: SessionMessageInfo[] = [ + userMessage('u1', 'hello'), + { id: 's1', type: 'system', text: 'note', time: { created: 1 } }, + assistantMessage('a-empty', []), + assistantMessage('a-valid', ['kept']), + ] + + expect(collectVisibleAssistantTexts(container, messages)).toEqual(['kept']) + }) + + it('returns an empty list when nothing intersects', () => { + const container = buildContainer() + appendMessageElement(container, 'a1', 200, 220) + + const messages: SessionMessageInfo[] = [assistantMessage('a1', ['hidden'])] + + expect(collectVisibleAssistantTexts(container, messages)).toEqual([]) + }) +}) diff --git a/frontend/src/lib/visibleMessageTexts.ts b/frontend/src/lib/visibleMessageTexts.ts new file mode 100644 index 000000000..5e2c651c1 --- /dev/null +++ b/frontend/src/lib/visibleMessageTexts.ts @@ -0,0 +1,17 @@ +import { assistantText, type SessionMessageInfo } from '@opencode-manager/shared/opencode' + +export function collectVisibleAssistantTexts(container: HTMLElement, messages: SessionMessageInfo[]): string[] { + const containerRect = container.getBoundingClientRect() + const textById = new Map( + messages + .filter((message) => message.type === 'assistant') + .map((message) => [message.id, assistantText(message.content).trim()]), + ) + return Array.from(container.querySelectorAll('[data-message-id]')) + .filter((element) => { + const rect = element.getBoundingClientRect() + return rect.bottom > containerRect.top && rect.top < containerRect.bottom + }) + .map((element) => textById.get(element.dataset.messageId ?? '') ?? '') + .filter((text) => text.length > 0) +} diff --git a/frontend/src/pages/SessionDetail.tsx b/frontend/src/pages/SessionDetail.tsx index 4ec7aeb05..453df9aae 100644 --- a/frontend/src/pages/SessionDetail.tsx +++ b/frontend/src/pages/SessionDetail.tsx @@ -35,6 +35,7 @@ import { MessageSkeleton } from "@/components/message/MessageSkeleton"; import { exportSession, downloadMarkdown } from "@/lib/exportSession"; import { copyTextToClipboard } from "@/lib/clipboard"; import { getMessagesContentVersion } from "./sessionContentVersion"; +import { collectVisibleAssistantTexts } from '@/lib/visibleMessageTexts'; import { showToast } from "@/lib/toast"; import { getWorkspaceFilePath } from "@/lib/markdownLinks"; import { getRepoDisplayName } from "@/lib/utils"; @@ -274,7 +275,7 @@ export function SessionDetail() { const { modelRef } = useModelSelection(sessionDirectory, modelSelectionSession); const setSessionStatus = useSessionStatus((state) => state.setStatus); const isEditingMessage = useUIState((state) => state.isEditingMessage); - const { isEnabled: ttsEnabled } = useTTS(); + const { isEnabled: ttsEnabled, speak, stop, isPlaying: isTTSPlaying, isLoading: isTTSLoading } = useTTS(); const { syncForSession: syncPermissionsForSession } = usePermissions(); const { getForSession: getFormForSession, reply: replyToForm, cancel: cancelForm, syncForSession: syncFormsForSession } = useForms(); const currentForm = sessionId ? getFormForSession(sessionId) : null; @@ -614,6 +615,19 @@ export function SessionDetail() { scrollByUser((direction * container.clientHeight) / 2); }, [scrollByUser]); + const handleSpeakVisibleMessages = useCallback(() => { + if (!ttsEnabled) return; + if (isTTSPlaying || isTTSLoading) { + stop(); + return; + } + const container = messageContainerRef.current; + if (!container) return; + const texts = collectVisibleAssistantTexts(container, messages); + if (texts.length === 0) return; + speak(texts.join('\n\n')); + }, [ttsEnabled, isTTSPlaying, isTTSLoading, stop, speak, messages]); + useShortcutActions({ selectModel: handleOpenModelDialog, sessions: handleShowSessionsDialog, @@ -633,6 +647,7 @@ export function SessionDetail() { clearPrompt: handleClearPrompt, halfPageUp: () => scrollMessagesByHalfPage(-1), halfPageDown: () => scrollMessagesByHalfPage(1), + speakVisible: handleSpeakVisibleMessages, }); const handleUndoMessage = useCallback((restoredPrompt: string) => { diff --git a/shared/src/schemas/settings.ts b/shared/src/schemas/settings.ts index d2921e370..b8b7928ca 100644 --- a/shared/src/schemas/settings.ts +++ b/shared/src/schemas/settings.ts @@ -88,6 +88,7 @@ export const DEFAULT_KEYBOARD_SHORTCUTS = { exportSession: 'X', halfPageUp: 'Ctrl+U', halfPageDown: 'Ctrl+D', + speakVisible: 'S', } satisfies Record; export type KeyboardShortcutAction = keyof typeof DEFAULT_KEYBOARD_SHORTCUTS; From eafa8941cb9c4747c0c7f71219494695896c21b5 Mon Sep 17 00:00:00 2001 From: Chris Scott <99081550+chriswritescode-dev@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:49:32 -0400 Subject: [PATCH 2/4] fix(tts): exclude covered messages and skip auto-play during shortcut playback --- .../useAutoPlayLastResponse.test.tsx | 73 +++++++ .../useSpeakVisibleMessages.test.tsx | 202 ++++++++++++++++++ frontend/src/hooks/useAutoPlayLastResponse.ts | 20 +- frontend/src/hooks/useSpeakVisibleMessages.ts | 41 ++++ frontend/src/lib/visibleMessageTexts.test.ts | 19 +- frontend/src/lib/visibleMessageTexts.ts | 13 +- frontend/src/pages/SessionDetail.tsx | 21 +- 7 files changed, 364 insertions(+), 25 deletions(-) create mode 100644 frontend/src/hooks/__tests__/useSpeakVisibleMessages.test.tsx create mode 100644 frontend/src/hooks/useSpeakVisibleMessages.ts diff --git a/frontend/src/hooks/__tests__/useAutoPlayLastResponse.test.tsx b/frontend/src/hooks/__tests__/useAutoPlayLastResponse.test.tsx index 7f9dbcb70..e2b2996c3 100644 --- a/frontend/src/hooks/__tests__/useAutoPlayLastResponse.test.tsx +++ b/frontend/src/hooks/__tests__/useAutoPlayLastResponse.test.tsx @@ -111,6 +111,8 @@ describe('useAutoPlayLastResponse', () => { }, updateSettings: mockUpdateSettings, }) + + return mockTTS } it('does NOT call speakMessage when tts.enabled is false', () => { @@ -191,6 +193,77 @@ describe('useAutoPlayLastResponse', () => { expect(mockSpeakMessage).toHaveBeenCalledWith('1', 'Test text') }) + it('does NOT call speakMessage when unattributed playback is active, and does not speak it after playback ends', () => { + const mockTTS = setup() + + const incompleteMessage = createMessage('1') + const { rerender } = renderHook( + (props) => useAutoPlayLastResponse(props), + { + initialProps: { + sessionId: 'test-session', + lastAssistantMessage: incompleteMessage, + lastAssistantText: 'Test text', + isStreamingResponse: false, + }, + } + ) + + mockTTS.isPlaying = true + mockTTS.activeMessageId = null + + const completedMessage = createMessage('1', Date.now()) + rerender({ + sessionId: 'test-session', + lastAssistantMessage: completedMessage, + lastAssistantText: 'Test text', + isStreamingResponse: false, + }) + + expect(mockSpeakMessage).not.toHaveBeenCalled() + + mockTTS.isPlaying = false + rerender({ + sessionId: 'test-session', + lastAssistantMessage: completedMessage, + lastAssistantText: 'Test text', + isStreamingResponse: false, + }) + + expect(mockSpeakMessage).not.toHaveBeenCalled() + }) + + it('CALLS speakMessage when a message-attributed playback is active', () => { + const mockTTS = setup() + + const incompleteMessage = createMessage('1') + const { rerender } = renderHook( + (props) => useAutoPlayLastResponse(props), + { + initialProps: { + sessionId: 'test-session', + lastAssistantMessage: incompleteMessage, + lastAssistantText: 'Test text', + isStreamingResponse: false, + }, + } + ) + + mockTTS.isPlaying = true + mockTTS.activeMessageId = 'other-message' + + const completedMessage = createMessage('1', Date.now()) + rerender({ + sessionId: 'test-session', + lastAssistantMessage: completedMessage, + lastAssistantText: 'Test text', + isStreamingResponse: false, + }) + + expect(mockSpeakMessage).toHaveBeenCalledTimes(1) + expect(mockSpeakMessage).toHaveBeenCalledWith('1', 'Test text') + }) + it('does NOT call speakMessage twice for the same message id across multiple rerenders', () => { setup() diff --git a/frontend/src/hooks/__tests__/useSpeakVisibleMessages.test.tsx b/frontend/src/hooks/__tests__/useSpeakVisibleMessages.test.tsx new file mode 100644 index 000000000..94accd0f4 --- /dev/null +++ b/frontend/src/hooks/__tests__/useSpeakVisibleMessages.test.tsx @@ -0,0 +1,202 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { renderHook } from '@testing-library/react' +import type { RefObject } from 'react' +import type { SessionMessageAssistant, SessionMessageInfo } from '@opencode-manager/shared/opencode' +import type { TTSState } from '@/hooks/useTTS' + +const mocks = vi.hoisted(() => ({ + useTTS: vi.fn(), + showToastError: vi.fn(), +})) + +vi.mock('@/hooks/useTTS', () => ({ + useTTS: mocks.useTTS, +})) + +vi.mock('@/lib/toast', () => ({ + showToast: { error: mocks.showToastError }, +})) + +import { useSpeakVisibleMessages } from '../useSpeakVisibleMessages' + +interface MockTTSReturn { + speak: ReturnType + stop: ReturnType + isEnabled: boolean + isPlaying: boolean + isLoading: boolean + state: TTSState + error: string | null + activeMessageId: string | null +} + +const rect = (top: number, bottom: number): DOMRect => ({ + top, + bottom, + left: 0, + right: 100, + width: 100, + height: bottom - top, + x: 0, + y: top, + toJSON: () => ({}), +} as DOMRect) + +const buildContainer = (messageId: string): HTMLDivElement => { + const container = document.createElement('div') + container.getBoundingClientRect = () => rect(0, 100) + const element = document.createElement('div') + element.dataset.messageId = messageId + element.getBoundingClientRect = () => rect(10, 30) + container.appendChild(element) + return container +} + +const assistantMessage = (id: string, text: string): SessionMessageAssistant => ({ + id, + type: 'assistant', + agent: 'test-agent', + model: { providerID: 'test-provider', id: 'test-model' }, + content: [{ type: 'text', text }], + time: { created: 1, completed: 2 }, +}) + +describe('useSpeakVisibleMessages', () => { + const mockSpeak = vi.fn() + const mockStop = vi.fn() + + const setupTTS = (overrides: Partial = {}) => { + const value: MockTTSReturn = { + speak: mockSpeak, + stop: mockStop, + isEnabled: true, + isPlaying: false, + isLoading: false, + state: 'idle', + error: null, + activeMessageId: null, + ...overrides, + } + mocks.useTTS.mockReturnValue(value) + } + + beforeEach(() => { + vi.clearAllMocks() + mockSpeak.mockResolvedValue(true) + }) + + it('stops playback while playing', () => { + setupTTS({ isPlaying: true }) + const containerRef: RefObject = { current: null } + + const { result } = renderHook(() => + useSpeakVisibleMessages(containerRef, [], 0) + ) + + result.current() + + expect(mockStop).toHaveBeenCalledTimes(1) + expect(mockSpeak).not.toHaveBeenCalled() + }) + + it('speaks the visible assistant texts joined by a blank line', () => { + setupTTS() + const container = buildContainer('a1') + const containerRef: RefObject = { current: container } + const messages: SessionMessageInfo[] = [assistantMessage('a1', 'hello world')] + + const { result } = renderHook(() => + useSpeakVisibleMessages(containerRef, messages, 0) + ) + + result.current() + + expect(mockSpeak).toHaveBeenCalledTimes(1) + expect(mockSpeak).toHaveBeenCalledWith('hello world') + }) + + it('does nothing when TTS is disabled', () => { + setupTTS({ isEnabled: false }) + const container = buildContainer('a1') + const containerRef: RefObject = { current: container } + const messages: SessionMessageInfo[] = [assistantMessage('a1', 'hello world')] + + const { result } = renderHook(() => + useSpeakVisibleMessages(containerRef, messages, 0) + ) + + result.current() + + expect(mockSpeak).not.toHaveBeenCalled() + }) + + it('shows an error toast once after shortcut-started playback fails', () => { + setupTTS() + const container = buildContainer('a1') + const containerRef: RefObject = { current: container } + const messages: SessionMessageInfo[] = [assistantMessage('a1', 'hello world')] + + const { result, rerender } = renderHook(() => + useSpeakVisibleMessages(containerRef, messages, 0) + ) + + result.current() + expect(mockSpeak).toHaveBeenCalledTimes(1) + + setupTTS({ state: 'error', error: 'network failure' }) + rerender() + + expect(mocks.showToastError).toHaveBeenCalledTimes(1) + expect(mocks.showToastError).toHaveBeenCalledWith('network failure') + + setupTTS({ state: 'error', error: 'another failure' }) + rerender() + + expect(mocks.showToastError).toHaveBeenCalledTimes(1) + }) + + it('does not toast when shortcut-started playback returns to idle', () => { + setupTTS() + const container = buildContainer('a1') + const containerRef: RefObject = { current: container } + const messages: SessionMessageInfo[] = [assistantMessage('a1', 'hello world')] + + const { result, rerender } = renderHook(() => + useSpeakVisibleMessages(containerRef, messages, 0) + ) + + result.current() + + setupTTS({ state: 'playing' }) + rerender() + + setupTTS({ state: 'idle' }) + rerender() + + setupTTS({ state: 'error', error: 'late failure' }) + rerender() + + expect(mocks.showToastError).not.toHaveBeenCalled() + }) + + it('does not toast when playback becomes message-attributed', () => { + setupTTS() + const container = buildContainer('a1') + const containerRef: RefObject = { current: container } + const messages: SessionMessageInfo[] = [assistantMessage('a1', 'hello world')] + + const { result, rerender } = renderHook(() => + useSpeakVisibleMessages(containerRef, messages, 0) + ) + + result.current() + + setupTTS({ state: 'playing', activeMessageId: 'a1' }) + rerender() + + setupTTS({ state: 'error', error: 'late failure', activeMessageId: 'a1' }) + rerender() + + expect(mocks.showToastError).not.toHaveBeenCalled() + }) +}) diff --git a/frontend/src/hooks/useAutoPlayLastResponse.ts b/frontend/src/hooks/useAutoPlayLastResponse.ts index fe8e6d913..33b1894d0 100644 --- a/frontend/src/hooks/useAutoPlayLastResponse.ts +++ b/frontend/src/hooks/useAutoPlayLastResponse.ts @@ -37,7 +37,7 @@ export function useAutoPlayLastResponse({ lastAssistantText, isStreamingResponse, }: UseAutoPlayLastResponseParams): void { - const { speakMessage, isEnabled: ttsEnabled } = useTTS() + const { speakMessage, isEnabled: ttsEnabled, isPlaying, isLoading, activeMessageId } = useTTS() const { preferences } = useSettings() const autoPlayEnabled = preferences?.tts?.autoPlay ?? false @@ -47,6 +47,13 @@ export function useAutoPlayLastResponse({ const lastKnownCompletedRef = useRef>({}) const hasSeenIncompleteRef = useRef(false) const previousSessionIdRef = useRef(null) + const isUnattributedPlaybackActiveRef = useRef(false) + + const isUnattributedPlaybackActive = (isPlaying || isLoading) && activeMessageId === null + + useEffect(() => { + isUnattributedPlaybackActiveRef.current = isUnattributedPlaybackActive + }, [isUnattributedPlaybackActive]) useEffect(() => { const isSessionChange = previousSessionIdRef.current !== null && previousSessionIdRef.current !== sessionId @@ -76,6 +83,7 @@ export function useAutoPlayLastResponse({ const messageId = lastAssistantMessage.id const isCompleted = isMessageCompleted(lastAssistantMessage) const wasCompleted = lastKnownCompletedRef.current[messageId] ?? false + const shouldAutoSpeak = !isUnattributedPlaybackActiveRef.current if (!isCompleted) { lastKnownCompletedRef.current[messageId] = false @@ -91,7 +99,7 @@ export function useAutoPlayLastResponse({ hasInitializedRef.current = true lastSpokenIdRef.current = messageId lastKnownCompletedRef.current[messageId] = true - if (hasSeenIncompleteRef.current) { + if (hasSeenIncompleteRef.current && shouldAutoSpeak) { speakMessage(messageId, lastAssistantText) } return @@ -99,14 +107,18 @@ export function useAutoPlayLastResponse({ if (!wasCompleted && messageId === lastSpokenIdRef.current) { lastKnownCompletedRef.current[messageId] = true - speakMessage(messageId, lastAssistantText) + if (shouldAutoSpeak) { + speakMessage(messageId, lastAssistantText) + } return } if (messageId !== lastSpokenIdRef.current) { lastSpokenIdRef.current = messageId lastKnownCompletedRef.current[messageId] = true - speakMessage(messageId, lastAssistantText) + if (shouldAutoSpeak) { + speakMessage(messageId, lastAssistantText) + } } }, [ ttsEnabled, diff --git a/frontend/src/hooks/useSpeakVisibleMessages.ts b/frontend/src/hooks/useSpeakVisibleMessages.ts new file mode 100644 index 000000000..a957110e6 --- /dev/null +++ b/frontend/src/hooks/useSpeakVisibleMessages.ts @@ -0,0 +1,41 @@ +import { useCallback, useEffect, useRef, type RefObject } from 'react' +import type { SessionMessageInfo } from '@opencode-manager/shared/opencode' +import { useTTS } from './useTTS' +import { collectVisibleAssistantTexts } from '@/lib/visibleMessageTexts' +import { showToast } from '@/lib/toast' + +export function useSpeakVisibleMessages( + containerRef: RefObject, + messages: SessionMessageInfo[], + bottomInset: number, +): () => void { + const { speak, stop, isPlaying, isLoading, isEnabled, state, error, activeMessageId } = useTTS() + const pendingRef = useRef(false) + + useEffect(() => { + if (!pendingRef.current) return + if (activeMessageId !== null || state === 'idle') { + pendingRef.current = false + return + } + if (state === 'error') { + showToast.error(error ?? 'Text-to-speech failed') + pendingRef.current = false + } + }, [state, error, activeMessageId]) + + return useCallback(() => { + if (!isEnabled) return + if (isPlaying || isLoading) { + pendingRef.current = false + stop() + return + } + const container = containerRef.current + if (!container) return + const texts = collectVisibleAssistantTexts(container, messages, bottomInset) + if (texts.length === 0) return + pendingRef.current = true + void speak(texts.join('\n\n')) + }, [isEnabled, isPlaying, isLoading, stop, speak, messages, containerRef, bottomInset]) +} diff --git a/frontend/src/lib/visibleMessageTexts.test.ts b/frontend/src/lib/visibleMessageTexts.test.ts index 3c5680eeb..68c57f010 100644 --- a/frontend/src/lib/visibleMessageTexts.test.ts +++ b/frontend/src/lib/visibleMessageTexts.test.ts @@ -74,7 +74,7 @@ describe('collectVisibleAssistantTexts', () => { assistantMessage('a5', ['five']), ] - expect(collectVisibleAssistantTexts(container, messages)).toEqual(['one', 'five']) + expect(collectVisibleAssistantTexts(container, messages, 0)).toEqual(['one', 'five']) }) it('ignores user and system messages and assistant messages with no text parts', () => { @@ -91,7 +91,7 @@ describe('collectVisibleAssistantTexts', () => { assistantMessage('a-valid', ['kept']), ] - expect(collectVisibleAssistantTexts(container, messages)).toEqual(['kept']) + expect(collectVisibleAssistantTexts(container, messages, 0)).toEqual(['kept']) }) it('returns an empty list when nothing intersects', () => { @@ -100,6 +100,19 @@ describe('collectVisibleAssistantTexts', () => { const messages: SessionMessageInfo[] = [assistantMessage('a1', ['hidden'])] - expect(collectVisibleAssistantTexts(container, messages)).toEqual([]) + expect(collectVisibleAssistantTexts(container, messages, 0)).toEqual([]) + }) + + it('excludes messages hidden within the bottom inset and includes messages straddling it', () => { + const container = buildContainer() + appendMessageElement(container, 'hidden', 85, 95) + appendMessageElement(container, 'straddling', 75, 95) + + const messages: SessionMessageInfo[] = [ + assistantMessage('hidden', ['hidden']), + assistantMessage('straddling', ['straddling']), + ] + + expect(collectVisibleAssistantTexts(container, messages, 20)).toEqual(['straddling']) }) }) diff --git a/frontend/src/lib/visibleMessageTexts.ts b/frontend/src/lib/visibleMessageTexts.ts index 5e2c651c1..df2519426 100644 --- a/frontend/src/lib/visibleMessageTexts.ts +++ b/frontend/src/lib/visibleMessageTexts.ts @@ -1,16 +1,21 @@ -import { assistantText, type SessionMessageInfo } from '@opencode-manager/shared/opencode' +import { type SessionMessageInfo } from '@opencode-manager/shared/opencode' +import { getAssistantText } from '@/hooks/useAutoPlayLastResponse' -export function collectVisibleAssistantTexts(container: HTMLElement, messages: SessionMessageInfo[]): string[] { +export function collectVisibleAssistantTexts( + container: HTMLElement, + messages: SessionMessageInfo[], + bottomInset: number, +): string[] { const containerRect = container.getBoundingClientRect() const textById = new Map( messages .filter((message) => message.type === 'assistant') - .map((message) => [message.id, assistantText(message.content).trim()]), + .map((message) => [message.id, getAssistantText(message).trim()]), ) return Array.from(container.querySelectorAll('[data-message-id]')) .filter((element) => { const rect = element.getBoundingClientRect() - return rect.bottom > containerRect.top && rect.top < containerRect.bottom + return rect.bottom > containerRect.top && rect.top < containerRect.bottom - bottomInset }) .map((element) => textById.get(element.dataset.messageId ?? '') ?? '') .filter((text) => text.length > 0) diff --git a/frontend/src/pages/SessionDetail.tsx b/frontend/src/pages/SessionDetail.tsx index 453df9aae..93daf8659 100644 --- a/frontend/src/pages/SessionDetail.tsx +++ b/frontend/src/pages/SessionDetail.tsx @@ -35,7 +35,7 @@ import { MessageSkeleton } from "@/components/message/MessageSkeleton"; import { exportSession, downloadMarkdown } from "@/lib/exportSession"; import { copyTextToClipboard } from "@/lib/clipboard"; import { getMessagesContentVersion } from "./sessionContentVersion"; -import { collectVisibleAssistantTexts } from '@/lib/visibleMessageTexts'; +import { useSpeakVisibleMessages } from "@/hooks/useSpeakVisibleMessages"; import { showToast } from "@/lib/toast"; import { getWorkspaceFilePath } from "@/lib/markdownLinks"; import { getRepoDisplayName } from "@/lib/utils"; @@ -275,7 +275,7 @@ export function SessionDetail() { const { modelRef } = useModelSelection(sessionDirectory, modelSelectionSession); const setSessionStatus = useSessionStatus((state) => state.setStatus); const isEditingMessage = useUIState((state) => state.isEditingMessage); - const { isEnabled: ttsEnabled, speak, stop, isPlaying: isTTSPlaying, isLoading: isTTSLoading } = useTTS(); + const { isEnabled: ttsEnabled } = useTTS(); const { syncForSession: syncPermissionsForSession } = usePermissions(); const { getForSession: getFormForSession, reply: replyToForm, cancel: cancelForm, syncForSession: syncFormsForSession } = useForms(); const currentForm = sessionId ? getFormForSession(sessionId) : null; @@ -615,18 +615,11 @@ export function SessionDetail() { scrollByUser((direction * container.clientHeight) / 2); }, [scrollByUser]); - const handleSpeakVisibleMessages = useCallback(() => { - if (!ttsEnabled) return; - if (isTTSPlaying || isTTSLoading) { - stop(); - return; - } - const container = messageContainerRef.current; - if (!container) return; - const texts = collectVisibleAssistantTexts(container, messages); - if (texts.length === 0) return; - speak(texts.join('\n\n')); - }, [ttsEnabled, isTTSPlaying, isTTSLoading, stop, speak, messages]); + const handleSpeakVisibleMessages = useSpeakVisibleMessages( + messageContainerRef, + messages, + promptOverlayHeight + inputBottomOffset, + ); useShortcutActions({ selectModel: handleOpenModelDialog, From 8a893d0d2aa31756692df7d369df95c865d336b9 Mon Sep 17 00:00:00 2001 From: Chris Scott <99081550+chriswritescode-dev@users.noreply.github.com> Date: Tue, 13 Oct 2026 21:31:03 -0400 Subject: [PATCH 3/4] refactor(tts): read visible replies when floating button is tapped while scrolled up --- e2e/inline-images.spec.ts | 164 ++-------------- e2e/mockProvider.ts | 178 ++++++++++++++++++ e2e/tts-floating-button.spec.ts | 171 +++++++++++++++++ .../message/FloatingTTSButton.test.tsx | 94 +++++++-- .../components/message/FloatingTTSButton.tsx | 19 +- frontend/src/pages/SessionDetail.tsx | 2 + 6 files changed, 459 insertions(+), 169 deletions(-) create mode 100644 e2e/mockProvider.ts create mode 100644 e2e/tts-floating-button.spec.ts diff --git a/e2e/inline-images.spec.ts b/e2e/inline-images.spec.ts index 88d29402a..4af10b541 100644 --- a/e2e/inline-images.spec.ts +++ b/e2e/inline-images.spec.ts @@ -1,10 +1,8 @@ -import { execFileSync } from 'node:child_process' import fs from 'node:fs' -import http from 'node:http' import path from 'node:path' -import { request as playwrightRequest, type APIRequestContext, type Locator } from '@playwright/test' +import { type Locator } from '@playwright/test' import { test, expect } from './fixtures' -import { BASE_URL, E2E_TMP, STORAGE_STATE } from './env' +import { createMockRepo, createSession, startMockProvider, type MockProvider, type MockRepo } from './mockProvider' const PNG_BASE64 = 'iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAYAAABytg0kAAAAFElEQVQI12P8z8Dwn4GBgYGJAQoAHxcCAmep4oIAAAAASUVORK5CYII=' @@ -14,17 +12,8 @@ const PNG_BUFFER = Buffer.from(PNG_BASE64, 'base64') const WEBP_BUFFER = Buffer.from(WEBP_BASE64, 'base64') const MOCK_PROVIDER_PORT = 5565 -const MOCK_MODEL_ID = 'mock-model' - -let mockReply = '' -let mockServer: http.Server | undefined - -interface MockRepo { - id: number - fullPath: string - sessionId: string -} +let mockProvider: MockProvider let mockRepo: MockRepo function rawFileUrl(filePath: string): string { @@ -39,144 +28,19 @@ async function expectImageLoaded(image: Locator): Promise { .toBeGreaterThan(0) } -function handleMockRequest(request: http.IncomingMessage, response: http.ServerResponse): void { - const chunks: Buffer[] = [] - request.on('data', (chunk: Buffer) => chunks.push(chunk)) - request.on('end', () => { - const url = request.url ?? '' - if (request.method === 'GET' && url.startsWith('/v1/models')) { - response.writeHead(200, { 'Content-Type': 'application/json' }) - response.end( - JSON.stringify({ - object: 'list', - data: [{ id: MOCK_MODEL_ID, object: 'model', created: 0, owned_by: 'mock' }], - }), - ) - return - } - - if (request.method === 'POST' && url.startsWith('/v1/chat/completions')) { - let body: { stream?: boolean } = {} - try { - body = JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}') - } catch { - body = {} - } - - const id = 'chatcmpl-mock' - const created = Math.floor(Date.now() / 1000) - const usage = { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } - - if (body.stream) { - response.writeHead(200, { - 'Content-Type': 'text/event-stream', - 'Cache-Control': 'no-cache', - Connection: 'keep-alive', - }) - const base = { id, object: 'chat.completion.chunk', created, model: MOCK_MODEL_ID } - response.write( - `data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { role: 'assistant', content: '' }, finish_reason: null }] })}\n\n`, - ) - response.write( - `data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { content: mockReply }, finish_reason: null }] })}\n\n`, - ) - response.write( - `data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], usage })}\n\n`, - ) - response.write('data: [DONE]\n\n') - response.end() - return - } - - response.writeHead(200, { 'Content-Type': 'application/json' }) - response.end( - JSON.stringify({ - id, - object: 'chat.completion', - created, - model: MOCK_MODEL_ID, - choices: [{ index: 0, message: { role: 'assistant', content: mockReply }, finish_reason: 'stop' }], - usage, - }), - ) - return - } - - response.writeHead(404, { 'Content-Type': 'application/json' }) - response.end(JSON.stringify({ error: { message: `unhandled ${request.method} ${url}` } })) - }) -} - -function git(directory: string, ...args: string[]): void { - execFileSync('git', args, { cwd: directory, stdio: 'pipe' }) -} - -async function createSession(api: APIRequestContext, directory: string): Promise { - const response = await api.post('/api/opencode/api/session', { data: { location: { directory } } }) - if (!response.ok()) { - throw new Error(`create session failed: ${response.status()} ${await response.text()}`) - } - const body = (await response.json()) as { data: { id: string } } - return body.data.id -} - test.beforeAll(async () => { - const server = http.createServer(handleMockRequest) - await new Promise((resolve, reject) => { - server.once('error', reject) - server.listen(MOCK_PROVIDER_PORT, '127.0.0.1', () => resolve()) + mockProvider = await startMockProvider(MOCK_PROVIDER_PORT) + mockRepo = await createMockRepo('e2e-inline-mock-repo', MOCK_PROVIDER_PORT, { + beforeCommit: (directory) => { + fs.mkdirSync(path.join(directory, 'docs', 'img'), { recursive: true }) + fs.writeFileSync(path.join(directory, 'docs', 'img', 'logo.png'), PNG_BUFFER) + fs.writeFileSync(path.join(directory, 'docs', 'notes.txt'), 'attachment notes\n') + }, }) - mockServer = server - - const directory = path.join(E2E_TMP, 'workspace', 'repos', 'e2e-inline-mock-repo') - fs.rmSync(directory, { recursive: true, force: true }) - fs.mkdirSync(path.join(directory, 'docs', 'img'), { recursive: true }) - fs.writeFileSync(path.join(directory, 'docs', 'img', 'logo.png'), PNG_BUFFER) - fs.writeFileSync(path.join(directory, 'docs', 'notes.txt'), 'attachment notes\n') - fs.writeFileSync( - path.join(directory, 'opencode.json'), - JSON.stringify( - { - $schema: 'https://opencode.ai/config.json', - provider: { - mock: { - npm: '@ai-sdk/openai-compatible', - name: 'Mock', - options: { baseURL: `http://127.0.0.1:${MOCK_PROVIDER_PORT}/v1`, apiKey: 'mock' }, - models: { [MOCK_MODEL_ID]: { name: 'Mock Model' } }, - }, - }, - model: `mock/${MOCK_MODEL_ID}`, - small_model: `mock/${MOCK_MODEL_ID}`, - }, - null, - 2, - ), - ) - fs.writeFileSync(path.join(directory, 'README.md'), '# mock repository\n') - git(directory, '-c', 'init.defaultBranch=main', 'init', '-q') - git(directory, '-c', 'user.name=e2e', '-c', 'user.email=e2e@example.com', 'add', '-A') - git(directory, '-c', 'user.name=e2e', '-c', 'user.email=e2e@example.com', 'commit', '-q', '-m', 'initial commit') - - const api = await playwrightRequest.newContext({ baseURL: BASE_URL, storageState: STORAGE_STATE }) - try { - const repoResponse = await api.post('/api/repos', { data: { localPath: directory } }) - if (!repoResponse.ok()) { - throw new Error(`register mock repo failed: ${repoResponse.status()} ${await repoResponse.text()}`) - } - const repo = (await repoResponse.json()) as { id: number; fullPath: string } - const sessionId = await createSession(api, repo.fullPath) - mockRepo = { id: repo.id, fullPath: repo.fullPath, sessionId } - } finally { - await api.dispose() - } }) test.afterAll(async () => { - const server = mockServer - if (!server) return - await new Promise((resolve) => server.close(() => resolve())) - mockServer = undefined + await mockProvider.close() }) test.describe('inline markdown images', () => { @@ -221,7 +85,7 @@ test.describe('inline markdown images', () => { }) test('assistant markdown images resolve relative, absolute and inline data sources', async ({ page }) => { - mockReply = [ + mockProvider.setReply([ 'Here are the images:', '', '![rel](docs/img/logo.png)', @@ -231,7 +95,7 @@ test.describe('inline markdown images', () => { `![dataimg](data:image/png;base64,${PNG_BASE64})`, '', '![bad](data:text/html,hello)', - ].join('\n') + ].join('\n')) await page.goto(`/repos/${mockRepo.id}/sessions/${mockRepo.sessionId}`) const composer = page.getByPlaceholder('Send a message...') @@ -257,7 +121,7 @@ test.describe('inline markdown images', () => { }) test('user attachments render images as thumbnails and other files as chips', async ({ page, request }) => { - mockReply = 'Received your files.' + mockProvider.setReply('Received your files.') const sessionId = await createSession(request, mockRepo.fullPath) await page.goto(`/repos/${mockRepo.id}/sessions/${sessionId}`) diff --git a/e2e/mockProvider.ts b/e2e/mockProvider.ts new file mode 100644 index 000000000..7ad5bff9c --- /dev/null +++ b/e2e/mockProvider.ts @@ -0,0 +1,178 @@ +import { execFileSync } from 'node:child_process' +import fs from 'node:fs' +import http from 'node:http' +import path from 'node:path' +import { request as playwrightRequest, type APIRequestContext } from '@playwright/test' +import { BASE_URL, E2E_TMP, STORAGE_STATE } from './env' + +const MOCK_MODEL_ID = 'mock-model' + +export interface MockProvider { + setReply(text: string): void + close(): Promise +} + +export interface MockRepo { + id: number + fullPath: string + sessionId: string +} + +export interface CreateMockRepoOptions { + beforeCommit?: (directory: string) => void +} + +function handleMockRequest( + request: http.IncomingMessage, + response: http.ServerResponse, + getReply: () => string, +): void { + const chunks: Buffer[] = [] + request.on('data', (chunk: Buffer) => chunks.push(chunk)) + request.on('end', () => { + const url = request.url ?? '' + if (request.method === 'GET' && url.startsWith('/v1/models')) { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end( + JSON.stringify({ + object: 'list', + data: [{ id: MOCK_MODEL_ID, object: 'model', created: 0, owned_by: 'mock' }], + }), + ) + return + } + + if (request.method === 'POST' && url.startsWith('/v1/chat/completions')) { + let body: { stream?: boolean } = {} + try { + body = JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}') + } catch { + body = {} + } + + const mockReply = getReply() + const id = 'chatcmpl-mock' + const created = Math.floor(Date.now() / 1000) + const usage = { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } + + if (body.stream) { + response.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive', + }) + const base = { id, object: 'chat.completion.chunk', created, model: MOCK_MODEL_ID } + response.write( + `data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { role: 'assistant', content: '' }, finish_reason: null }] })}\n\n`, + ) + response.write( + `data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: { content: mockReply }, finish_reason: null }] })}\n\n`, + ) + response.write( + `data: ${JSON.stringify({ ...base, choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], usage })}\n\n`, + ) + response.write('data: [DONE]\n\n') + response.end() + return + } + + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end( + JSON.stringify({ + id, + object: 'chat.completion', + created, + model: MOCK_MODEL_ID, + choices: [{ index: 0, message: { role: 'assistant', content: mockReply }, finish_reason: 'stop' }], + usage, + }), + ) + return + } + + response.writeHead(404, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ error: { message: `unhandled ${request.method} ${url}` } })) + }) +} + +export async function startMockProvider(port: number): Promise { + let mockReply = '' + const server = http.createServer((request, response) => { + handleMockRequest(request, response, () => mockReply) + }) + await new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(port, '127.0.0.1', () => resolve()) + }) + return { + setReply: (text) => { + mockReply = text + }, + close: () => + new Promise((resolve) => { + server.close(() => resolve()) + server.closeAllConnections() + }), + } +} + +function git(directory: string, ...args: string[]): void { + execFileSync('git', args, { cwd: directory, stdio: 'pipe' }) +} + +export async function createSession(api: APIRequestContext, directory: string): Promise { + const response = await api.post('/api/opencode/api/session', { data: { location: { directory } } }) + if (!response.ok()) { + throw new Error(`create session failed: ${response.status()} ${await response.text()}`) + } + const body = (await response.json()) as { data: { id: string } } + return body.data.id +} + +export async function createMockRepo( + name: string, + port: number, + options: CreateMockRepoOptions = {}, +): Promise { + const directory = path.join(E2E_TMP, 'workspace', 'repos', name) + fs.rmSync(directory, { recursive: true, force: true }) + fs.mkdirSync(directory, { recursive: true }) + fs.writeFileSync( + path.join(directory, 'opencode.json'), + JSON.stringify( + { + $schema: 'https://opencode.ai/config.json', + provider: { + mock: { + npm: '@ai-sdk/openai-compatible', + name: 'Mock', + options: { baseURL: `http://127.0.0.1:${port}/v1`, apiKey: 'mock' }, + models: { [MOCK_MODEL_ID]: { name: 'Mock Model' } }, + }, + }, + model: `mock/${MOCK_MODEL_ID}`, + small_model: `mock/${MOCK_MODEL_ID}`, + }, + null, + 2, + ), + ) + fs.writeFileSync(path.join(directory, 'README.md'), '# mock repository\n') + options.beforeCommit?.(directory) + git(directory, '-c', 'init.defaultBranch=main', 'init', '-q') + git(directory, '-c', 'user.name=e2e', '-c', 'user.email=e2e@example.com', 'add', '-A') + git(directory, '-c', 'user.name=e2e', '-c', 'user.email=e2e@example.com', 'commit', '-q', '-m', 'initial commit') + + const api = await playwrightRequest.newContext({ baseURL: BASE_URL, storageState: STORAGE_STATE }) + try { + const repoResponse = await api.post('/api/repos', { data: { localPath: directory } }) + if (!repoResponse.ok()) { + throw new Error(`register mock repo failed: ${repoResponse.status()} ${await repoResponse.text()}`) + } + const repo = (await repoResponse.json()) as { id: number; fullPath: string } + const sessionId = await createSession(api, repo.fullPath) + return { id: repo.id, fullPath: repo.fullPath, sessionId } + } finally { + await api.dispose() + } +} diff --git a/e2e/tts-floating-button.spec.ts b/e2e/tts-floating-button.spec.ts new file mode 100644 index 000000000..d9fcee36a --- /dev/null +++ b/e2e/tts-floating-button.spec.ts @@ -0,0 +1,171 @@ +import { request as playwrightRequest, type APIRequestContext, type Page } from '@playwright/test' +import { test, expect } from './fixtures' +import { createMockRepo, startMockProvider, type MockProvider, type MockRepo } from './mockProvider' +import { BASE_URL, STORAGE_STATE } from './env' + +const MOCK_PROVIDER_PORT = 5566 +const MARKERS = ['ALPHAMARKER', 'BRAVOMARKER', 'CHARLIEMARKER'] as const + +const ENABLED_TTS = { + enabled: true, + provider: 'external' as const, + autoPlay: false, + endpoint: 'https://api.openai.com', + apiKey: 'e2e-test-key', + voice: 'alloy', + model: 'tts-1', + speed: 1, +} + +const DEFAULT_TTS = { + enabled: false, + provider: 'external' as const, + autoPlay: false, + endpoint: 'https://api.openai.com', + apiKey: '', + voice: 'alloy', + model: 'tts-1', + speed: 1, +} + +function buildSilentWav(durationSeconds: number): Buffer { + const sampleRate = 8000 + const channels = 1 + const bitsPerSample = 16 + const bytesPerSample = bitsPerSample / 8 + const dataSize = sampleRate * channels * bytesPerSample * durationSeconds + const buffer = Buffer.alloc(44 + dataSize) + buffer.write('RIFF', 0, 'ascii') + buffer.writeUInt32LE(36 + dataSize, 4) + buffer.write('WAVE', 8, 'ascii') + buffer.write('fmt ', 12, 'ascii') + buffer.writeUInt32LE(16, 16) + buffer.writeUInt16LE(1, 20) + buffer.writeUInt16LE(channels, 22) + buffer.writeUInt32LE(sampleRate, 24) + buffer.writeUInt32LE(sampleRate * channels * bytesPerSample, 28) + buffer.writeUInt16LE(channels * bytesPerSample, 32) + buffer.writeUInt16LE(bitsPerSample, 34) + buffer.write('data', 36, 'ascii') + buffer.writeUInt32LE(dataSize, 40) + return buffer +} + +const SILENT_WAV = buildSilentWav(3) + +function buildReply(marker: string): string { + return Array.from( + { length: 30 }, + (_, index) => + `${marker} paragraph ${index + 1}: this deliberately long filler sentence wraps across several lines on a narrow mobile viewport so the reply spans more than one screen.`, + ).join('\n\n') +} + +let api: APIRequestContext +let mockProvider: MockProvider +let mockRepo: MockRepo + +async function seedReplies(sessionId: string): Promise { + for (const marker of MARKERS) { + mockProvider.setReply(buildReply(marker)) + const response = await api.post(`/api/opencode/api/session/${sessionId}/prompt`, { + data: { text: 'Reply with the next prepared paragraph set.' }, + }) + if (!response.ok()) { + throw new Error(`seed prompt failed: ${response.status()} ${await response.text()}`) + } + await expect + .poll( + async () => { + const messagesResponse = await api.get(`/api/opencode/api/session/${sessionId}/message?order=desc`) + if (!messagesResponse.ok()) return false + const body = (await messagesResponse.json()) as { + data: Array<{ type?: string; time?: { completed?: number } }> + } + return body.data.some( + (message) => + message.type === 'assistant' && + message.time?.completed !== undefined && + JSON.stringify(message).includes(marker), + ) + }, + { timeout: 60_000 }, + ) + .toBe(true) + } +} + +test.use({ viewport: { width: 390, height: 844 }, hasTouch: true, isMobile: true }) + +test.beforeAll(async () => { + mockProvider = await startMockProvider(MOCK_PROVIDER_PORT) + mockRepo = await createMockRepo('e2e-tts-mock-repo', MOCK_PROVIDER_PORT) + api = await playwrightRequest.newContext({ baseURL: BASE_URL, storageState: STORAGE_STATE }) + const settingsResponse = await api.patch('/api/settings', { data: { preferences: { tts: ENABLED_TTS } } }) + if (!settingsResponse.ok()) { + throw new Error(`enable tts failed: ${settingsResponse.status()} ${await settingsResponse.text()}`) + } + await seedReplies(mockRepo.sessionId) +}) + +test.afterAll(async () => { + if (api) { + await api.patch('/api/settings', { data: { preferences: { tts: DEFAULT_TTS } } }) + await api.dispose() + } + if (mockProvider) { + await mockProvider.close() + } +}) + +async function recordSynthesis(page: Page, synthTexts: string[]): Promise { + await page.route('**/api/tts/synthesize', async (route) => { + const body = route.request().postDataJSON() as { text: string } + synthTexts.push(body.text) + await route.fulfill({ status: 200, contentType: 'audio/wav', body: SILENT_WAV }) + }) +} + +test('plays the latest reply at the bottom and stops on the next tap', async ({ page }) => { + const synthTexts: string[] = [] + await recordSynthesis(page, synthTexts) + + await page.goto(`/repos/${mockRepo.id}/sessions/${mockRepo.sessionId}`) + + const playButton = page.getByRole('button', { name: /play latest reply\. auto-play disabled/i }) + await expect(playButton).toBeVisible() + await playButton.tap() + + await expect.poll(() => synthTexts.join('\n')).toContain('CHARLIEMARKER') + expect(synthTexts.join('\n')).not.toContain('ALPHAMARKER') + expect(synthTexts.join('\n')).not.toContain('BRAVOMARKER') + + await expect(page.getByRole('button', { name: /stop playback\. auto-play disabled/i })).toBeVisible() + await page.getByRole('button', { name: /stop playback\. auto-play disabled/i }).tap() + await expect(page.getByRole('button', { name: /play latest reply\. auto-play disabled/i })).toBeVisible() +}) + +test('reads the visible replies when scrolled up and stops on the next tap', async ({ page }) => { + const synthTexts: string[] = [] + await recordSynthesis(page, synthTexts) + + await page.goto(`/repos/${mockRepo.id}/sessions/${mockRepo.sessionId}`) + await expect(page.getByRole('button', { name: /play latest reply\. auto-play disabled/i })).toBeVisible() + + const container = page.getByTestId('session-message-scroll') + await container.evaluate((element) => { + element.scrollTop = 0 + element.dispatchEvent(new WheelEvent('wheel', { deltaY: -120, bubbles: true })) + }) + + const readButton = page.getByRole('button', { name: /read visible replies\. auto-play disabled/i }) + await expect(readButton).toBeVisible() + await readButton.tap() + + await expect.poll(() => synthTexts.join('\n')).toContain('ALPHAMARKER') + expect(synthTexts.join('\n')).not.toContain('CHARLIEMARKER') + + await expect(page.getByRole('button', { name: /stop playback\. auto-play disabled/i })).toBeVisible() + await page.getByRole('button', { name: /stop playback\. auto-play disabled/i }).tap() + await expect(page.getByRole('button', { name: /read visible replies\. auto-play disabled/i })).toBeVisible() +}) diff --git a/frontend/src/components/message/FloatingTTSButton.test.tsx b/frontend/src/components/message/FloatingTTSButton.test.tsx index 0e50d4e7d..dca77b911 100644 --- a/frontend/src/components/message/FloatingTTSButton.test.tsx +++ b/frontend/src/components/message/FloatingTTSButton.test.tsx @@ -35,12 +35,14 @@ interface MockTTSReturn { describe('FloatingTTSButton', () => { const mockSpeakMessage = vi.fn() const mockStop = vi.fn() + const mockSpeakVisible = vi.fn() const mockUpdateSettings = vi.fn() beforeEach(() => { vi.clearAllMocks() mockSpeakMessage.mockClear() mockStop.mockClear() + mockSpeakVisible.mockClear() mockUpdateSettings.mockClear() mocks.showToastInfo.mockClear() }) @@ -84,19 +86,19 @@ describe('FloatingTTSButton', () => { it('renders pill when content is empty but autoplay is enabled', () => { setup({ autoPlay: true }) - render() + render() expect(screen.getByRole('button', { name: /hold to toggle auto-play/i })).toBeInTheDocument() }) it('renders pill when content is whitespace but autoplay is enabled', () => { setup({ autoPlay: true }) - render() + render() expect(screen.getByRole('button', { name: /hold to toggle auto-play/i })).toBeInTheDocument() }) it('calls speakMessage on tap when idle', () => { setup() - render() + render() const button = screen.getByRole('button', { name: /play latest reply/i }) fireEvent.pointerDown(button) @@ -110,7 +112,7 @@ describe('FloatingTTSButton', () => { it('calls stop on tap when this message is active', () => { setup({ activeMessageId: TEST_MESSAGE_ID, isPlaying: true }) - render() + render() const button = screen.getByRole('button', { name: /stop playback/i }) fireEvent.pointerDown(button) @@ -124,7 +126,7 @@ describe('FloatingTTSButton', () => { it('calls stop when any playback is active, even for a different messageId (stream conflict fix)', () => { setup({ activeMessageId: 'other-message-id', isPlaying: true }) - render() + render() const button = screen.getByRole('button', { name: /stop playback/i }) fireEvent.pointerDown(button) @@ -139,7 +141,7 @@ describe('FloatingTTSButton', () => { it('toggles autoplay on long press', async () => { vi.useFakeTimers() setup({ autoPlay: false }) - render() + render() const button = screen.getByRole('button', { name: /play latest reply/i }) await act(async () => { @@ -172,7 +174,7 @@ describe('FloatingTTSButton', () => { it('shows a toast when long press disables autoplay', async () => { vi.useFakeTimers() setup({ autoPlay: true }) - render() + render() const button = screen.getByRole('button', { name: /auto-play enabled/i }) await act(async () => { @@ -195,7 +197,7 @@ describe('FloatingTTSButton', () => { it('does not call speakMessage after long press (race guard)', async () => { vi.useFakeTimers() setup() - render() + render() const button = screen.getByRole('button', { name: /play latest reply/i }) await act(async () => { @@ -212,7 +214,7 @@ describe('FloatingTTSButton', () => { it('does not call speakMessage if pointer leaves before release after long press fires', async () => { vi.useFakeTimers() setup() - render() + render() const button = screen.getByRole('button', { name: /play latest reply/i }) await act(async () => { @@ -227,7 +229,7 @@ describe('FloatingTTSButton', () => { it('exposes the gesture hint in the aria-label for accessibility', () => { setup() - render() + render() const button = screen.getByRole('button', { name: /hold to toggle auto-play/i }) expect(button).toBeInTheDocument() @@ -235,7 +237,7 @@ describe('FloatingTTSButton', () => { it('shows stop icon when this message is playing', () => { setup({ autoPlay: true, activeMessageId: TEST_MESSAGE_ID, isPlaying: true }) - render() + render() const button = screen.getByRole('button', { name: /stop playback/i }) @@ -249,14 +251,14 @@ describe('FloatingTTSButton', () => { it('shows the same stop icon while this message is loading', () => { setup({ activeMessageId: TEST_MESSAGE_ID, isLoading: true }) - render() + render() expect(screen.getByRole('button', { name: /stop playback/i })).toHaveClass('bg-destructive') }) it('shows autoplay state with button color when autoPlay is enabled', () => { setup({ autoPlay: true }) - render() + render() const button = screen.getByRole('button', { name: /auto-play enabled/i }) @@ -269,7 +271,7 @@ describe('FloatingTTSButton', () => { it('uses the default play color when autoPlay is disabled', () => { setup({ autoPlay: false }) - render() + render() const button = screen.getByRole('button', { name: /play latest reply/i }) @@ -277,4 +279,68 @@ describe('FloatingTTSButton', () => { expect(button).not.toHaveClass('bg-primary') expect(screen.queryByText('Play')).not.toBeInTheDocument() }) + + it('reads visible replies on tap when scrolled up and idle', () => { + setup() + render( + + ) + + const button = screen.getByRole('button', { name: /read visible replies/i }) + fireEvent.pointerDown(button) + act(() => { + fireEvent.pointerUp(button) + }) + + expect(mockSpeakVisible).toHaveBeenCalledTimes(1) + expect(mockSpeakMessage).not.toHaveBeenCalled() + }) + + it('stops playback on tap when scrolled up and playing', () => { + setup({ activeMessageId: TEST_MESSAGE_ID, isPlaying: true }) + render( + + ) + + const button = screen.getByRole('button', { name: /stop playback/i }) + fireEvent.pointerDown(button) + act(() => { + fireEvent.pointerUp(button) + }) + + expect(mockStop).toHaveBeenCalledTimes(1) + expect(mockSpeakVisible).not.toHaveBeenCalled() + }) + + it('speaks the latest reply on tap when not scrolled up and idle', () => { + setup() + render( + + ) + + const button = screen.getByRole('button', { name: /play latest reply/i }) + fireEvent.pointerDown(button) + act(() => { + fireEvent.pointerUp(button) + }) + + expect(mockSpeakMessage).toHaveBeenCalledTimes(1) + expect(mockSpeakMessage).toHaveBeenCalledWith(TEST_MESSAGE_ID, TEST_CONTENT) + expect(mockSpeakVisible).not.toHaveBeenCalled() + }) }) diff --git a/frontend/src/components/message/FloatingTTSButton.tsx b/frontend/src/components/message/FloatingTTSButton.tsx index 5d4198bb3..2f67df556 100644 --- a/frontend/src/components/message/FloatingTTSButton.tsx +++ b/frontend/src/components/message/FloatingTTSButton.tsx @@ -8,11 +8,13 @@ import { DEFAULT_TTS_CONFIG } from '@opencode-manager/shared' interface FloatingTTSButtonProps { messageId: string content: string + isScrolledUp: boolean + onSpeakVisible: () => void } const LONG_PRESS_DURATION = 500 -export function FloatingTTSButton({ messageId, content }: FloatingTTSButtonProps) { +export function FloatingTTSButton({ messageId, content, isScrolledUp, onSpeakVisible }: FloatingTTSButtonProps) { const { speakMessage, stop, isPlaying, isLoading } = useTTS() const { preferences, updateSettings } = useSettings() @@ -63,10 +65,15 @@ export function FloatingTTSButton({ messageId, content }: FloatingTTSButtonProps return } + if (isScrolledUp) { + onSpeakVisible() + return + } + if (hasContent) { speakMessage(messageId, content) } - }, [clearLongPressTimer, isAnyPlaybackActive, stop, speakMessage, messageId, content, hasContent]) + }, [clearLongPressTimer, isAnyPlaybackActive, stop, isScrolledUp, onSpeakVisible, speakMessage, messageId, content, hasContent]) const handlePointerLeave = useCallback(() => { clearLongPressTimer() @@ -77,9 +84,11 @@ export function FloatingTTSButton({ messageId, content }: FloatingTTSButtonProps const showStop = isAnyPlaybackActive const pillTitle = showStop ? 'Stop playback' - : hasContent - ? 'Play latest reply' - : 'TTS controls' + : isScrolledUp + ? 'Read visible replies' + : hasContent + ? 'Play latest reply' + : 'TTS controls' const pillAriaLabel = `${pillTitle}. ${autoPlay ? 'Auto-play enabled.' : 'Auto-play disabled.'} hold to toggle auto-play` const buttonToneClasses = showStop ? 'justify-center px-3 py-1.5 rounded-lg bg-destructive border border-destructive/60 shadow-destructive/30 ring-destructive/20 hover:ring-destructive/40 text-destructive-foreground' diff --git a/frontend/src/pages/SessionDetail.tsx b/frontend/src/pages/SessionDetail.tsx index 93daf8659..e5eae4fe0 100644 --- a/frontend/src/pages/SessionDetail.tsx +++ b/frontend/src/pages/SessionDetail.tsx @@ -776,6 +776,8 @@ export function SessionDetail() { )} {hasPromptContent && !isSessionActive && ( From cc0e136f3cac2af0736d0bbf5b152d8c70e1f900 Mon Sep 17 00:00:00 2001 From: Chris Scott <99081550+chriswritescode-dev@users.noreply.github.com> Date: Tue, 13 Oct 2026 22:57:29 -0400 Subject: [PATCH 4/4] test(e2e): wait for TTS synthesis to settle before asserting exclusions --- e2e/tts-floating-button.spec.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/e2e/tts-floating-button.spec.ts b/e2e/tts-floating-button.spec.ts index d9fcee36a..567584f5e 100644 --- a/e2e/tts-floating-button.spec.ts +++ b/e2e/tts-floating-button.spec.ts @@ -126,6 +126,21 @@ async function recordSynthesis(page: Page, synthTexts: string[]): Promise }) } +async function waitForSynthesisSettled(synthTexts: string[], lastExpectedText: string): Promise { + await expect.poll(() => synthTexts.join('\n')).toContain(lastExpectedText) + let previousCount = -1 + await expect + .poll( + () => { + const isSettled = synthTexts.length === previousCount + previousCount = synthTexts.length + return isSettled + }, + { intervals: [500] }, + ) + .toBe(true) +} + test('plays the latest reply at the bottom and stops on the next tap', async ({ page }) => { const synthTexts: string[] = [] await recordSynthesis(page, synthTexts) @@ -136,7 +151,7 @@ test('plays the latest reply at the bottom and stops on the next tap', async ({ await expect(playButton).toBeVisible() await playButton.tap() - await expect.poll(() => synthTexts.join('\n')).toContain('CHARLIEMARKER') + await waitForSynthesisSettled(synthTexts, 'CHARLIEMARKER paragraph 30') expect(synthTexts.join('\n')).not.toContain('ALPHAMARKER') expect(synthTexts.join('\n')).not.toContain('BRAVOMARKER') @@ -162,7 +177,7 @@ test('reads the visible replies when scrolled up and stops on the next tap', asy await expect(readButton).toBeVisible() await readButton.tap() - await expect.poll(() => synthTexts.join('\n')).toContain('ALPHAMARKER') + await waitForSynthesisSettled(synthTexts, 'ALPHAMARKER paragraph 30') expect(synthTexts.join('\n')).not.toContain('CHARLIEMARKER') await expect(page.getByRole('button', { name: /stop playback\. auto-play disabled/i })).toBeVisible()