diff --git a/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts b/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts index 449471dc07..ffe6c4707f 100644 --- a/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts +++ b/apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts @@ -4,6 +4,7 @@ import { useCallback, useEffect, useRef, useState } from 'react' import { useI18n } from '@/i18n' import { chatMessageText, collectUnspokenTurnSpeech } from '@/lib/chat-messages' import { triggerHaptic } from '@/lib/haptics' +import { markAssistantIdSpoken, resolveSpokenReply } from '@/lib/spoken-reply' import { clearWakeIndicator, syncWakeIndicatorWithVoice } from '@/lib/wake-indicator' import { $voiceConversationStartRequest, takeVoiceConversationStart } from '@/store/composer' import { resetBrowseState } from '@/store/composer-input-history' @@ -62,7 +63,6 @@ export function useComposerVoice({ // A tile's composer speaks ITS transcript, not the primary chat's. const { $messages } = useComposerScope() const [voiceConversationActive, setVoiceConversationActive] = useState(false) - const lastSpokenIdRef = useRef(null) const ownsWakeIndicatorRef = useRef(false) const voiceStartRequest = useStore($voiceConversationStartRequest) @@ -77,8 +77,9 @@ export function useComposerVoice({ const pendingResponse = () => { const messages = $messages.get() const last = messages.findLast(m => m.role === 'assistant' && !m.hidden) + const spoken = resolveSpokenReply(sessionId, messages) - if (!last || last.id === lastSpokenIdRef.current) { + if (!last || last.id === spoken?.id) { return null } @@ -100,14 +101,18 @@ export function useComposerVoice({ * in order — narration interims AND the final answer, not just whichever * bubble happens to be last. See `collectUnspokenTurnSpeech`. */ - const pendingTurnResponse = () => collectUnspokenTurnSpeech($messages.get(), lastSpokenIdRef.current) + const pendingTurnResponse = () => { + const messages = $messages.get() + + return collectUnspokenTurnSpeech(messages, resolveSpokenReply(sessionId, messages)?.id ?? null) + } const consumePendingResponse = () => { const messages = $messages.get() const last = messages.findLast(m => m.role === 'assistant' && !m.hidden) if (last) { - lastSpokenIdRef.current = last.id + markAssistantIdSpoken(sessionId, messages, last.id) } } diff --git a/apps/desktop/src/components/assistant-ui/thread/assistant-message.tsx b/apps/desktop/src/components/assistant-ui/thread/assistant-message.tsx index e75121de8d..cdd725865f 100644 --- a/apps/desktop/src/components/assistant-ui/thread/assistant-message.tsx +++ b/apps/desktop/src/components/assistant-ui/thread/assistant-message.tsx @@ -9,6 +9,7 @@ import { import { useStore } from '@nanostores/react' import { type FC, useCallback, useMemo, useState } from 'react' +import { useSessionView } from '@/app/chat/session-view' import { ChangedFilesCard } from '@/components/assistant-ui/thread/changed-files-card' import { contentHasVisibleText, @@ -30,6 +31,7 @@ import { useI18n } from '@/i18n' import { triggerHaptic } from '@/lib/haptics' import { AudioLines, GitForkIcon, Loader2Icon, RefreshCwIcon, SmilePlusIcon, VolumeXIcon, XIcon } from '@/lib/icons' import { extractPreviewTargets } from '@/lib/preview-targets' +import { markAssistantIdSpoken } from '@/lib/spoken-reply' import { useEnterAnimation } from '@/lib/use-enter-animation' import { cn } from '@/lib/utils' import { playSpeechText, stopVoicePlayback } from '@/lib/voice-playback' @@ -342,6 +344,8 @@ const ReadAloudButton: FC<{ getText: () => string; messageId: string }> = ({ get const { t } = useI18n() const copy = t.assistant.thread const voicePlayback = useStore($voicePlayback) + const view = useSessionView() + const sessionId = useStore(view.$runtimeId) const readAloudStatus = voicePlayback.source === 'read-aloud' && voicePlayback.messageId === messageId ? voicePlayback.status : 'idle' @@ -361,10 +365,11 @@ const ReadAloudButton: FC<{ getText: () => string; messageId: string }> = ({ get try { await playSpeechText(text, { messageId, source: 'read-aloud' }) + markAssistantIdSpoken(sessionId, view.$messages.get(), messageId) } catch (error) { notifyError(error, copy.readAloudFailed) } - }, [copy.readAloudFailed, getText, messageId]) + }, [copy.readAloudFailed, getText, messageId, sessionId, view.$messages]) return ( ({ id, role: 'assistant' as const }) +const user = (id: string) => ({ id, role: 'user' as const }) +const hidden = (id: string) => ({ hidden: true, id, role: 'assistant' as const }) + +afterEach(() => { + clearSpokenRepliesForTests() +}) + +describe('isLiveTailReplyId', () => { + it('matches renderer stream and inflight ids only', () => { + expect(isLiveTailReplyId('assistant-stream-s1')).toBe(true) + expect(isLiveTailReplyId('inflight-assistant-9')).toBe(true) + expect(isLiveTailReplyId('42')).toBe(false) + expect(isLiveTailReplyId('1770-3-assistant')).toBe(false) + }) +}) + +describe('assistantReplyOrdinal', () => { + it('counts visible assistant bubbles and skips hidden ones', () => { + const messages = [user('u1'), assistant('a1'), hidden('skip'), assistant('a2')] + + expect(assistantReplyOrdinal(messages, 'a1')).toBe(0) + expect(assistantReplyOrdinal(messages, 'a2')).toBe(1) + expect(assistantReplyOrdinal(messages, 'missing')).toBe(-1) + }) +}) + +describe('absorbSpokenReplyRewrite', () => { + it('stays silent when the live-tail id is rewritten at the same ordinal', () => { + const spoken = { id: 'assistant-stream-s', ordinal: 1 } + const after = [user('u1'), assistant('a0'), assistant('42')] + + expect(absorbSpokenReplyRewrite(spoken, after)).toEqual({ id: '42', ordinal: 1 }) + }) + + it('does not treat a later same-slot-looking turn as the rewrite when ordinal moved', () => { + const spoken = { id: 'assistant-stream-s', ordinal: 0 } + const after = [user('u1'), assistant('durable-1'), user('u2'), assistant('assistant-stream-next')] + + expect(absorbSpokenReplyRewrite(spoken, after)).toEqual(spoken) + }) + + it('does not migrate a durable id that simply vanished', () => { + const spoken = { id: 'durable-old', ordinal: 0 } + const after = [assistant('durable-new')] + + expect(absorbSpokenReplyRewrite(spoken, after)).toEqual(spoken) + }) + + it('keeps the anchor when the spoken id is still in the list', () => { + const spoken = { id: 'assistant-stream-s', ordinal: 0 } + const messages = [assistant('assistant-stream-s')] + + expect(absorbSpokenReplyRewrite(spoken, messages)).toBe(spoken) + }) +}) + +describe('resolveSpokenReply', () => { + it('migrates per session and does not leak across sessions', () => { + const before = [assistant('assistant-stream-s')] + markAssistantIdSpoken('session-a', before, 'assistant-stream-s') + + const after = [assistant('42')] + expect(resolveSpokenReply('session-a', after)?.id).toBe('42') + expect(spokenReplyOf('session-b')).toBeNull() + expect(resolveSpokenReply('session-b', after)).toBeNull() + }) + + it('lets a second turn at the next ordinal stay unspoken', () => { + markAssistantIdSpoken('s', [assistant('assistant-stream-1')], 'assistant-stream-1') + resolveSpokenReply('s', [assistant('durable-1')]) + + const nextTurn = [assistant('durable-1'), assistant('assistant-stream-2')] + const spoken = resolveSpokenReply('s', nextTurn) + + expect(spoken?.id).toBe('durable-1') + expect(assistantReplyOrdinal(nextTurn, 'assistant-stream-2')).toBe(1) + expect(spoken?.ordinal).toBe(0) + }) +}) diff --git a/apps/desktop/src/lib/spoken-reply.ts b/apps/desktop/src/lib/spoken-reply.ts new file mode 100644 index 0000000000..fe275824d2 --- /dev/null +++ b/apps/desktop/src/lib/spoken-reply.ts @@ -0,0 +1,131 @@ +/** + * Spoken-reply identity for Desktop auto-speak / Read Aloud. + * + * The live assistant row id (`assistant-stream-*`, `inflight-assistant-*`) is + * not stable: hydrate rewrites that row under its durable backend id. Keying + * "already spoken" on id alone then re-reads the same turn at the playback-idle + * edge. A content fingerprint would swallow a later distinct turn that happens + * to say the same thing ("Done."). + * + * Anchor on the assistant-role ordinal (nth visible assistant bubble). The + * rewrite keeps that slot; a new turn appends and the ordinal moves. + */ + +export interface SpokenReplyAnchor { + id: string + ordinal: number +} + +export interface SpokenReplyMessage { + hidden?: boolean + id: string + role: string +} + +const NO_SESSION = '\0' + +const lastSpokenBySession = new Map() + +export function isLiveTailReplyId(id: string): boolean { + return id.startsWith('assistant-stream-') || id.startsWith('inflight-assistant-') +} + +function sessionKey(sessionId: string | null | undefined): string { + return sessionId ?? NO_SESSION +} + +export function assistantReplyOrdinal(messages: readonly SpokenReplyMessage[], id: string): number { + let ordinal = -1 + + for (const message of messages) { + if (message.role !== 'assistant' || message.hidden) { + continue + } + + ordinal += 1 + + if (message.id === id) { + return ordinal + } + } + + return -1 +} + +function lastVisibleAssistant(messages: readonly SpokenReplyMessage[]): SpokenReplyMessage | undefined { + return messages.findLast(message => message.role === 'assistant' && !message.hidden) +} + +/** If a spoken live-tail row vanished and the same assistant slot now has a + * durable id, migrate the anchor. Leave durable ids and later turns alone. */ +export function absorbSpokenReplyRewrite( + spoken: SpokenReplyAnchor | null, + messages: readonly SpokenReplyMessage[] +): SpokenReplyAnchor | null { + if (!spoken) { + return null + } + + if (assistantReplyOrdinal(messages, spoken.id) >= 0) { + return spoken + } + + if (!isLiveTailReplyId(spoken.id)) { + return spoken + } + + const last = lastVisibleAssistant(messages) + + if (!last) { + return spoken + } + + const ordinal = assistantReplyOrdinal(messages, last.id) + + if (ordinal !== spoken.ordinal) { + return spoken + } + + return { id: last.id, ordinal } +} + +export function spokenReplyOf(sessionId: string | null | undefined): SpokenReplyAnchor | null { + return lastSpokenBySession.get(sessionKey(sessionId)) ?? null +} + +function markSpokenReply(sessionId: string | null | undefined, anchor: SpokenReplyAnchor): void { + lastSpokenBySession.set(sessionKey(sessionId), anchor) +} + +export function markAssistantIdSpoken( + sessionId: string | null | undefined, + messages: readonly SpokenReplyMessage[], + id: string +): void { + const ordinal = assistantReplyOrdinal(messages, id) + + if (ordinal < 0) { + return + } + + markSpokenReply(sessionId, { id, ordinal }) +} + +/** Current spoken anchor, migrated in place when the live row was rewritten. */ +export function resolveSpokenReply( + sessionId: string | null | undefined, + messages: readonly SpokenReplyMessage[] +): SpokenReplyAnchor | null { + const current = spokenReplyOf(sessionId) + const next = absorbSpokenReplyRewrite(current, messages) + + if (next && next.id !== current?.id) { + markSpokenReply(sessionId, next) + } + + return next +} + +export function clearSpokenRepliesForTests(): void { + lastSpokenBySession.clear() +}