fix(desktop): keep auto-speak silent across the stream-id rewrite

Supersedes #75649, #86637, #87672, #88642.

Fixes #86601
Fixes #87652
Fixes #87823

Co-authored-by: Charmmy <lilShawtty@qq.com>
Co-authored-by: chelsealong <chelsealong@126.com>
Co-authored-by: Olympusbuildz <Olympus.roots@outlook.com>
Co-authored-by: Ricardo Mendes <ricardo.mendes@maiolabs.ai>

apps/desktop/src/app/chat/composer/hooks/use-composer-voice.ts

@@ -4,6 +4,7 @@ import { useCallback, useEffect, useRef, useState } from 'react'
 import { useI18n } from '@/i18n'
 import { chatMessageText, collectUnspokenTurnSpeech } from '@/lib/chat-messages'
 import { triggerHaptic } from '@/lib/haptics'
+import { markAssistantIdSpoken, resolveSpokenReply } from '@/lib/spoken-reply'
 import { clearWakeIndicator, syncWakeIndicatorWithVoice } from '@/lib/wake-indicator'
 import { $voiceConversationStartRequest, takeVoiceConversationStart } from '@/store/composer'
 import { resetBrowseState } from '@/store/composer-input-history'
@@ -62,7 +63,6 @@ export function useComposerVoice({
   // A tile's composer speaks ITS transcript, not the primary chat's.
   const { $messages } = useComposerScope()
   const [voiceConversationActive, setVoiceConversationActive] = useState(false)
-  const lastSpokenIdRef = useRef<string | null>(null)
   const ownsWakeIndicatorRef = useRef(false)
   const voiceStartRequest = useStore($voiceConversationStartRequest)
 
@@ -77,8 +77,9 @@ export function useComposerVoice({
   const pendingResponse = () => {
     const messages = $messages.get()
     const last = messages.findLast(m => m.role === 'assistant' && !m.hidden)
+    const spoken = resolveSpokenReply(sessionId, messages)
 
-    if (!last || last.id === lastSpokenIdRef.current) {
+    if (!last || last.id === spoken?.id) {
       return null
     }
 
@@ -100,14 +101,18 @@ export function useComposerVoice({
    * in order — narration interims AND the final answer, not just whichever
    * bubble happens to be last. See `collectUnspokenTurnSpeech`.
    */
-  const pendingTurnResponse = () => collectUnspokenTurnSpeech($messages.get(), lastSpokenIdRef.current)
+  const pendingTurnResponse = () => {
+    const messages = $messages.get()
+
+    return collectUnspokenTurnSpeech(messages, resolveSpokenReply(sessionId, messages)?.id ?? null)
+  }
 
   const consumePendingResponse = () => {
     const messages = $messages.get()
     const last = messages.findLast(m => m.role === 'assistant' && !m.hidden)
 
     if (last) {
-      lastSpokenIdRef.current = last.id
+      markAssistantIdSpoken(sessionId, messages, last.id)
     }
   }
 

apps/desktop/src/components/assistant-ui/thread/assistant-message.tsx

@@ -9,6 +9,7 @@ import {
 import { useStore } from '@nanostores/react'
 import { type FC, useCallback, useMemo, useState } from 'react'
 
+import { useSessionView } from '@/app/chat/session-view'
 import { ChangedFilesCard } from '@/components/assistant-ui/thread/changed-files-card'
 import {
   contentHasVisibleText,
@@ -30,6 +31,7 @@ import { useI18n } from '@/i18n'
 import { triggerHaptic } from '@/lib/haptics'
 import { AudioLines, GitForkIcon, Loader2Icon, RefreshCwIcon, SmilePlusIcon, VolumeXIcon, XIcon } from '@/lib/icons'
 import { extractPreviewTargets } from '@/lib/preview-targets'
+import { markAssistantIdSpoken } from '@/lib/spoken-reply'
 import { useEnterAnimation } from '@/lib/use-enter-animation'
 import { cn } from '@/lib/utils'
 import { playSpeechText, stopVoicePlayback } from '@/lib/voice-playback'
@@ -342,6 +344,8 @@ const ReadAloudButton: FC<{ getText: () => string; messageId: string }> = ({ get
   const { t } = useI18n()
   const copy = t.assistant.thread
   const voicePlayback = useStore($voicePlayback)
+  const view = useSessionView()
+  const sessionId = useStore(view.$runtimeId)
 
   const readAloudStatus =
     voicePlayback.source === 'read-aloud' && voicePlayback.messageId === messageId ? voicePlayback.status : 'idle'
@@ -361,10 +365,11 @@ const ReadAloudButton: FC<{ getText: () => string; messageId: string }> = ({ get
 
     try {
       await playSpeechText(text, { messageId, source: 'read-aloud' })
+      markAssistantIdSpoken(sessionId, view.$messages.get(), messageId)
     } catch (error) {
       notifyError(error, copy.readAloudFailed)
     }
-  }, [copy.readAloudFailed, getText, messageId])
+  }, [copy.readAloudFailed, getText, messageId, sessionId, view.$messages])
 
   return (
     <TooltipIconButton

apps/desktop/src/lib/spoken-reply.test.ts

@@ -0,0 +1,92 @@
+import { afterEach, describe, expect, it } from 'vitest'
+
+import {
+  absorbSpokenReplyRewrite,
+  assistantReplyOrdinal,
+  clearSpokenRepliesForTests,
+  isLiveTailReplyId,
+  markAssistantIdSpoken,
+  resolveSpokenReply,
+  spokenReplyOf
+} from './spoken-reply'
+
+const assistant = (id: string) => ({ id, role: 'assistant' as const })
+const user = (id: string) => ({ id, role: 'user' as const })
+const hidden = (id: string) => ({ hidden: true, id, role: 'assistant' as const })
+
+afterEach(() => {
+  clearSpokenRepliesForTests()
+})
+
+describe('isLiveTailReplyId', () => {
+  it('matches renderer stream and inflight ids only', () => {
+    expect(isLiveTailReplyId('assistant-stream-s1')).toBe(true)
+    expect(isLiveTailReplyId('inflight-assistant-9')).toBe(true)
+    expect(isLiveTailReplyId('42')).toBe(false)
+    expect(isLiveTailReplyId('1770-3-assistant')).toBe(false)
+  })
+})
+
+describe('assistantReplyOrdinal', () => {
+  it('counts visible assistant bubbles and skips hidden ones', () => {
+    const messages = [user('u1'), assistant('a1'), hidden('skip'), assistant('a2')]
+
+    expect(assistantReplyOrdinal(messages, 'a1')).toBe(0)
+    expect(assistantReplyOrdinal(messages, 'a2')).toBe(1)
+    expect(assistantReplyOrdinal(messages, 'missing')).toBe(-1)
+  })
+})
+
+describe('absorbSpokenReplyRewrite', () => {
+  it('stays silent when the live-tail id is rewritten at the same ordinal', () => {
+    const spoken = { id: 'assistant-stream-s', ordinal: 1 }
+    const after = [user('u1'), assistant('a0'), assistant('42')]
+
+    expect(absorbSpokenReplyRewrite(spoken, after)).toEqual({ id: '42', ordinal: 1 })
+  })
+
+  it('does not treat a later same-slot-looking turn as the rewrite when ordinal moved', () => {
+    const spoken = { id: 'assistant-stream-s', ordinal: 0 }
+    const after = [user('u1'), assistant('durable-1'), user('u2'), assistant('assistant-stream-next')]
+
+    expect(absorbSpokenReplyRewrite(spoken, after)).toEqual(spoken)
+  })
+
+  it('does not migrate a durable id that simply vanished', () => {
+    const spoken = { id: 'durable-old', ordinal: 0 }
+    const after = [assistant('durable-new')]
+
+    expect(absorbSpokenReplyRewrite(spoken, after)).toEqual(spoken)
+  })
+
+  it('keeps the anchor when the spoken id is still in the list', () => {
+    const spoken = { id: 'assistant-stream-s', ordinal: 0 }
+    const messages = [assistant('assistant-stream-s')]
+
+    expect(absorbSpokenReplyRewrite(spoken, messages)).toBe(spoken)
+  })
+})
+
+describe('resolveSpokenReply', () => {
+  it('migrates per session and does not leak across sessions', () => {
+    const before = [assistant('assistant-stream-s')]
+    markAssistantIdSpoken('session-a', before, 'assistant-stream-s')
+
+    const after = [assistant('42')]
+    expect(resolveSpokenReply('session-a', after)?.id).toBe('42')
+    expect(spokenReplyOf('session-b')).toBeNull()
+    expect(resolveSpokenReply('session-b', after)).toBeNull()
+  })
+
+  it('lets a second turn at the next ordinal stay unspoken', () => {
+    markAssistantIdSpoken('s', [assistant('assistant-stream-1')], 'assistant-stream-1')
+    resolveSpokenReply('s', [assistant('durable-1')])
+
+    const nextTurn = [assistant('durable-1'), assistant('assistant-stream-2')]
+    const spoken = resolveSpokenReply('s', nextTurn)
+
+    expect(spoken?.id).toBe('durable-1')
+    expect(assistantReplyOrdinal(nextTurn, 'assistant-stream-2')).toBe(1)
+    expect(spoken?.ordinal).toBe(0)
+  })
+})

apps/desktop/src/lib/spoken-reply.ts

@@ -0,0 +1,131 @@
+/**
+ * Spoken-reply identity for Desktop auto-speak / Read Aloud.
+ *
+ * The live assistant row id (`assistant-stream-*`, `inflight-assistant-*`) is
+ * not stable: hydrate rewrites that row under its durable backend id. Keying
+ * "already spoken" on id alone then re-reads the same turn at the playback-idle
+ * edge. A content fingerprint would swallow a later distinct turn that happens
+ * to say the same thing ("Done.").
+ *
+ * Anchor on the assistant-role ordinal (nth visible assistant bubble). The
+ * rewrite keeps that slot; a new turn appends and the ordinal moves.
+ */
+
+export interface SpokenReplyAnchor {
+  id: string
+  ordinal: number
+}
+
+export interface SpokenReplyMessage {
+  hidden?: boolean
+  id: string
+  role: string
+}
+
+const NO_SESSION = '\0'
+
+const lastSpokenBySession = new Map<string, SpokenReplyAnchor>()
+
+export function isLiveTailReplyId(id: string): boolean {
+  return id.startsWith('assistant-stream-') || id.startsWith('inflight-assistant-')
+}
+
+function sessionKey(sessionId: string | null | undefined): string {
+  return sessionId ?? NO_SESSION
+}
+
+export function assistantReplyOrdinal(messages: readonly SpokenReplyMessage[], id: string): number {
+  let ordinal = -1
+
+  for (const message of messages) {
+    if (message.role !== 'assistant' || message.hidden) {
+      continue
+    }
+
+    ordinal += 1
+
+    if (message.id === id) {
+      return ordinal
+    }
+  }
+
+  return -1
+}
+
+function lastVisibleAssistant(messages: readonly SpokenReplyMessage[]): SpokenReplyMessage | undefined {
+  return messages.findLast(message => message.role === 'assistant' && !message.hidden)
+}
+
+/** If a spoken live-tail row vanished and the same assistant slot now has a
+ *  durable id, migrate the anchor. Leave durable ids and later turns alone. */
+export function absorbSpokenReplyRewrite(
+  spoken: SpokenReplyAnchor | null,
+  messages: readonly SpokenReplyMessage[]
+): SpokenReplyAnchor | null {
+  if (!spoken) {
+    return null
+  }
+
+  if (assistantReplyOrdinal(messages, spoken.id) >= 0) {
+    return spoken
+  }
+
+  if (!isLiveTailReplyId(spoken.id)) {
+    return spoken
+  }
+
+  const last = lastVisibleAssistant(messages)
+
+  if (!last) {
+    return spoken
+  }
+
+  const ordinal = assistantReplyOrdinal(messages, last.id)
+
+  if (ordinal !== spoken.ordinal) {
+    return spoken
+  }
+
+  return { id: last.id, ordinal }
+}
+
+export function spokenReplyOf(sessionId: string | null | undefined): SpokenReplyAnchor | null {
+  return lastSpokenBySession.get(sessionKey(sessionId)) ?? null
+}
+
+function markSpokenReply(sessionId: string | null | undefined, anchor: SpokenReplyAnchor): void {
+  lastSpokenBySession.set(sessionKey(sessionId), anchor)
+}
+
+export function markAssistantIdSpoken(
+  sessionId: string | null | undefined,
+  messages: readonly SpokenReplyMessage[],
+  id: string
+): void {
+  const ordinal = assistantReplyOrdinal(messages, id)
+
+  if (ordinal < 0) {
+    return
+  }
+
+  markSpokenReply(sessionId, { id, ordinal })
+}
+
+/** Current spoken anchor, migrated in place when the live row was rewritten. */
+export function resolveSpokenReply(
+  sessionId: string | null | undefined,
+  messages: readonly SpokenReplyMessage[]
+): SpokenReplyAnchor | null {
+  const current = spokenReplyOf(sessionId)
+  const next = absorbSpokenReplyRewrite(current, messages)
+
+  if (next && next.id !== current?.id) {
+    markSpokenReply(sessionId, next)
+  }
+
+  return next
+}
+
+export function clearSpokenRepliesForTests(): void {
+  lastSpokenBySession.clear()
+}