fix(desktop): stale running arcs clear on gateway reconnect (#53902, #73082)

A respawned backend re-mints runtime ids, so a pre-reconnect busy state
never receives its terminal busy:false publish and its session stayed in
$workingSessionIds forever - the sidebar running arc and agents-panel
'running' chrome lied for hours after the turn ended (the stale-flag
half of #53902/#73082; the CSS cost half landed in #91383).

reconcileBusyStatesOnReconnect() downgrades busy/awaitingResponse states
through publishSessionState (watchdogs disarm, stall hints drop, settle/
unread bookkeeping stays consistent), scoped by event-source: the primary
reconnect touches only scope-less runtimes, a secondary (registry)
reconnect touches only its own connection's. needsInput survives - a
blocking prompt is the user's to answer. A genuinely live turn re-asserts
busy on its next post-reconnect event, so the worst case is one arc blink.

Regression tests proven by sabotage run (neutered reconcile -> 5/6 fail).
This commit is contained in:
Teknium
2026-08-21 03:40:23 -07:00
parent 17d1095443
commit 4f64807f5d
4 changed files with 175 additions and 0 deletions
@@ -52,6 +52,7 @@ import {
$attentionSessionIds,
$workingSessionIds,
liveSessionScopes,
reconcileBusyStatesOnReconnect,
recordSessionEventScope,
resetTileRuntimeBindings
} from '@/store/session-states'
@@ -252,6 +253,12 @@ export function useGatewayBoot({
// A respawned backend re-mints (recycles) runtime ids, so any tile's
// bound runtime id is now stale — drop them so each tile re-resumes.
resetTileRuntimeBindings()
// Same staleness, other half: pre-reconnect busy flags are keyed by
// those dead runtime ids and would never receive their terminal
// busy:false — clear them or the sidebar running arc lies forever
// (#53902/#73082). A genuinely live turn re-asserts busy on its next
// post-reconnect event.
reconcileBusyStatesOnReconnect()
// Resync state that may have moved on the backend while we were asleep.
await callbacksRef.current.refreshHermesConfig().catch(() => undefined)
await callbacksRef.current.refreshSessions().catch(() => undefined)
+9
View File
@@ -6,6 +6,7 @@ import { HermesGateway, setApiRequestConnection } from '@/hermes'
import { reconnectBackoffDelayMs } from '@/lib/reconnect-backoff'
import { markNativeNotifyBaseline } from '@/store/notify-baseline'
import { setConnection, setGatewayState } from '@/store/session'
import { reconcileBusyStatesOnReconnect } from '@/store/session-states'
// ── Multi-profile gateway routing ──────────────────────────────────────────
// Concurrent sessions across profiles need concurrent sockets: the renderer's
@@ -381,6 +382,14 @@ async function reconnectSecondary(entry: Secondary): Promise<void> {
try {
await openSecondary(entry)
entry.reconnectAttempt = 0
// The re-dialed backend may have respawned and re-minted runtime ids —
// busy flags recorded from THIS socket's pre-drop events would then never
// receive their terminal busy:false, leaving the session's running arc
// armed forever (#53902/#73082 stale-flag half). Scoped: only runtimes
// whose events arrived on this connection are reconciled; live work on
// other sockets is untouched, and a genuinely live turn here re-asserts
// busy on its next event.
reconcileBusyStatesOnReconnect(entry.scope)
} catch (error) {
// The registry no longer knows this connection (removed while we were
// backing off), or Electron's deletion guard reports the profile itself
@@ -0,0 +1,116 @@
import { registryBackendScopeKey } from '@hermes/shared'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { ClientSessionState } from '@/app/types'
import { createClientSessionState } from '@/lib/chat-runtime'
import { $activeSessionId, $selectedStoredSessionId, $unreadFinishedSessionIds } from './session'
import {
$attentionSessionIds,
$stalledSessionIds,
$workingSessionIds,
clearAllSessionStates,
publishSessionState,
reconcileBusyStatesOnReconnect,
recordSessionEventScope,
SESSION_WATCHDOG_TIMEOUT_MS
} from './session-states'
function state(over: Partial<ClientSessionState> = {}): ClientSessionState {
return { ...createClientSessionState(null), storedSessionId: 's1', ...over }
}
// The stale-flag half of #53902/#73082: a backend respawn re-mints runtime
// ids, so a pre-reconnect busy state never receives its terminal busy:false
// and the session's running arc stays armed forever. The reconnect paths call
// reconcileBusyStatesOnReconnect to retire those claims.
describe('reconcileBusyStatesOnReconnect', () => {
beforeEach(() => {
vi.useFakeTimers()
vi.setSystemTime(0)
clearAllSessionStates()
$unreadFinishedSessionIds.set([])
$selectedStoredSessionId.set(null)
$activeSessionId.set(null)
})
afterEach(() => {
vi.runOnlyPendingTimers()
vi.useRealTimers()
clearAllSessionStates()
$unreadFinishedSessionIds.set([])
$selectedStoredSessionId.set(null)
$activeSessionId.set(null)
})
it('clears a stale busy session on primary reconnect', () => {
publishSessionState('rt1', state({ busy: true, storedSessionId: 's1' }))
expect($workingSessionIds.get()).toContain('s1')
reconcileBusyStatesOnReconnect()
expect($workingSessionIds.get()).not.toContain('s1')
})
it('disarms the stall watchdog with the busy claim', () => {
publishSessionState('rt1', state({ busy: true, storedSessionId: 's1' }))
reconcileBusyStatesOnReconnect()
// Without reconcile the watchdog would fire and paint s1 stalled.
vi.advanceTimersByTime(SESSION_WATCHDOG_TIMEOUT_MS + 1000)
expect($stalledSessionIds.get()).not.toContain('s1')
})
it('preserves needsInput — a blocking prompt is not a stale flag', () => {
publishSessionState('rt1', state({ busy: true, needsInput: true, storedSessionId: 's1' }))
expect($attentionSessionIds.get()).toContain('s1')
reconcileBusyStatesOnReconnect()
expect($workingSessionIds.get()).not.toContain('s1')
expect($attentionSessionIds.get()).toContain('s1')
})
it('primary reconcile leaves registry-scoped sessions alone', () => {
const scope = registryBackendScopeKey('connA', 'default')
publishSessionState('rtA', state({ busy: true, storedSessionId: 'sA' }))
recordSessionEventScope({ connectionId: 'connA', profile: 'default', session_id: 'rtA' })
publishSessionState('rtLocal', state({ busy: true, storedSessionId: 'sLocal' }))
reconcileBusyStatesOnReconnect()
expect($workingSessionIds.get()).toContain('sA')
expect($workingSessionIds.get()).not.toContain('sLocal')
// And the scoped variant clears ONLY its own connection's sessions.
reconcileBusyStatesOnReconnect(scope)
expect($workingSessionIds.get()).not.toContain('sA')
})
it('scoped reconcile does not touch other connections or the primary', () => {
publishSessionState('rtA', state({ busy: true, storedSessionId: 'sA' }))
recordSessionEventScope({ connectionId: 'connA', profile: 'default', session_id: 'rtA' })
publishSessionState('rtB', state({ busy: true, storedSessionId: 'sB' }))
recordSessionEventScope({ connectionId: 'connB', profile: 'default', session_id: 'rtB' })
publishSessionState('rtLocal', state({ busy: true, storedSessionId: 'sLocal' }))
reconcileBusyStatesOnReconnect(registryBackendScopeKey('connA', 'default'))
expect($workingSessionIds.get()).not.toContain('sA')
expect($workingSessionIds.get()).toContain('sB')
expect($workingSessionIds.get()).toContain('sLocal')
})
it('a live turn re-asserting busy after reconcile re-arms the arc', () => {
const s = state({ busy: true, storedSessionId: 's1' })
publishSessionState('rt1', s)
reconcileBusyStatesOnReconnect()
expect($workingSessionIds.get()).not.toContain('s1')
// The still-alive backend's next event republishes busy under a live id.
publishSessionState('rt2', state({ busy: true, storedSessionId: 's1' }))
expect($workingSessionIds.get()).toContain('s1')
})
})
+43
View File
@@ -373,6 +373,49 @@ export function clearAllSessionStates() {
$sessionStates.set({})
}
/** Downgrade cached busy/awaiting states after a gateway reconnect.
*
* A respawned backend re-mints runtime ids (the same fact that drives
* resetTileRuntimeBindings), so a pre-reconnect `busy` can never receive its
* terminal `busy: false` publish — the runtime id it would arrive under is
* dead. Left alone, that state keeps its session in $workingSessionIds
* forever: the sidebar running arc and agents-panel "running" chrome lie for
* hours after the turn actually ended (#53902, #73082 — stale-flag half).
*
* `scope` picks which socket's sessions to reconcile, keyed by the event-
* source scope recorded at fan-in: a SECONDARY (registry) reconnect passes
* its composite scope and touches only runtimes that arrived on that socket;
* the PRIMARY reconnect passes undefined and touches only scope-less
* runtimes (primary/local events record no scope). Neither can clear live
* work riding a different, still-healthy connection.
*
* Direction of failure is deliberate: a turn that IS still live (transient
* socket blip, same backend) re-asserts busy on its next event or inflight
* snapshot within a beat, so at worst its arc blinks once. A dead turn's
* state, by contrast, would never clear on its own. `needsInput` is left
* untouched — a blocking prompt is the one claim the user must explicitly
* answer, and post-reconnect refresh re-asserts or retires it via its own
* path. Transition side-effects run through publishSessionState, so
* watchdogs disarm, stall hints drop, and settle/unread bookkeeping stays
* consistent. */
export function reconcileBusyStatesOnReconnect(scope?: string) {
const states = $sessionStates.get()
for (const [runtimeId, state] of Object.entries(states)) {
if (!state || (!state.busy && !state.awaitingResponse)) {
continue
}
const recorded = sessionScopeByRuntimeId.get(runtimeId)
if (scope === undefined ? recorded !== undefined : recorded !== scope) {
continue
}
publishSessionState(runtimeId, { ...state, awaitingResponse: false, busy: false })
}
}
// Derived per-session status sets — pure projections of `$sessionStates` (which
// holds `busy`/`needsInput` per runtime), keeping the data flow one-directional:
// gateway event → cache → $sessionStates → computed views.