From 4f64807f5da1efe765ff90e8774e3d09d4487c81 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Fri, 21 Aug 2026 03:40:23 -0700 Subject: [PATCH] fix(desktop): stale running arcs clear on gateway reconnect (#53902, #73082) A respawned backend re-mints runtime ids, so a pre-reconnect busy state never receives its terminal busy:false publish and its session stayed in $workingSessionIds forever - the sidebar running arc and agents-panel 'running' chrome lied for hours after the turn ended (the stale-flag half of #53902/#73082; the CSS cost half landed in #91383). reconcileBusyStatesOnReconnect() downgrades busy/awaitingResponse states through publishSessionState (watchdogs disarm, stall hints drop, settle/ unread bookkeeping stays consistent), scoped by event-source: the primary reconnect touches only scope-less runtimes, a secondary (registry) reconnect touches only its own connection's. needsInput survives - a blocking prompt is the user's to answer. A genuinely live turn re-asserts busy on its next post-reconnect event, so the worst case is one arc blink. Regression tests proven by sabotage run (neutered reconcile -> 5/6 fail). --- .../src/app/gateway/hooks/use-gateway-boot.ts | 7 ++ apps/desktop/src/store/gateway.ts | 9 ++ .../store/session-states-reconnect.test.ts | 116 ++++++++++++++++++ apps/desktop/src/store/session-states.ts | 43 +++++++ 4 files changed, 175 insertions(+) create mode 100644 apps/desktop/src/store/session-states-reconnect.test.ts diff --git a/apps/desktop/src/app/gateway/hooks/use-gateway-boot.ts b/apps/desktop/src/app/gateway/hooks/use-gateway-boot.ts index 0300430b30..3d0e7e35b6 100644 --- a/apps/desktop/src/app/gateway/hooks/use-gateway-boot.ts +++ b/apps/desktop/src/app/gateway/hooks/use-gateway-boot.ts @@ -52,6 +52,7 @@ import { $attentionSessionIds, $workingSessionIds, liveSessionScopes, + reconcileBusyStatesOnReconnect, recordSessionEventScope, resetTileRuntimeBindings } from '@/store/session-states' @@ -252,6 +253,12 @@ export function useGatewayBoot({ // A respawned backend re-mints (recycles) runtime ids, so any tile's // bound runtime id is now stale — drop them so each tile re-resumes. resetTileRuntimeBindings() + // Same staleness, other half: pre-reconnect busy flags are keyed by + // those dead runtime ids and would never receive their terminal + // busy:false — clear them or the sidebar running arc lies forever + // (#53902/#73082). A genuinely live turn re-asserts busy on its next + // post-reconnect event. + reconcileBusyStatesOnReconnect() // Resync state that may have moved on the backend while we were asleep. await callbacksRef.current.refreshHermesConfig().catch(() => undefined) await callbacksRef.current.refreshSessions().catch(() => undefined) diff --git a/apps/desktop/src/store/gateway.ts b/apps/desktop/src/store/gateway.ts index 05a7920bf5..a6f321f4d6 100644 --- a/apps/desktop/src/store/gateway.ts +++ b/apps/desktop/src/store/gateway.ts @@ -6,6 +6,7 @@ import { HermesGateway, setApiRequestConnection } from '@/hermes' import { reconnectBackoffDelayMs } from '@/lib/reconnect-backoff' import { markNativeNotifyBaseline } from '@/store/notify-baseline' import { setConnection, setGatewayState } from '@/store/session' +import { reconcileBusyStatesOnReconnect } from '@/store/session-states' // ── Multi-profile gateway routing ────────────────────────────────────────── // Concurrent sessions across profiles need concurrent sockets: the renderer's @@ -381,6 +382,14 @@ async function reconnectSecondary(entry: Secondary): Promise { try { await openSecondary(entry) entry.reconnectAttempt = 0 + // The re-dialed backend may have respawned and re-minted runtime ids — + // busy flags recorded from THIS socket's pre-drop events would then never + // receive their terminal busy:false, leaving the session's running arc + // armed forever (#53902/#73082 stale-flag half). Scoped: only runtimes + // whose events arrived on this connection are reconciled; live work on + // other sockets is untouched, and a genuinely live turn here re-asserts + // busy on its next event. + reconcileBusyStatesOnReconnect(entry.scope) } catch (error) { // The registry no longer knows this connection (removed while we were // backing off), or Electron's deletion guard reports the profile itself diff --git a/apps/desktop/src/store/session-states-reconnect.test.ts b/apps/desktop/src/store/session-states-reconnect.test.ts new file mode 100644 index 0000000000..8dd8d956ca --- /dev/null +++ b/apps/desktop/src/store/session-states-reconnect.test.ts @@ -0,0 +1,116 @@ +import { registryBackendScopeKey } from '@hermes/shared' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import type { ClientSessionState } from '@/app/types' +import { createClientSessionState } from '@/lib/chat-runtime' + +import { $activeSessionId, $selectedStoredSessionId, $unreadFinishedSessionIds } from './session' +import { + $attentionSessionIds, + $stalledSessionIds, + $workingSessionIds, + clearAllSessionStates, + publishSessionState, + reconcileBusyStatesOnReconnect, + recordSessionEventScope, + SESSION_WATCHDOG_TIMEOUT_MS +} from './session-states' + +function state(over: Partial = {}): ClientSessionState { + return { ...createClientSessionState(null), storedSessionId: 's1', ...over } +} + +// The stale-flag half of #53902/#73082: a backend respawn re-mints runtime +// ids, so a pre-reconnect busy state never receives its terminal busy:false +// and the session's running arc stays armed forever. The reconnect paths call +// reconcileBusyStatesOnReconnect to retire those claims. +describe('reconcileBusyStatesOnReconnect', () => { + beforeEach(() => { + vi.useFakeTimers() + vi.setSystemTime(0) + clearAllSessionStates() + $unreadFinishedSessionIds.set([]) + $selectedStoredSessionId.set(null) + $activeSessionId.set(null) + }) + + afterEach(() => { + vi.runOnlyPendingTimers() + vi.useRealTimers() + clearAllSessionStates() + $unreadFinishedSessionIds.set([]) + $selectedStoredSessionId.set(null) + $activeSessionId.set(null) + }) + + it('clears a stale busy session on primary reconnect', () => { + publishSessionState('rt1', state({ busy: true, storedSessionId: 's1' })) + expect($workingSessionIds.get()).toContain('s1') + + reconcileBusyStatesOnReconnect() + + expect($workingSessionIds.get()).not.toContain('s1') + }) + + it('disarms the stall watchdog with the busy claim', () => { + publishSessionState('rt1', state({ busy: true, storedSessionId: 's1' })) + + reconcileBusyStatesOnReconnect() + + // Without reconcile the watchdog would fire and paint s1 stalled. + vi.advanceTimersByTime(SESSION_WATCHDOG_TIMEOUT_MS + 1000) + expect($stalledSessionIds.get()).not.toContain('s1') + }) + + it('preserves needsInput — a blocking prompt is not a stale flag', () => { + publishSessionState('rt1', state({ busy: true, needsInput: true, storedSessionId: 's1' })) + expect($attentionSessionIds.get()).toContain('s1') + + reconcileBusyStatesOnReconnect() + + expect($workingSessionIds.get()).not.toContain('s1') + expect($attentionSessionIds.get()).toContain('s1') + }) + + it('primary reconcile leaves registry-scoped sessions alone', () => { + const scope = registryBackendScopeKey('connA', 'default') + publishSessionState('rtA', state({ busy: true, storedSessionId: 'sA' })) + recordSessionEventScope({ connectionId: 'connA', profile: 'default', session_id: 'rtA' }) + publishSessionState('rtLocal', state({ busy: true, storedSessionId: 'sLocal' })) + + reconcileBusyStatesOnReconnect() + + expect($workingSessionIds.get()).toContain('sA') + expect($workingSessionIds.get()).not.toContain('sLocal') + + // And the scoped variant clears ONLY its own connection's sessions. + reconcileBusyStatesOnReconnect(scope) + expect($workingSessionIds.get()).not.toContain('sA') + }) + + it('scoped reconcile does not touch other connections or the primary', () => { + publishSessionState('rtA', state({ busy: true, storedSessionId: 'sA' })) + recordSessionEventScope({ connectionId: 'connA', profile: 'default', session_id: 'rtA' }) + publishSessionState('rtB', state({ busy: true, storedSessionId: 'sB' })) + recordSessionEventScope({ connectionId: 'connB', profile: 'default', session_id: 'rtB' }) + publishSessionState('rtLocal', state({ busy: true, storedSessionId: 'sLocal' })) + + reconcileBusyStatesOnReconnect(registryBackendScopeKey('connA', 'default')) + + expect($workingSessionIds.get()).not.toContain('sA') + expect($workingSessionIds.get()).toContain('sB') + expect($workingSessionIds.get()).toContain('sLocal') + }) + + it('a live turn re-asserting busy after reconcile re-arms the arc', () => { + const s = state({ busy: true, storedSessionId: 's1' }) + publishSessionState('rt1', s) + reconcileBusyStatesOnReconnect() + expect($workingSessionIds.get()).not.toContain('s1') + + // The still-alive backend's next event republishes busy under a live id. + publishSessionState('rt2', state({ busy: true, storedSessionId: 's1' })) + + expect($workingSessionIds.get()).toContain('s1') + }) +}) diff --git a/apps/desktop/src/store/session-states.ts b/apps/desktop/src/store/session-states.ts index 7f87673244..b570a5056e 100644 --- a/apps/desktop/src/store/session-states.ts +++ b/apps/desktop/src/store/session-states.ts @@ -373,6 +373,49 @@ export function clearAllSessionStates() { $sessionStates.set({}) } +/** Downgrade cached busy/awaiting states after a gateway reconnect. + * + * A respawned backend re-mints runtime ids (the same fact that drives + * resetTileRuntimeBindings), so a pre-reconnect `busy` can never receive its + * terminal `busy: false` publish — the runtime id it would arrive under is + * dead. Left alone, that state keeps its session in $workingSessionIds + * forever: the sidebar running arc and agents-panel "running" chrome lie for + * hours after the turn actually ended (#53902, #73082 — stale-flag half). + * + * `scope` picks which socket's sessions to reconcile, keyed by the event- + * source scope recorded at fan-in: a SECONDARY (registry) reconnect passes + * its composite scope and touches only runtimes that arrived on that socket; + * the PRIMARY reconnect passes undefined and touches only scope-less + * runtimes (primary/local events record no scope). Neither can clear live + * work riding a different, still-healthy connection. + * + * Direction of failure is deliberate: a turn that IS still live (transient + * socket blip, same backend) re-asserts busy on its next event or inflight + * snapshot within a beat, so at worst its arc blinks once. A dead turn's + * state, by contrast, would never clear on its own. `needsInput` is left + * untouched — a blocking prompt is the one claim the user must explicitly + * answer, and post-reconnect refresh re-asserts or retires it via its own + * path. Transition side-effects run through publishSessionState, so + * watchdogs disarm, stall hints drop, and settle/unread bookkeeping stays + * consistent. */ +export function reconcileBusyStatesOnReconnect(scope?: string) { + const states = $sessionStates.get() + + for (const [runtimeId, state] of Object.entries(states)) { + if (!state || (!state.busy && !state.awaitingResponse)) { + continue + } + + const recorded = sessionScopeByRuntimeId.get(runtimeId) + + if (scope === undefined ? recorded !== undefined : recorded !== scope) { + continue + } + + publishSessionState(runtimeId, { ...state, awaitingResponse: false, busy: false }) + } +} + // Derived per-session status sets — pure projections of `$sessionStates` (which // holds `busy`/`needsInput` per runtime), keeping the data flow one-directional: // gateway event → cache → $sessionStates → computed views.