A respawned backend re-mints runtime ids, so a pre-reconnect busy state never receives its terminal busy:false publish and its session stayed in $workingSessionIds forever - the sidebar running arc and agents-panel 'running' chrome lied for hours after the turn ended (the stale-flag half of #53902/#73082; the CSS cost half landed in #91383). reconcileBusyStatesOnReconnect() downgrades busy/awaitingResponse states through publishSessionState (watchdogs disarm, stall hints drop, settle/ unread bookkeeping stays consistent), scoped by event-source: the primary reconnect touches only scope-less runtimes, a secondary (registry) reconnect touches only its own connection's. needsInput survives - a blocking prompt is the user's to answer. A genuinely live turn re-asserts busy on its next post-reconnect event, so the worst case is one arc blink. Regression tests proven by sabotage run (neutered reconcile -> 5/6 fail).
This commit is contained in:
@@ -52,6 +52,7 @@ import {
|
||||
$attentionSessionIds,
|
||||
$workingSessionIds,
|
||||
liveSessionScopes,
|
||||
reconcileBusyStatesOnReconnect,
|
||||
recordSessionEventScope,
|
||||
resetTileRuntimeBindings
|
||||
} from '@/store/session-states'
|
||||
@@ -252,6 +253,12 @@ export function useGatewayBoot({
|
||||
// A respawned backend re-mints (recycles) runtime ids, so any tile's
|
||||
// bound runtime id is now stale — drop them so each tile re-resumes.
|
||||
resetTileRuntimeBindings()
|
||||
// Same staleness, other half: pre-reconnect busy flags are keyed by
|
||||
// those dead runtime ids and would never receive their terminal
|
||||
// busy:false — clear them or the sidebar running arc lies forever
|
||||
// (#53902/#73082). A genuinely live turn re-asserts busy on its next
|
||||
// post-reconnect event.
|
||||
reconcileBusyStatesOnReconnect()
|
||||
// Resync state that may have moved on the backend while we were asleep.
|
||||
await callbacksRef.current.refreshHermesConfig().catch(() => undefined)
|
||||
await callbacksRef.current.refreshSessions().catch(() => undefined)
|
||||
|
||||
@@ -6,6 +6,7 @@ import { HermesGateway, setApiRequestConnection } from '@/hermes'
|
||||
import { reconnectBackoffDelayMs } from '@/lib/reconnect-backoff'
|
||||
import { markNativeNotifyBaseline } from '@/store/notify-baseline'
|
||||
import { setConnection, setGatewayState } from '@/store/session'
|
||||
import { reconcileBusyStatesOnReconnect } from '@/store/session-states'
|
||||
|
||||
// ── Multi-profile gateway routing ──────────────────────────────────────────
|
||||
// Concurrent sessions across profiles need concurrent sockets: the renderer's
|
||||
@@ -381,6 +382,14 @@ async function reconnectSecondary(entry: Secondary): Promise<void> {
|
||||
try {
|
||||
await openSecondary(entry)
|
||||
entry.reconnectAttempt = 0
|
||||
// The re-dialed backend may have respawned and re-minted runtime ids —
|
||||
// busy flags recorded from THIS socket's pre-drop events would then never
|
||||
// receive their terminal busy:false, leaving the session's running arc
|
||||
// armed forever (#53902/#73082 stale-flag half). Scoped: only runtimes
|
||||
// whose events arrived on this connection are reconciled; live work on
|
||||
// other sockets is untouched, and a genuinely live turn here re-asserts
|
||||
// busy on its next event.
|
||||
reconcileBusyStatesOnReconnect(entry.scope)
|
||||
} catch (error) {
|
||||
// The registry no longer knows this connection (removed while we were
|
||||
// backing off), or Electron's deletion guard reports the profile itself
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import { registryBackendScopeKey } from '@hermes/shared'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import type { ClientSessionState } from '@/app/types'
|
||||
import { createClientSessionState } from '@/lib/chat-runtime'
|
||||
|
||||
import { $activeSessionId, $selectedStoredSessionId, $unreadFinishedSessionIds } from './session'
|
||||
import {
|
||||
$attentionSessionIds,
|
||||
$stalledSessionIds,
|
||||
$workingSessionIds,
|
||||
clearAllSessionStates,
|
||||
publishSessionState,
|
||||
reconcileBusyStatesOnReconnect,
|
||||
recordSessionEventScope,
|
||||
SESSION_WATCHDOG_TIMEOUT_MS
|
||||
} from './session-states'
|
||||
|
||||
function state(over: Partial<ClientSessionState> = {}): ClientSessionState {
|
||||
return { ...createClientSessionState(null), storedSessionId: 's1', ...over }
|
||||
}
|
||||
|
||||
// The stale-flag half of #53902/#73082: a backend respawn re-mints runtime
|
||||
// ids, so a pre-reconnect busy state never receives its terminal busy:false
|
||||
// and the session's running arc stays armed forever. The reconnect paths call
|
||||
// reconcileBusyStatesOnReconnect to retire those claims.
|
||||
describe('reconcileBusyStatesOnReconnect', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(0)
|
||||
clearAllSessionStates()
|
||||
$unreadFinishedSessionIds.set([])
|
||||
$selectedStoredSessionId.set(null)
|
||||
$activeSessionId.set(null)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.runOnlyPendingTimers()
|
||||
vi.useRealTimers()
|
||||
clearAllSessionStates()
|
||||
$unreadFinishedSessionIds.set([])
|
||||
$selectedStoredSessionId.set(null)
|
||||
$activeSessionId.set(null)
|
||||
})
|
||||
|
||||
it('clears a stale busy session on primary reconnect', () => {
|
||||
publishSessionState('rt1', state({ busy: true, storedSessionId: 's1' }))
|
||||
expect($workingSessionIds.get()).toContain('s1')
|
||||
|
||||
reconcileBusyStatesOnReconnect()
|
||||
|
||||
expect($workingSessionIds.get()).not.toContain('s1')
|
||||
})
|
||||
|
||||
it('disarms the stall watchdog with the busy claim', () => {
|
||||
publishSessionState('rt1', state({ busy: true, storedSessionId: 's1' }))
|
||||
|
||||
reconcileBusyStatesOnReconnect()
|
||||
|
||||
// Without reconcile the watchdog would fire and paint s1 stalled.
|
||||
vi.advanceTimersByTime(SESSION_WATCHDOG_TIMEOUT_MS + 1000)
|
||||
expect($stalledSessionIds.get()).not.toContain('s1')
|
||||
})
|
||||
|
||||
it('preserves needsInput — a blocking prompt is not a stale flag', () => {
|
||||
publishSessionState('rt1', state({ busy: true, needsInput: true, storedSessionId: 's1' }))
|
||||
expect($attentionSessionIds.get()).toContain('s1')
|
||||
|
||||
reconcileBusyStatesOnReconnect()
|
||||
|
||||
expect($workingSessionIds.get()).not.toContain('s1')
|
||||
expect($attentionSessionIds.get()).toContain('s1')
|
||||
})
|
||||
|
||||
it('primary reconcile leaves registry-scoped sessions alone', () => {
|
||||
const scope = registryBackendScopeKey('connA', 'default')
|
||||
publishSessionState('rtA', state({ busy: true, storedSessionId: 'sA' }))
|
||||
recordSessionEventScope({ connectionId: 'connA', profile: 'default', session_id: 'rtA' })
|
||||
publishSessionState('rtLocal', state({ busy: true, storedSessionId: 'sLocal' }))
|
||||
|
||||
reconcileBusyStatesOnReconnect()
|
||||
|
||||
expect($workingSessionIds.get()).toContain('sA')
|
||||
expect($workingSessionIds.get()).not.toContain('sLocal')
|
||||
|
||||
// And the scoped variant clears ONLY its own connection's sessions.
|
||||
reconcileBusyStatesOnReconnect(scope)
|
||||
expect($workingSessionIds.get()).not.toContain('sA')
|
||||
})
|
||||
|
||||
it('scoped reconcile does not touch other connections or the primary', () => {
|
||||
publishSessionState('rtA', state({ busy: true, storedSessionId: 'sA' }))
|
||||
recordSessionEventScope({ connectionId: 'connA', profile: 'default', session_id: 'rtA' })
|
||||
publishSessionState('rtB', state({ busy: true, storedSessionId: 'sB' }))
|
||||
recordSessionEventScope({ connectionId: 'connB', profile: 'default', session_id: 'rtB' })
|
||||
publishSessionState('rtLocal', state({ busy: true, storedSessionId: 'sLocal' }))
|
||||
|
||||
reconcileBusyStatesOnReconnect(registryBackendScopeKey('connA', 'default'))
|
||||
|
||||
expect($workingSessionIds.get()).not.toContain('sA')
|
||||
expect($workingSessionIds.get()).toContain('sB')
|
||||
expect($workingSessionIds.get()).toContain('sLocal')
|
||||
})
|
||||
|
||||
it('a live turn re-asserting busy after reconcile re-arms the arc', () => {
|
||||
const s = state({ busy: true, storedSessionId: 's1' })
|
||||
publishSessionState('rt1', s)
|
||||
reconcileBusyStatesOnReconnect()
|
||||
expect($workingSessionIds.get()).not.toContain('s1')
|
||||
|
||||
// The still-alive backend's next event republishes busy under a live id.
|
||||
publishSessionState('rt2', state({ busy: true, storedSessionId: 's1' }))
|
||||
|
||||
expect($workingSessionIds.get()).toContain('s1')
|
||||
})
|
||||
})
|
||||
@@ -373,6 +373,49 @@ export function clearAllSessionStates() {
|
||||
$sessionStates.set({})
|
||||
}
|
||||
|
||||
/** Downgrade cached busy/awaiting states after a gateway reconnect.
|
||||
*
|
||||
* A respawned backend re-mints runtime ids (the same fact that drives
|
||||
* resetTileRuntimeBindings), so a pre-reconnect `busy` can never receive its
|
||||
* terminal `busy: false` publish — the runtime id it would arrive under is
|
||||
* dead. Left alone, that state keeps its session in $workingSessionIds
|
||||
* forever: the sidebar running arc and agents-panel "running" chrome lie for
|
||||
* hours after the turn actually ended (#53902, #73082 — stale-flag half).
|
||||
*
|
||||
* `scope` picks which socket's sessions to reconcile, keyed by the event-
|
||||
* source scope recorded at fan-in: a SECONDARY (registry) reconnect passes
|
||||
* its composite scope and touches only runtimes that arrived on that socket;
|
||||
* the PRIMARY reconnect passes undefined and touches only scope-less
|
||||
* runtimes (primary/local events record no scope). Neither can clear live
|
||||
* work riding a different, still-healthy connection.
|
||||
*
|
||||
* Direction of failure is deliberate: a turn that IS still live (transient
|
||||
* socket blip, same backend) re-asserts busy on its next event or inflight
|
||||
* snapshot within a beat, so at worst its arc blinks once. A dead turn's
|
||||
* state, by contrast, would never clear on its own. `needsInput` is left
|
||||
* untouched — a blocking prompt is the one claim the user must explicitly
|
||||
* answer, and post-reconnect refresh re-asserts or retires it via its own
|
||||
* path. Transition side-effects run through publishSessionState, so
|
||||
* watchdogs disarm, stall hints drop, and settle/unread bookkeeping stays
|
||||
* consistent. */
|
||||
export function reconcileBusyStatesOnReconnect(scope?: string) {
|
||||
const states = $sessionStates.get()
|
||||
|
||||
for (const [runtimeId, state] of Object.entries(states)) {
|
||||
if (!state || (!state.busy && !state.awaitingResponse)) {
|
||||
continue
|
||||
}
|
||||
|
||||
const recorded = sessionScopeByRuntimeId.get(runtimeId)
|
||||
|
||||
if (scope === undefined ? recorded !== undefined : recorded !== scope) {
|
||||
continue
|
||||
}
|
||||
|
||||
publishSessionState(runtimeId, { ...state, awaitingResponse: false, busy: false })
|
||||
}
|
||||
}
|
||||
|
||||
// Derived per-session status sets — pure projections of `$sessionStates` (which
|
||||
// holds `busy`/`needsInput` per runtime), keeping the data flow one-directional:
|
||||
// gateway event → cache → $sessionStates → computed views.
|
||||
|
||||
Reference in New Issue
Block a user