fix(desktop): probe half-open gateway socket on wake and reconnect
macOS sleep/wake (or a silent network drop) can leave the renderer's WebSocket half-open: no close event fires, so connectionState stays 'open' while every RPC hangs until its per-call timeout. prompt.submit's timeout is 30 minutes, so the user's next message reads as "enter does nothing until I restart the app". - Add a minimal ping RPC (tui_gateway/server.py) answered synchronously on the WS reader thread. - On wake signals, reconnectNow now probes the open-looking socket with a 5s-bounded ping and force-closes it on failure, letting the existing reconnect machinery (backoff, tile rebinding, session refresh) take over. A pre-ping backend answering -32601 is treated as healthy. - Tests: half-open socket force-reconnects; healthy socket untouched; method-not-found backend untouched; backend ping envelope contract.
This commit is contained in:
@@ -49,6 +49,12 @@ class FakeWebSocket {
|
||||
// errors (a dead remote). Mirrors a VPS going away after the first connect.
|
||||
static mode: 'open' | 'fail' = 'open'
|
||||
static instances: FakeWebSocket[] = []
|
||||
// Ping behavior: 'pong' answers with a healthy pong frame; 'silent' swallows
|
||||
// the request (the half-open-socket simulation — connection looks OPEN but
|
||||
// every RPC hangs until its per-call timeout); 'method-not-found' answers
|
||||
// the JSON-RPC error a PRE-ping backend returns (a healthy, version-skewed
|
||||
// response that must NOT trigger a reconnect).
|
||||
static pingMode: 'pong' | 'silent' | 'method-not-found' = 'pong'
|
||||
|
||||
readyState = 0
|
||||
private listeners: Record<string, Set<Listener>> = {}
|
||||
@@ -88,6 +94,35 @@ class FakeWebSocket {
|
||||
this.emit('close', {})
|
||||
}
|
||||
|
||||
send(data: string) {
|
||||
let frame: { id?: unknown; method?: string }
|
||||
|
||||
try {
|
||||
frame = JSON.parse(data) as { id?: unknown; method?: string }
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
|
||||
if (frame.method !== 'ping') {
|
||||
return
|
||||
}
|
||||
|
||||
if (FakeWebSocket.pingMode === 'pong') {
|
||||
this.emit('message', {
|
||||
data: JSON.stringify({ jsonrpc: '2.0', id: frame.id, result: { pong: true } })
|
||||
})
|
||||
} else if (FakeWebSocket.pingMode === 'method-not-found') {
|
||||
this.emit('message', {
|
||||
data: JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
id: frame.id,
|
||||
error: { code: -32601, message: 'Method not found' }
|
||||
})
|
||||
})
|
||||
}
|
||||
// 'silent': swallow — a healthy socket answers, a half-open one never does.
|
||||
}
|
||||
|
||||
private emit(type: string, ev: unknown) {
|
||||
for (const fn of this.listeners[type] ?? []) {
|
||||
fn(ev)
|
||||
@@ -204,6 +239,7 @@ beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
FakeWebSocket.mode = 'open'
|
||||
FakeWebSocket.instances = []
|
||||
FakeWebSocket.pingMode = 'pong'
|
||||
connectionApplied = null
|
||||
powerResume = null
|
||||
;(globalThis as { WebSocket: unknown }).WebSocket = FakeWebSocket
|
||||
@@ -761,4 +797,66 @@ describe('useGatewayBoot remote reconnect loop (real hook, fake socket)', () =>
|
||||
await advanceBackoff()
|
||||
expect(desktop.getConnection).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('wake probe: an open-looking but unresponsive socket is force-closed and reconnected', async () => {
|
||||
render(<Harness />)
|
||||
await flushAsync()
|
||||
expect($gatewayState.get()).toBe('open')
|
||||
const socketCountBefore = FakeWebSocket.instances.length
|
||||
|
||||
// Half-open socket: connectionState reads 'open' (no close event) but the
|
||||
// backend never answers — the sleep/wake TCP black hole.
|
||||
FakeWebSocket.pingMode = 'silent'
|
||||
|
||||
// A wake signal (power resume / network online / window visible) nudges
|
||||
// reconnectNow. With the socket still reporting open it must PROBE rather
|
||||
// than skip; the swallowed ping times out and forces the socket down.
|
||||
act(() => window.dispatchEvent(new Event('online')))
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(5_100)
|
||||
})
|
||||
// The probe timeout (5s) force-closed the socket → 'closed' → the backoff
|
||||
// timer schedules a reconnect; let it fire and re-dial.
|
||||
await advanceBackoff()
|
||||
|
||||
// A fresh socket was dialed.
|
||||
expect(FakeWebSocket.instances.length).toBeGreaterThan(socketCountBefore)
|
||||
expect($gatewayState.get()).toBe('open')
|
||||
})
|
||||
|
||||
it('wake probe: a healthy socket answers the ping and stays untouched', async () => {
|
||||
render(<Harness />)
|
||||
await flushAsync()
|
||||
expect($gatewayState.get()).toBe('open')
|
||||
const socketCountBefore = FakeWebSocket.instances.length
|
||||
|
||||
// Default FakeWebSocket behavior: answer pings with a pong frame.
|
||||
act(() => window.dispatchEvent(new Event('online')))
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(5_100)
|
||||
})
|
||||
|
||||
// Probe succeeded → no forced close, no reconnect, connection untouched.
|
||||
expect(FakeWebSocket.instances.length).toBe(socketCountBefore)
|
||||
expect($gatewayState.get()).toBe('open')
|
||||
})
|
||||
|
||||
it('wake probe: a pre-ping backend (-32601) is healthy, not reconnected', async () => {
|
||||
render(<Harness />)
|
||||
await flushAsync()
|
||||
expect($gatewayState.get()).toBe('open')
|
||||
const socketCountBefore = FakeWebSocket.instances.length
|
||||
|
||||
// Version skew: this gateway predates the ping method. The error response
|
||||
// proves the socket is alive; forcing a reconnect would loop forever.
|
||||
FakeWebSocket.pingMode = 'method-not-found'
|
||||
|
||||
act(() => window.dispatchEvent(new Event('online')))
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(5_100)
|
||||
})
|
||||
|
||||
expect(FakeWebSocket.instances.length).toBe(socketCountBefore)
|
||||
expect($gatewayState.get()).toBe('open')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { isGatewayReauthRequired, resolveGatewayWsUrl } from '@hermes/shared'
|
||||
import { isGatewayReauthRequired, JsonRpcGatewayError, resolveGatewayWsUrl } from '@hermes/shared'
|
||||
import { useEffect, useRef } from 'react'
|
||||
|
||||
import { shouldApplyPostBootProgressError } from '@/components/boot-failure-reauth'
|
||||
@@ -75,6 +75,13 @@ import { stashGatewaySurvivor, survivorIsStale, takeGatewaySurvivor } from './ga
|
||||
// toast. Chat stays readable/draftable the whole time either way.
|
||||
const RECONNECT_ESCALATE_AFTER_MS = 300_000
|
||||
|
||||
// Bound for the sleep/wake liveness probe (see reconnectNow): long enough to
|
||||
// ride out a busy-but-healthy backend's scheduling jitter, short enough that a
|
||||
// half-open socket fails fast instead of hanging the wake path. Independent of
|
||||
// PROMPT_SUBMIT_REQUEST_TIMEOUT_MS (30 min) — that long timeout is correct for
|
||||
// an in-flight turn, but must never be what a dead connection burns.
|
||||
const GATEWAY_LIVENESS_PROBE_TIMEOUT_MS = 5_000
|
||||
|
||||
// Bounded self-heal for a failed REMOTE boot (#82679): when the primary boot
|
||||
// fails on a transient remote fault (dropped SSH/HTTP registered connection,
|
||||
// mint timeout — main tags those `retryable` on the boot progress), the
|
||||
@@ -357,6 +364,30 @@ export function useGatewayBoot({
|
||||
|
||||
if (!gatewayOpen()) {
|
||||
await attemptReconnect()
|
||||
return
|
||||
}
|
||||
|
||||
// The socket reports open, but sleep/wake (or a silent network drop)
|
||||
// can leave a half-open TCP connection: no close event fires, so
|
||||
// connectionState stays 'open' while every RPC hangs until its per-call
|
||||
// timeout — prompt.submit's is 30 minutes, which reads as "enter does
|
||||
// nothing until I restart the app". Probe liveness with a short-bounded
|
||||
// ping; on failure force the socket down so the onState handler above
|
||||
// schedules a reconnect (and resetTileRuntimeBindings re-resumes tiles),
|
||||
// instead of letting the user's next submit hang against a dead socket.
|
||||
try {
|
||||
await gateway.request('ping', {}, GATEWAY_LIVENESS_PROBE_TIMEOUT_MS)
|
||||
} catch (probeErr) {
|
||||
// A version-skewed backend that predates the ping method answers
|
||||
// -32601 (method not found) — a HEALTHY response, not a dead socket.
|
||||
// Force-closing on it would spin the reconnect loop forever. Every
|
||||
// other failure (timeout on a swallowed ping, transport error) means
|
||||
// the socket is not actually alive and must be rebuilt.
|
||||
if (probeErr instanceof JsonRpcGatewayError && probeErr.code === -32601) {
|
||||
return
|
||||
}
|
||||
|
||||
gateway.close()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Tests for the ``ping`` liveness probe (tui_gateway/server.py).
|
||||
|
||||
The desktop client probes ``ping`` after sleep/wake to distinguish a
|
||||
half-open TCP socket (connectionState still ``open`` while every RPC hangs)
|
||||
from a genuinely healthy connection. The contract is minimal on purpose:
|
||||
answered synchronously on the WS reader thread, no session, no IO, no agent.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tui_gateway.server as srv
|
||||
|
||||
|
||||
def _call(method: str, params: dict) -> dict:
|
||||
"""Invoke a registered RPC method and return its result dict."""
|
||||
envelope = srv._methods[method](1, params)
|
||||
return envelope["result"]
|
||||
|
||||
|
||||
def test_ping_registered_and_answers_pong():
|
||||
res = _call("ping", {})
|
||||
assert res == {"pong": True}
|
||||
|
||||
|
||||
def test_ping_ignores_params_and_returns_ok_envelope():
|
||||
# The probe must be parameter-agnostic: the client sends {} but a stray
|
||||
# future caller must not be able to make it fail.
|
||||
envelope = srv._methods["ping"](7, {"anything": "value"})
|
||||
assert envelope["jsonrpc"] == "2.0"
|
||||
assert envelope["id"] == 7
|
||||
assert envelope["result"] == {"pong": True}
|
||||
@@ -15318,6 +15318,21 @@ def _persist_wake_enabled(enabled: bool) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@method("ping")
|
||||
def _(rid, params: dict) -> dict:
|
||||
"""Cheapest possible liveness probe for the desktop client.
|
||||
|
||||
Answered synchronously on the WS reader thread, so it works even while
|
||||
every agent is mid-turn or the GIL is contended — the round-trip only
|
||||
measures socket health, not backend load. A desktop client uses it after
|
||||
sleep/wake to distinguish a half-open TCP connection (no close event, so
|
||||
``connectionState`` still reads ``open`` while every RPC hangs until its
|
||||
per-call timeout) from a genuinely healthy socket, and forces a reconnect
|
||||
in the former case instead of letting the next ``prompt.submit`` hang.
|
||||
"""
|
||||
return _ok(rid, {"pong": True})
|
||||
|
||||
|
||||
@method("wake.start")
|
||||
def _(rid, params: dict) -> dict:
|
||||
"""Arm the wake-word listener for the calling surface ("tui" | "gui").
|
||||
|
||||
Reference in New Issue
Block a user