fix(desktop): probe half-open gateway socket on wake and reconnect

macOS sleep/wake (or a silent network drop) can leave the renderer's
WebSocket half-open: no close event fires, so connectionState stays
'open' while every RPC hangs until its per-call timeout. prompt.submit's
timeout is 30 minutes, so the user's next message reads as "enter does
nothing until I restart the app".

- Add a minimal ping RPC (tui_gateway/server.py) answered synchronously
  on the WS reader thread.
- On wake signals, reconnectNow now probes the open-looking socket with a
  5s-bounded ping and force-closes it on failure, letting the existing
  reconnect machinery (backoff, tile rebinding, session refresh) take
  over. A pre-ping backend answering -32601 is treated as healthy.
- Tests: half-open socket force-reconnects; healthy socket untouched;
  method-not-found backend untouched; backend ping envelope contract.
This commit is contained in:
Owenz-creator
2026-08-20 19:03:46 +08:00
committed by Teknium
parent 29c5a12e04
commit cdd37035d9
4 changed files with 176 additions and 1 deletions
@@ -49,6 +49,12 @@ class FakeWebSocket {
// errors (a dead remote). Mirrors a VPS going away after the first connect.
static mode: 'open' | 'fail' = 'open'
static instances: FakeWebSocket[] = []
// Ping behavior: 'pong' answers with a healthy pong frame; 'silent' swallows
// the request (the half-open-socket simulation — connection looks OPEN but
// every RPC hangs until its per-call timeout); 'method-not-found' answers
// the JSON-RPC error a PRE-ping backend returns (a healthy, version-skewed
// response that must NOT trigger a reconnect).
static pingMode: 'pong' | 'silent' | 'method-not-found' = 'pong'
readyState = 0
private listeners: Record<string, Set<Listener>> = {}
@@ -88,6 +94,35 @@ class FakeWebSocket {
this.emit('close', {})
}
send(data: string) {
let frame: { id?: unknown; method?: string }
try {
frame = JSON.parse(data) as { id?: unknown; method?: string }
} catch {
return
}
if (frame.method !== 'ping') {
return
}
if (FakeWebSocket.pingMode === 'pong') {
this.emit('message', {
data: JSON.stringify({ jsonrpc: '2.0', id: frame.id, result: { pong: true } })
})
} else if (FakeWebSocket.pingMode === 'method-not-found') {
this.emit('message', {
data: JSON.stringify({
jsonrpc: '2.0',
id: frame.id,
error: { code: -32601, message: 'Method not found' }
})
})
}
// 'silent': swallow — a healthy socket answers, a half-open one never does.
}
private emit(type: string, ev: unknown) {
for (const fn of this.listeners[type] ?? []) {
fn(ev)
@@ -204,6 +239,7 @@ beforeEach(() => {
vi.useFakeTimers()
FakeWebSocket.mode = 'open'
FakeWebSocket.instances = []
FakeWebSocket.pingMode = 'pong'
connectionApplied = null
powerResume = null
;(globalThis as { WebSocket: unknown }).WebSocket = FakeWebSocket
@@ -761,4 +797,66 @@ describe('useGatewayBoot remote reconnect loop (real hook, fake socket)', () =>
await advanceBackoff()
expect(desktop.getConnection).toHaveBeenCalledTimes(1)
})
it('wake probe: an open-looking but unresponsive socket is force-closed and reconnected', async () => {
render(<Harness />)
await flushAsync()
expect($gatewayState.get()).toBe('open')
const socketCountBefore = FakeWebSocket.instances.length
// Half-open socket: connectionState reads 'open' (no close event) but the
// backend never answers — the sleep/wake TCP black hole.
FakeWebSocket.pingMode = 'silent'
// A wake signal (power resume / network online / window visible) nudges
// reconnectNow. With the socket still reporting open it must PROBE rather
// than skip; the swallowed ping times out and forces the socket down.
act(() => window.dispatchEvent(new Event('online')))
await act(async () => {
await vi.advanceTimersByTimeAsync(5_100)
})
// The probe timeout (5s) force-closed the socket → 'closed' → the backoff
// timer schedules a reconnect; let it fire and re-dial.
await advanceBackoff()
// A fresh socket was dialed.
expect(FakeWebSocket.instances.length).toBeGreaterThan(socketCountBefore)
expect($gatewayState.get()).toBe('open')
})
it('wake probe: a healthy socket answers the ping and stays untouched', async () => {
render(<Harness />)
await flushAsync()
expect($gatewayState.get()).toBe('open')
const socketCountBefore = FakeWebSocket.instances.length
// Default FakeWebSocket behavior: answer pings with a pong frame.
act(() => window.dispatchEvent(new Event('online')))
await act(async () => {
await vi.advanceTimersByTimeAsync(5_100)
})
// Probe succeeded → no forced close, no reconnect, connection untouched.
expect(FakeWebSocket.instances.length).toBe(socketCountBefore)
expect($gatewayState.get()).toBe('open')
})
it('wake probe: a pre-ping backend (-32601) is healthy, not reconnected', async () => {
render(<Harness />)
await flushAsync()
expect($gatewayState.get()).toBe('open')
const socketCountBefore = FakeWebSocket.instances.length
// Version skew: this gateway predates the ping method. The error response
// proves the socket is alive; forcing a reconnect would loop forever.
FakeWebSocket.pingMode = 'method-not-found'
act(() => window.dispatchEvent(new Event('online')))
await act(async () => {
await vi.advanceTimersByTimeAsync(5_100)
})
expect(FakeWebSocket.instances.length).toBe(socketCountBefore)
expect($gatewayState.get()).toBe('open')
})
})
@@ -1,4 +1,4 @@
import { isGatewayReauthRequired, resolveGatewayWsUrl } from '@hermes/shared'
import { isGatewayReauthRequired, JsonRpcGatewayError, resolveGatewayWsUrl } from '@hermes/shared'
import { useEffect, useRef } from 'react'
import { shouldApplyPostBootProgressError } from '@/components/boot-failure-reauth'
@@ -75,6 +75,13 @@ import { stashGatewaySurvivor, survivorIsStale, takeGatewaySurvivor } from './ga
// toast. Chat stays readable/draftable the whole time either way.
const RECONNECT_ESCALATE_AFTER_MS = 300_000
// Bound for the sleep/wake liveness probe (see reconnectNow): long enough to
// ride out a busy-but-healthy backend's scheduling jitter, short enough that a
// half-open socket fails fast instead of hanging the wake path. Independent of
// PROMPT_SUBMIT_REQUEST_TIMEOUT_MS (30 min) — that long timeout is correct for
// an in-flight turn, but must never be what a dead connection burns.
const GATEWAY_LIVENESS_PROBE_TIMEOUT_MS = 5_000
// Bounded self-heal for a failed REMOTE boot (#82679): when the primary boot
// fails on a transient remote fault (dropped SSH/HTTP registered connection,
// mint timeout — main tags those `retryable` on the boot progress), the
@@ -357,6 +364,30 @@ export function useGatewayBoot({
if (!gatewayOpen()) {
await attemptReconnect()
return
}
// The socket reports open, but sleep/wake (or a silent network drop)
// can leave a half-open TCP connection: no close event fires, so
// connectionState stays 'open' while every RPC hangs until its per-call
// timeout — prompt.submit's is 30 minutes, which reads as "enter does
// nothing until I restart the app". Probe liveness with a short-bounded
// ping; on failure force the socket down so the onState handler above
// schedules a reconnect (and resetTileRuntimeBindings re-resumes tiles),
// instead of letting the user's next submit hang against a dead socket.
try {
await gateway.request('ping', {}, GATEWAY_LIVENESS_PROBE_TIMEOUT_MS)
} catch (probeErr) {
// A version-skewed backend that predates the ping method answers
// -32601 (method not found) — a HEALTHY response, not a dead socket.
// Force-closing on it would spin the reconnect loop forever. Every
// other failure (timeout on a swallowed ping, transport error) means
// the socket is not actually alive and must be rebuilt.
if (probeErr instanceof JsonRpcGatewayError && probeErr.code === -32601) {
return
}
gateway.close()
}
}
+31
View File
@@ -0,0 +1,31 @@
"""Tests for the ``ping`` liveness probe (tui_gateway/server.py).
The desktop client probes ``ping`` after sleep/wake to distinguish a
half-open TCP socket (connectionState still ``open`` while every RPC hangs)
from a genuinely healthy connection. The contract is minimal on purpose:
answered synchronously on the WS reader thread, no session, no IO, no agent.
"""
from __future__ import annotations
import tui_gateway.server as srv
def _call(method: str, params: dict) -> dict:
"""Invoke a registered RPC method and return its result dict."""
envelope = srv._methods[method](1, params)
return envelope["result"]
def test_ping_registered_and_answers_pong():
res = _call("ping", {})
assert res == {"pong": True}
def test_ping_ignores_params_and_returns_ok_envelope():
# The probe must be parameter-agnostic: the client sends {} but a stray
# future caller must not be able to make it fail.
envelope = srv._methods["ping"](7, {"anything": "value"})
assert envelope["jsonrpc"] == "2.0"
assert envelope["id"] == 7
assert envelope["result"] == {"pong": True}
+15
View File
@@ -15318,6 +15318,21 @@ def _persist_wake_enabled(enabled: bool) -> bool:
return False
@method("ping")
def _(rid, params: dict) -> dict:
"""Cheapest possible liveness probe for the desktop client.
Answered synchronously on the WS reader thread, so it works even while
every agent is mid-turn or the GIL is contended — the round-trip only
measures socket health, not backend load. A desktop client uses it after
sleep/wake to distinguish a half-open TCP connection (no close event, so
``connectionState`` still reads ``open`` while every RPC hangs until its
per-call timeout) from a genuinely healthy socket, and forces a reconnect
in the former case instead of letting the next ``prompt.submit`` hang.
"""
return _ok(rid, {"pong": True})
@method("wake.start")
def _(rid, params: dict) -> dict:
"""Arm the wake-word listener for the calling surface ("tui" | "gui").