diff --git a/apps/desktop/src/app/gateway/hooks/use-gateway-boot.test.tsx b/apps/desktop/src/app/gateway/hooks/use-gateway-boot.test.tsx index be1ef9418a..73c79fc88a 100644 --- a/apps/desktop/src/app/gateway/hooks/use-gateway-boot.test.tsx +++ b/apps/desktop/src/app/gateway/hooks/use-gateway-boot.test.tsx @@ -49,6 +49,12 @@ class FakeWebSocket { // errors (a dead remote). Mirrors a VPS going away after the first connect. static mode: 'open' | 'fail' = 'open' static instances: FakeWebSocket[] = [] + // Ping behavior: 'pong' answers with a healthy pong frame; 'silent' swallows + // the request (the half-open-socket simulation — connection looks OPEN but + // every RPC hangs until its per-call timeout); 'method-not-found' answers + // the JSON-RPC error a PRE-ping backend returns (a healthy, version-skewed + // response that must NOT trigger a reconnect). + static pingMode: 'pong' | 'silent' | 'method-not-found' = 'pong' readyState = 0 private listeners: Record> = {} @@ -88,6 +94,35 @@ class FakeWebSocket { this.emit('close', {}) } + send(data: string) { + let frame: { id?: unknown; method?: string } + + try { + frame = JSON.parse(data) as { id?: unknown; method?: string } + } catch { + return + } + + if (frame.method !== 'ping') { + return + } + + if (FakeWebSocket.pingMode === 'pong') { + this.emit('message', { + data: JSON.stringify({ jsonrpc: '2.0', id: frame.id, result: { pong: true } }) + }) + } else if (FakeWebSocket.pingMode === 'method-not-found') { + this.emit('message', { + data: JSON.stringify({ + jsonrpc: '2.0', + id: frame.id, + error: { code: -32601, message: 'Method not found' } + }) + }) + } + // 'silent': swallow — a healthy socket answers, a half-open one never does. + } + private emit(type: string, ev: unknown) { for (const fn of this.listeners[type] ?? []) { fn(ev) @@ -204,6 +239,7 @@ beforeEach(() => { vi.useFakeTimers() FakeWebSocket.mode = 'open' FakeWebSocket.instances = [] + FakeWebSocket.pingMode = 'pong' connectionApplied = null powerResume = null ;(globalThis as { WebSocket: unknown }).WebSocket = FakeWebSocket @@ -761,4 +797,66 @@ describe('useGatewayBoot remote reconnect loop (real hook, fake socket)', () => await advanceBackoff() expect(desktop.getConnection).toHaveBeenCalledTimes(1) }) + + it('wake probe: an open-looking but unresponsive socket is force-closed and reconnected', async () => { + render() + await flushAsync() + expect($gatewayState.get()).toBe('open') + const socketCountBefore = FakeWebSocket.instances.length + + // Half-open socket: connectionState reads 'open' (no close event) but the + // backend never answers — the sleep/wake TCP black hole. + FakeWebSocket.pingMode = 'silent' + + // A wake signal (power resume / network online / window visible) nudges + // reconnectNow. With the socket still reporting open it must PROBE rather + // than skip; the swallowed ping times out and forces the socket down. + act(() => window.dispatchEvent(new Event('online'))) + await act(async () => { + await vi.advanceTimersByTimeAsync(5_100) + }) + // The probe timeout (5s) force-closed the socket → 'closed' → the backoff + // timer schedules a reconnect; let it fire and re-dial. + await advanceBackoff() + + // A fresh socket was dialed. + expect(FakeWebSocket.instances.length).toBeGreaterThan(socketCountBefore) + expect($gatewayState.get()).toBe('open') + }) + + it('wake probe: a healthy socket answers the ping and stays untouched', async () => { + render() + await flushAsync() + expect($gatewayState.get()).toBe('open') + const socketCountBefore = FakeWebSocket.instances.length + + // Default FakeWebSocket behavior: answer pings with a pong frame. + act(() => window.dispatchEvent(new Event('online'))) + await act(async () => { + await vi.advanceTimersByTimeAsync(5_100) + }) + + // Probe succeeded → no forced close, no reconnect, connection untouched. + expect(FakeWebSocket.instances.length).toBe(socketCountBefore) + expect($gatewayState.get()).toBe('open') + }) + + it('wake probe: a pre-ping backend (-32601) is healthy, not reconnected', async () => { + render() + await flushAsync() + expect($gatewayState.get()).toBe('open') + const socketCountBefore = FakeWebSocket.instances.length + + // Version skew: this gateway predates the ping method. The error response + // proves the socket is alive; forcing a reconnect would loop forever. + FakeWebSocket.pingMode = 'method-not-found' + + act(() => window.dispatchEvent(new Event('online'))) + await act(async () => { + await vi.advanceTimersByTimeAsync(5_100) + }) + + expect(FakeWebSocket.instances.length).toBe(socketCountBefore) + expect($gatewayState.get()).toBe('open') + }) }) diff --git a/apps/desktop/src/app/gateway/hooks/use-gateway-boot.ts b/apps/desktop/src/app/gateway/hooks/use-gateway-boot.ts index ae300f36b0..d3dd0195ab 100644 --- a/apps/desktop/src/app/gateway/hooks/use-gateway-boot.ts +++ b/apps/desktop/src/app/gateway/hooks/use-gateway-boot.ts @@ -1,4 +1,4 @@ -import { isGatewayReauthRequired, resolveGatewayWsUrl } from '@hermes/shared' +import { isGatewayReauthRequired, JsonRpcGatewayError, resolveGatewayWsUrl } from '@hermes/shared' import { useEffect, useRef } from 'react' import { shouldApplyPostBootProgressError } from '@/components/boot-failure-reauth' @@ -75,6 +75,13 @@ import { stashGatewaySurvivor, survivorIsStale, takeGatewaySurvivor } from './ga // toast. Chat stays readable/draftable the whole time either way. const RECONNECT_ESCALATE_AFTER_MS = 300_000 +// Bound for the sleep/wake liveness probe (see reconnectNow): long enough to +// ride out a busy-but-healthy backend's scheduling jitter, short enough that a +// half-open socket fails fast instead of hanging the wake path. Independent of +// PROMPT_SUBMIT_REQUEST_TIMEOUT_MS (30 min) — that long timeout is correct for +// an in-flight turn, but must never be what a dead connection burns. +const GATEWAY_LIVENESS_PROBE_TIMEOUT_MS = 5_000 + // Bounded self-heal for a failed REMOTE boot (#82679): when the primary boot // fails on a transient remote fault (dropped SSH/HTTP registered connection, // mint timeout — main tags those `retryable` on the boot progress), the @@ -357,6 +364,30 @@ export function useGatewayBoot({ if (!gatewayOpen()) { await attemptReconnect() + return + } + + // The socket reports open, but sleep/wake (or a silent network drop) + // can leave a half-open TCP connection: no close event fires, so + // connectionState stays 'open' while every RPC hangs until its per-call + // timeout — prompt.submit's is 30 minutes, which reads as "enter does + // nothing until I restart the app". Probe liveness with a short-bounded + // ping; on failure force the socket down so the onState handler above + // schedules a reconnect (and resetTileRuntimeBindings re-resumes tiles), + // instead of letting the user's next submit hang against a dead socket. + try { + await gateway.request('ping', {}, GATEWAY_LIVENESS_PROBE_TIMEOUT_MS) + } catch (probeErr) { + // A version-skewed backend that predates the ping method answers + // -32601 (method not found) — a HEALTHY response, not a dead socket. + // Force-closing on it would spin the reconnect loop forever. Every + // other failure (timeout on a swallowed ping, transport error) means + // the socket is not actually alive and must be rebuilt. + if (probeErr instanceof JsonRpcGatewayError && probeErr.code === -32601) { + return + } + + gateway.close() } } diff --git a/tests/tui_gateway/test_ping_probe.py b/tests/tui_gateway/test_ping_probe.py new file mode 100644 index 0000000000..514d42036e --- /dev/null +++ b/tests/tui_gateway/test_ping_probe.py @@ -0,0 +1,31 @@ +"""Tests for the ``ping`` liveness probe (tui_gateway/server.py). + +The desktop client probes ``ping`` after sleep/wake to distinguish a +half-open TCP socket (connectionState still ``open`` while every RPC hangs) +from a genuinely healthy connection. The contract is minimal on purpose: +answered synchronously on the WS reader thread, no session, no IO, no agent. +""" + +from __future__ import annotations + +import tui_gateway.server as srv + + +def _call(method: str, params: dict) -> dict: + """Invoke a registered RPC method and return its result dict.""" + envelope = srv._methods[method](1, params) + return envelope["result"] + + +def test_ping_registered_and_answers_pong(): + res = _call("ping", {}) + assert res == {"pong": True} + + +def test_ping_ignores_params_and_returns_ok_envelope(): + # The probe must be parameter-agnostic: the client sends {} but a stray + # future caller must not be able to make it fail. + envelope = srv._methods["ping"](7, {"anything": "value"}) + assert envelope["jsonrpc"] == "2.0" + assert envelope["id"] == 7 + assert envelope["result"] == {"pong": True} diff --git a/tui_gateway/server.py b/tui_gateway/server.py index d7db8dc30d..ca9c47f83e 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -15318,6 +15318,21 @@ def _persist_wake_enabled(enabled: bool) -> bool: return False +@method("ping") +def _(rid, params: dict) -> dict: + """Cheapest possible liveness probe for the desktop client. + + Answered synchronously on the WS reader thread, so it works even while + every agent is mid-turn or the GIL is contended — the round-trip only + measures socket health, not backend load. A desktop client uses it after + sleep/wake to distinguish a half-open TCP connection (no close event, so + ``connectionState`` still reads ``open`` while every RPC hangs until its + per-call timeout) from a genuinely healthy socket, and forces a reconnect + in the former case instead of letting the next ``prompt.submit`` hang. + """ + return _ok(rid, {"pong": True}) + + @method("wake.start") def _(rid, params: dict) -> dict: """Arm the wake-word listener for the calling surface ("tui" | "gui").