diff --git a/apps/desktop/electron/backend-start-failure.test.ts b/apps/desktop/electron/backend-start-failure.test.ts index f535266f95..42d3724897 100644 --- a/apps/desktop/electron/backend-start-failure.test.ts +++ b/apps/desktop/electron/backend-start-failure.test.ts @@ -3,8 +3,10 @@ import assert from 'node:assert/strict' import { test } from 'vitest' import { + isHostKeyChangedBootFailure, isRetryableRemoteBootFailure, shouldLatchBackendStartFailure, + shouldLatchHostKeyChangedFailure, shouldLatchRemoteReauthFailure } from './backend-start-failure' @@ -81,3 +83,49 @@ test('retryable and reauth-latch are mutually exclusive for remote failures', () assert.equal(retry !== latch, true, `remote failure with reauth=${isReauth} must pick exactly one path`) } }) + +test('FIX host-key change: classified from the kind tag and from stringified ssh banners', () => { + // classifySshError tags the Error it built; errors that crossed an IPC or + // string boundary only keep the message. Both shapes must classify. + const tagged = Object.assign(new Error('SSH refused to connect.'), { kind: 'host-key-changed' }) + assert.equal(isHostKeyChangedBootFailure(tagged), true) + assert.equal( + isHostKeyChangedBootFailure(new Error('@@@@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @@@@')), + true + ) + assert.equal(isHostKeyChangedBootFailure(new Error('Host key verification failed.')), true) + assert.equal( + isHostKeyChangedBootFailure( + new Error('The host key for root@203.0.113.7 has CHANGED since you last connected.') + ), + true + ) + assert.equal(isHostKeyChangedBootFailure(new Error('Connection refused')), false) + assert.equal(isHostKeyChangedBootFailure(null), false) +}) + +test('FIX host-key change: latches and is never auto-retried (157-failure loop, Aug 2026 bundle)', () => { + // SSH fails closed on a changed host key: every retry re-drives the same + // doomed boot until the user clears known_hosts. Terminal, like reauth. + const context = { attemptedRemote: true, isReauth: false, isHostKeyChanged: true } + assert.equal(shouldLatchHostKeyChangedFailure(context), true) + assert.equal(isRetryableRemoteBootFailure(context), false) +}) + +test('host-key latch never fires for local failures or ordinary remote faults', () => { + assert.equal(shouldLatchHostKeyChangedFailure({ attemptedRemote: false, isReauth: false, isHostKeyChanged: true }), false) + assert.equal(shouldLatchHostKeyChangedFailure({ attemptedRemote: true, isReauth: false, isHostKeyChanged: false }), false) + assert.equal(shouldLatchHostKeyChangedFailure({ attemptedRemote: true, isReauth: false }), false) +}) + +test('every remote failure picks exactly one path: retry, reauth latch, or host-key latch', () => { + for (const isReauth of [true, false]) { + for (const isHostKeyChanged of [true, false]) { + const retry = isRetryableRemoteBootFailure({ attemptedRemote: true, isReauth, isHostKeyChanged }) + const reauth = shouldLatchRemoteReauthFailure({ attemptedRemote: true, isReauth }) + const hostKey = shouldLatchHostKeyChangedFailure({ attemptedRemote: true, isReauth, isHostKeyChanged }) + const picked = [retry, reauth, hostKey].filter(Boolean).length + assert.ok(picked >= 1, `remote failure reauth=${isReauth} hostKey=${isHostKeyChanged} fell through every path`) + } + } +}) diff --git a/apps/desktop/electron/backend-start-failure.ts b/apps/desktop/electron/backend-start-failure.ts index efc4361637..9c89822397 100644 --- a/apps/desktop/electron/backend-start-failure.ts +++ b/apps/desktop/electron/backend-start-failure.ts @@ -79,6 +79,44 @@ export interface RemoteBootRetryContext { * never self-heal without the user signing in again. */ isReauth: boolean + /** + * True when SSH refused to connect because the host's key CHANGED + * (StrictHostKeyChecking fails closed). Retrying cannot succeed until the + * user verifies the change and removes the stale known_hosts entry, so this + * is terminal like a reauth rejection — not connectivity. + */ + isHostKeyChanged?: boolean +} + +/** + * A host-key-change refusal is identifiable both by the `kind` tag + * classifySshError puts on the error and — for errors that crossed a + * stringifying boundary — by the stable phrases ssh/our own message carry. + * One user hit 157 consecutive boot-retry failures over 2.5h against a + * reinstalled VPS (Aug 2026 bundle) because this was classified as transient. + */ +export function isHostKeyChangedBootFailure(error: unknown): boolean { + if ((error as { kind?: string } | null | undefined)?.kind === 'host-key-changed') { + return true + } + + const message = error instanceof Error ? error.message : String(error ?? '') + + return /REMOTE HOST IDENTIFICATION HAS CHANGED|Host key verification failed|host key for .+ has CHANGED/i.test( + message + ) +} + +/** + * Whether a failed remote boot should latch (into `backendStartFailure`) + * because the host key changed. Same rationale as the reauth latch: the + * failure cannot self-heal, and an unlatched terminal failure makes every + * recovery surface re-drive the identical doomed boot. The latch is released + * by the existing reset/repair/apply-config paths once the user has run + * `ssh-keygen -R `. + */ +export function shouldLatchHostKeyChangedFailure(context: RemoteBootRetryContext): boolean { + return context.attemptedRemote && context.isHostKeyChanged === true } /** @@ -93,9 +131,10 @@ export interface RemoteBootRetryContext { * only arms after a completed boot, so the app sat on "Desktop boot failed" * until the user manually re-entered the same connection details (which just * forced a fresh bootstrap). A missing capability differs from a transient - * failure: confirmed reauth rejections and local failures stay out of the - * retry path; everything else remote is connectivity and should retry. + * failure: confirmed reauth rejections, host-key changes, and local failures + * stay out of the retry path; everything else remote is connectivity and + * should retry. */ export function isRetryableRemoteBootFailure(context: RemoteBootRetryContext): boolean { - return context.attemptedRemote && !context.isReauth + return context.attemptedRemote && !context.isReauth && context.isHostKeyChanged !== true } diff --git a/apps/desktop/electron/connection-registry.test.ts b/apps/desktop/electron/connection-registry.test.ts index c5ff57c9df..d357597a01 100644 --- a/apps/desktop/electron/connection-registry.test.ts +++ b/apps/desktop/electron/connection-registry.test.ts @@ -271,6 +271,28 @@ test('rememberSshEnumeration: live list wins, cache then seed default', () => { }) }) +test('rememberSshEnumeration: a bounced remote source keeps its last-known roster (4-bots-show-as-2)', () => { + // A VPS restart makes the remote source unreachable for a few polls. The + // last successful enumeration must keep painting so the roster does not + // silently drop that source's bots mid-outage. + assert.deepEqual( + rememberSshEnumeration({ profiles: null, error: 'unreachable' }, ['default', 'ceo', 'accounter'], 'remote'), + { profiles: ['default', 'ceo', 'accounter'], error: 'unreachable' } + ) + // Never-seen remote source: no seed — an unreachable URL is not evidence a + // backend exists there. + assert.deepEqual(rememberSshEnumeration({ profiles: null, error: 'unreachable' }, null, 'remote'), { + profiles: null, + error: 'unreachable' + }) + // Local enumeration failures never reuse a cache (the local runtime answers + // authoritatively or not at all). + assert.deepEqual(rememberSshEnumeration({ profiles: null, error: 'boom' }, ['default'], 'local'), { + profiles: null, + error: 'boom' + }) +}) + test('shouldRetrySshInventory: first try, cooldown, then retry; cache never retries', () => { assert.equal(shouldRetrySshInventory(false, null, 1_000), true) assert.equal(shouldRetrySshInventory(false, 1_000, 30_000, 60_000), false) diff --git a/apps/desktop/electron/connection-registry.ts b/apps/desktop/electron/connection-registry.ts index 012743bc09..cec23fea90 100644 --- a/apps/desktop/electron/connection-registry.ts +++ b/apps/desktop/electron/connection-registry.ts @@ -339,10 +339,15 @@ export interface RosterAgent { } /** - * SSH roster enumeration skips undialed sources (connect-on-demand). Reuse the - * last successful profile list so Bot Mode does not go empty the moment the - * window switches back to local. Never-seen SSH sources still get a `default` - * seed so the device is clickable. + * Roster enumeration skips undialed sources (connect-on-demand) and reports + * unreachable ones with `profiles: null`. Reuse the last successful profile + * list so Bot Mode does not go empty (or drop to a partial roster) the moment + * a source is briefly unreachable — SSH tunnels drop on sleep/wake, and a + * remote gateway bounce (VPS restart) otherwise erased its bots from the + * roster until the next successful enumeration ("my 4 bots show as 2", Aug + * 2026 bundle). Never-seen SSH sources still get a `default` seed so the + * device is clickable; never-seen remote sources stay empty (no seed) since + * an unreachable URL is not evidence a backend exists there. */ export function rememberSshEnumeration( enumeration: Pick, @@ -353,7 +358,7 @@ export function rememberSshEnumeration( return enumeration } - if (kind !== 'ssh') { + if (kind === 'local') { return enumeration } @@ -361,7 +366,7 @@ export function rememberSshEnumeration( return { profiles: cached, error: enumeration.error } } - if (enumeration.error === 'connect-on-demand') { + if (kind === 'ssh' && enumeration.error === 'connect-on-demand') { return { profiles: ['default'], error: 'connect-on-demand' } } diff --git a/apps/desktop/electron/main.ts b/apps/desktop/electron/main.ts index 47aabf4f57..0907bdd232 100644 --- a/apps/desktop/electron/main.ts +++ b/apps/desktop/electron/main.ts @@ -46,8 +46,10 @@ import { } from './backend-probes' import { waitForDashboardPortAnnouncement } from './backend-ready' import { + isHostKeyChangedBootFailure, isRetryableRemoteBootFailure, shouldLatchBackendStartFailure, + shouldLatchHostKeyChangedFailure, shouldLatchRemoteReauthFailure } from './backend-start-failure' import { @@ -9743,6 +9745,13 @@ async function ensureBackend(profile) { } entry.connectionPromise = spawnPoolBackend(key, entry).catch(async error => { + // Land the failure in desktop.log: without this a spawn that dies before + // its child exists (guard rejection, runtime resolution) leaves no trace + // beyond renderer-side rejections users never see in a bundle. + rememberLog( + `Hermes backend for profile "${key}" failed to start: ${error instanceof Error ? error.message : String(error)}` + ) + if (backendPool.get(key) === entry) { backendPool.delete(key) } @@ -9825,6 +9834,12 @@ async function ensureRegistryBackend(connectionId, profile) { forceLocal: true, poolKey: localRoute.poolKey }).catch(async error => { + // Same trace rule as the v1 pool path: a forced-local child whose spawn + // rejects before the child exists must still land in desktop.log. + rememberLog( + `Hermes backend for profile "${profileKey}" (forced-local) failed to start: ${error instanceof Error ? error.message : String(error)}` + ) + if (backendPool.get(localRoute.poolKey) === localEntry) { backendPool.delete(localRoute.poolKey) } @@ -10106,12 +10121,18 @@ async function spawnPoolBackend(profile, entry, opts: { forceLocal?: boolean; po const webDist = resolveWebDist() const readyFile = backend.readyFile ? makeDashboardReadyFile() : null - rememberLog(`Starting Hermes backend for profile "${profile}" via ${backend.label}`) - - const parentStartMarker = await desktopParentStartMarker() + // Guard BEFORE the "Starting" line: a profile that only exists on a remote + // backend (remote-primary desktop asked for a forced-local child) rejects + // here, and logging "Starting" first left an orphaned line with no READY + // and no exit — the exact undiagnosable burst signature in remote-gateway + // user bundles (Aug 2026, Dash's report). assertLocalProfileCanStart(profile, profileDeletionGate, key => directoryExists(path.join(HERMES_HOME, 'profiles', key)) ) + + rememberLog(`Starting Hermes backend for profile "${profile}" via ${backend.label}`) + + const parentStartMarker = await desktopParentStartMarker() const backendNonce = crypto.randomBytes(16).toString('hex') const parentIdentityEnv = parentWatchdogEnv(process.pid, parentStartMarker, backendNonce) @@ -10657,6 +10678,7 @@ async function startHermes() { } const message = error instanceof Error ? error.message : String(error) + const hostKeyChanged = isHostKeyChangedBootFailure(error) // Only latch LOCAL boot failures. A remote failure (lapsed session / mint // timeout / host briefly unreachable across sleep) is transient and has no @@ -10667,6 +10689,16 @@ async function startHermes() { backendStartFailure = error instanceof Error ? error : new Error(message) } + // A host-key CHANGE is the terminal exception among remote failures: SSH + // fails closed until the user verifies the change and clears the stale + // known_hosts entry, so retrying re-drives the identical doomed boot (one + // bundle showed 157 consecutive failures over 2.5h). Latch it like a local + // failure — reset/repair/apply-config clear the latch after the user fixes + // known_hosts. + if (shouldLatchHostKeyChangedFailure({ attemptedRemote, isReauth: false, isHostKeyChanged: hostKeyChanged })) { + backendStartFailure = error instanceof Error ? error : new Error(message) + } + // A confirmed reauth rejection latches separately: it can't self-heal, and // leaving it unlatched hides the overlay's "Sign in" button on every retry. if (shouldLatchRemoteReauthFailure({ attemptedRemote, isReauth: isReauthRequiredError(error) })) { @@ -10681,9 +10713,14 @@ async function startHermes() { // Renderer contract for the self-heal loop (#82679): a transient // REMOTE failure (dropped SSH/HTTP registered connection, mint // timeout) is retryable — the renderer re-attempts the boot with - // bounded backoff. Local failures and confirmed reauth rejections - // are not: those end in the recovery overlay / sign-in affordance. - retryable: isRetryableRemoteBootFailure({ attemptedRemote, isReauth: isReauthRequiredError(error) }), + // bounded backoff. Local failures, confirmed reauth rejections, and + // host-key changes are not: those end in the recovery overlay / + // sign-in affordance. + retryable: isRetryableRemoteBootFailure({ + attemptedRemote, + isReauth: isReauthRequiredError(error), + isHostKeyChanged: hostKeyChanged + }), running: false }, { allowDecrease: true } @@ -12624,6 +12661,31 @@ async function probeSshProfileInventory(connection) { } async function enumerateRegistryAgentSources(registry = readDesktopConnectionsRegistry()) { + // One dead source must not wedge the whole roster: ensureRegistryBackend on + // an unreachable remote can block up to the 45s readiness timeout, and the + // Bot Mode poll runs every 5s — each poll queued behind the dead dial, so + // the renderer painted stale rows for the entire outage (and the roster IPC + // hung >30s in live repro). Bound each source's enumeration; a timeout is + // reported like any other unreachable source and retried on the next poll. + const perSourceTimeoutMs = 10_000 + + const withEnumerationDeadline = async (work: Promise): Promise => { + let timer: ReturnType | null = null + + try { + return await Promise.race([ + work, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error('roster enumeration timed out')), perSourceTimeoutMs) + }) + ]) + } finally { + if (timer !== null) { + clearTimeout(timer) + } + } + } + return Promise.all( registry.connections.map(async connection => { let raw: { connection: typeof connection; error?: string; installId?: string; profiles: null | string[] } @@ -12657,7 +12719,10 @@ async function enumerateRegistryAgentSources(registry = readDesktopConnectionsRe } } - const descriptor: any = await ensureRegistryBackend(connection.id, null) + const descriptor: any = await withEnumerationDeadline( + Promise.resolve(ensureRegistryBackend(connection.id, null)) + ) + const body: any = await getJsonForBackend(descriptor, '/api/profiles', { timeoutMs: 8_000 }) // Cached with a TTL, so the 5s roster poll usually pays zero extra