fix(desktop): remote-gateway desktop stops lying after disconnects — roster survives outages, spawn failures log, host-key change stops the retry wall

Three fixes from one remote-gateway (VPS) debug bundle, all live-reproduced
and re-verified on a headed Electron seat via CDP:

- Bots roster no longer shrinks during a gateway outage: source enumeration
  is bounded (10s/source instead of wedging the roster IPC >30s behind a
  dead dial) and a bounced remote source keeps painting its last-known
  profile list (was SSH-only), so 4 bots never show as 2 mid-outage.
- Pool backend spawns that die before the child exists (forced-local spawn
  of a profile that only exists on the remote) now log the failure to
  desktop.log, and the profile-exists guard runs BEFORE the Starting line —
  no more orphaned no-READY/no-exit spawn bursts in bundles.
- An SSH host-key change (VPS reinstall) is classified terminal like a
  reauth rejection: it latches, the boot-failure overlay shows the
  ssh-keygen -R guidance, and the renderer stops the infinite boot-retry
  loop (one bundle had 157 consecutive failures over 2.5h). Reset/repair/
  apply-config clear the latch; live-verified Retry-after-fix boots clean.

apps/desktop/electron/backend-start-failure.test.ts

@@ -3,8 +3,10 @@ import assert from 'node:assert/strict'
 import { test } from 'vitest'
 
 import {
+  isHostKeyChangedBootFailure,
   isRetryableRemoteBootFailure,
   shouldLatchBackendStartFailure,
+  shouldLatchHostKeyChangedFailure,
   shouldLatchRemoteReauthFailure
 } from './backend-start-failure'
 
@@ -81,3 +83,49 @@ test('retryable and reauth-latch are mutually exclusive for remote failures', ()
     assert.equal(retry !== latch, true, `remote failure with reauth=${isReauth} must pick exactly one path`)
   }
 })
+
+test('FIX host-key change: classified from the kind tag and from stringified ssh banners', () => {
+  // classifySshError tags the Error it built; errors that crossed an IPC or
+  // string boundary only keep the message. Both shapes must classify.
+  const tagged = Object.assign(new Error('SSH refused to connect.'), { kind: 'host-key-changed' })
+  assert.equal(isHostKeyChangedBootFailure(tagged), true)
+  assert.equal(
+    isHostKeyChangedBootFailure(new Error('@@@@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @@@@')),
+    true
+  )
+  assert.equal(isHostKeyChangedBootFailure(new Error('Host key verification failed.')), true)
+  assert.equal(
+    isHostKeyChangedBootFailure(
+      new Error('The host key for root@203.0.113.7 has CHANGED since you last connected.')
+    ),
+    true
+  )
+  assert.equal(isHostKeyChangedBootFailure(new Error('Connection refused')), false)
+  assert.equal(isHostKeyChangedBootFailure(null), false)
+})
+
+test('FIX host-key change: latches and is never auto-retried (157-failure loop, Aug 2026 bundle)', () => {
+  // SSH fails closed on a changed host key: every retry re-drives the same
+  // doomed boot until the user clears known_hosts. Terminal, like reauth.
+  const context = { attemptedRemote: true, isReauth: false, isHostKeyChanged: true }
+  assert.equal(shouldLatchHostKeyChangedFailure(context), true)
+  assert.equal(isRetryableRemoteBootFailure(context), false)
+})
+
+test('host-key latch never fires for local failures or ordinary remote faults', () => {
+  assert.equal(shouldLatchHostKeyChangedFailure({ attemptedRemote: false, isReauth: false, isHostKeyChanged: true }), false)
+  assert.equal(shouldLatchHostKeyChangedFailure({ attemptedRemote: true, isReauth: false, isHostKeyChanged: false }), false)
+  assert.equal(shouldLatchHostKeyChangedFailure({ attemptedRemote: true, isReauth: false }), false)
+})
+
+test('every remote failure picks exactly one path: retry, reauth latch, or host-key latch', () => {
+  for (const isReauth of [true, false]) {
+    for (const isHostKeyChanged of [true, false]) {
+      const retry = isRetryableRemoteBootFailure({ attemptedRemote: true, isReauth, isHostKeyChanged })
+      const reauth = shouldLatchRemoteReauthFailure({ attemptedRemote: true, isReauth })
+      const hostKey = shouldLatchHostKeyChangedFailure({ attemptedRemote: true, isReauth, isHostKeyChanged })
+      const picked = [retry, reauth, hostKey].filter(Boolean).length
+      assert.ok(picked >= 1, `remote failure reauth=${isReauth} hostKey=${isHostKeyChanged} fell through every path`)
+    }
+  }
+})

apps/desktop/electron/backend-start-failure.ts

@@ -79,6 +79,44 @@ export interface RemoteBootRetryContext {
    * never self-heal without the user signing in again.
    */
   isReauth: boolean
+  /**
+   * True when SSH refused to connect because the host's key CHANGED
+   * (StrictHostKeyChecking fails closed). Retrying cannot succeed until the
+   * user verifies the change and removes the stale known_hosts entry, so this
+   * is terminal like a reauth rejection — not connectivity.
+   */
+  isHostKeyChanged?: boolean
+}
+
+/**
+ * A host-key-change refusal is identifiable both by the `kind` tag
+ * classifySshError puts on the error and — for errors that crossed a
+ * stringifying boundary — by the stable phrases ssh/our own message carry.
+ * One user hit 157 consecutive boot-retry failures over 2.5h against a
+ * reinstalled VPS (Aug 2026 bundle) because this was classified as transient.
+ */
+export function isHostKeyChangedBootFailure(error: unknown): boolean {
+  if ((error as { kind?: string } | null | undefined)?.kind === 'host-key-changed') {
+    return true
+  }
+
+  const message = error instanceof Error ? error.message : String(error ?? '')
+
+  return /REMOTE HOST IDENTIFICATION HAS CHANGED|Host key verification failed|host key for .+ has CHANGED/i.test(
+    message
+  )
+}
+
+/**
+ * Whether a failed remote boot should latch (into `backendStartFailure`)
+ * because the host key changed. Same rationale as the reauth latch: the
+ * failure cannot self-heal, and an unlatched terminal failure makes every
+ * recovery surface re-drive the identical doomed boot. The latch is released
+ * by the existing reset/repair/apply-config paths once the user has run
+ * `ssh-keygen -R <host>`.
+ */
+export function shouldLatchHostKeyChangedFailure(context: RemoteBootRetryContext): boolean {
+  return context.attemptedRemote && context.isHostKeyChanged === true
 }
 
 /**
@@ -93,9 +131,10 @@ export interface RemoteBootRetryContext {
  * only arms after a completed boot, so the app sat on "Desktop boot failed"
  * until the user manually re-entered the same connection details (which just
  * forced a fresh bootstrap). A missing capability differs from a transient
- * failure: confirmed reauth rejections and local failures stay out of the
- * retry path; everything else remote is connectivity and should retry.
+ * failure: confirmed reauth rejections, host-key changes, and local failures
+ * stay out of the retry path; everything else remote is connectivity and
+ * should retry.
  */
 export function isRetryableRemoteBootFailure(context: RemoteBootRetryContext): boolean {
-  return context.attemptedRemote && !context.isReauth
+  return context.attemptedRemote && !context.isReauth && context.isHostKeyChanged !== true
 }

apps/desktop/electron/connection-registry.test.ts

@@ -271,6 +271,28 @@ test('rememberSshEnumeration: live list wins, cache then seed default', () => {
   })
 })
 
+test('rememberSshEnumeration: a bounced remote source keeps its last-known roster (4-bots-show-as-2)', () => {
+  // A VPS restart makes the remote source unreachable for a few polls. The
+  // last successful enumeration must keep painting so the roster does not
+  // silently drop that source's bots mid-outage.
+  assert.deepEqual(
+    rememberSshEnumeration({ profiles: null, error: 'unreachable' }, ['default', 'ceo', 'accounter'], 'remote'),
+    { profiles: ['default', 'ceo', 'accounter'], error: 'unreachable' }
+  )
+  // Never-seen remote source: no seed — an unreachable URL is not evidence a
+  // backend exists there.
+  assert.deepEqual(rememberSshEnumeration({ profiles: null, error: 'unreachable' }, null, 'remote'), {
+    profiles: null,
+    error: 'unreachable'
+  })
+  // Local enumeration failures never reuse a cache (the local runtime answers
+  // authoritatively or not at all).
+  assert.deepEqual(rememberSshEnumeration({ profiles: null, error: 'boom' }, ['default'], 'local'), {
+    profiles: null,
+    error: 'boom'
+  })
+})
+
 test('shouldRetrySshInventory: first try, cooldown, then retry; cache never retries', () => {
   assert.equal(shouldRetrySshInventory(false, null, 1_000), true)
   assert.equal(shouldRetrySshInventory(false, 1_000, 30_000, 60_000), false)

apps/desktop/electron/connection-registry.ts

@@ -339,10 +339,15 @@ export interface RosterAgent {
 }
 
 /**
- * SSH roster enumeration skips undialed sources (connect-on-demand). Reuse the
- * last successful profile list so Bot Mode does not go empty the moment the
- * window switches back to local. Never-seen SSH sources still get a `default`
- * seed so the device is clickable.
+ * Roster enumeration skips undialed sources (connect-on-demand) and reports
+ * unreachable ones with `profiles: null`. Reuse the last successful profile
+ * list so Bot Mode does not go empty (or drop to a partial roster) the moment
+ * a source is briefly unreachable — SSH tunnels drop on sleep/wake, and a
+ * remote gateway bounce (VPS restart) otherwise erased its bots from the
+ * roster until the next successful enumeration ("my 4 bots show as 2", Aug
+ * 2026 bundle). Never-seen SSH sources still get a `default` seed so the
+ * device is clickable; never-seen remote sources stay empty (no seed) since
+ * an unreachable URL is not evidence a backend exists there.
  */
 export function rememberSshEnumeration(
   enumeration: Pick<ConnectionAgents, 'error' | 'profiles'>,
@@ -353,7 +358,7 @@ export function rememberSshEnumeration(
     return enumeration
   }
 
-  if (kind !== 'ssh') {
+  if (kind === 'local') {
     return enumeration
   }
 
@@ -361,7 +366,7 @@ export function rememberSshEnumeration(
     return { profiles: cached, error: enumeration.error }
   }
 
-  if (enumeration.error === 'connect-on-demand') {
+  if (kind === 'ssh' && enumeration.error === 'connect-on-demand') {
     return { profiles: ['default'], error: 'connect-on-demand' }
   }
 

apps/desktop/electron/main.ts

@@ -46,8 +46,10 @@ import {
 } from './backend-probes'
 import { waitForDashboardPortAnnouncement } from './backend-ready'
 import {
+  isHostKeyChangedBootFailure,
   isRetryableRemoteBootFailure,
   shouldLatchBackendStartFailure,
+  shouldLatchHostKeyChangedFailure,
   shouldLatchRemoteReauthFailure
 } from './backend-start-failure'
 import {
@@ -9743,6 +9745,13 @@ async function ensureBackend(profile) {
   }
 
   entry.connectionPromise = spawnPoolBackend(key, entry).catch(async error => {
+    // Land the failure in desktop.log: without this a spawn that dies before
+    // its child exists (guard rejection, runtime resolution) leaves no trace
+    // beyond renderer-side rejections users never see in a bundle.
+    rememberLog(
+      `Hermes backend for profile "${key}" failed to start: ${error instanceof Error ? error.message : String(error)}`
+    )
+
     if (backendPool.get(key) === entry) {
       backendPool.delete(key)
     }
@@ -9825,6 +9834,12 @@ async function ensureRegistryBackend(connectionId, profile) {
       forceLocal: true,
       poolKey: localRoute.poolKey
     }).catch(async error => {
+      // Same trace rule as the v1 pool path: a forced-local child whose spawn
+      // rejects before the child exists must still land in desktop.log.
+      rememberLog(
+        `Hermes backend for profile "${profileKey}" (forced-local) failed to start: ${error instanceof Error ? error.message : String(error)}`
+      )
+
       if (backendPool.get(localRoute.poolKey) === localEntry) {
         backendPool.delete(localRoute.poolKey)
       }
@@ -10106,12 +10121,18 @@ async function spawnPoolBackend(profile, entry, opts: { forceLocal?: boolean; po
   const webDist = resolveWebDist()
   const readyFile = backend.readyFile ? makeDashboardReadyFile() : null
 
-  rememberLog(`Starting Hermes backend for profile "${profile}" via ${backend.label}`)
-
-  const parentStartMarker = await desktopParentStartMarker()
+  // Guard BEFORE the "Starting" line: a profile that only exists on a remote
+  // backend (remote-primary desktop asked for a forced-local child) rejects
+  // here, and logging "Starting" first left an orphaned line with no READY
+  // and no exit — the exact undiagnosable burst signature in remote-gateway
+  // user bundles (Aug 2026, Dash's report).
   assertLocalProfileCanStart(profile, profileDeletionGate, key =>
     directoryExists(path.join(HERMES_HOME, 'profiles', key))
   )
+
+  rememberLog(`Starting Hermes backend for profile "${profile}" via ${backend.label}`)
+
+  const parentStartMarker = await desktopParentStartMarker()
   const backendNonce = crypto.randomBytes(16).toString('hex')
   const parentIdentityEnv = parentWatchdogEnv(process.pid, parentStartMarker, backendNonce)
 
@@ -10657,6 +10678,7 @@ async function startHermes() {
     }
 
     const message = error instanceof Error ? error.message : String(error)
+    const hostKeyChanged = isHostKeyChangedBootFailure(error)
 
     // Only latch LOCAL boot failures. A remote failure (lapsed session / mint
     // timeout / host briefly unreachable across sleep) is transient and has no
@@ -10667,6 +10689,16 @@ async function startHermes() {
       backendStartFailure = error instanceof Error ? error : new Error(message)
     }
 
+    // A host-key CHANGE is the terminal exception among remote failures: SSH
+    // fails closed until the user verifies the change and clears the stale
+    // known_hosts entry, so retrying re-drives the identical doomed boot (one
+    // bundle showed 157 consecutive failures over 2.5h). Latch it like a local
+    // failure — reset/repair/apply-config clear the latch after the user fixes
+    // known_hosts.
+    if (shouldLatchHostKeyChangedFailure({ attemptedRemote, isReauth: false, isHostKeyChanged: hostKeyChanged })) {
+      backendStartFailure = error instanceof Error ? error : new Error(message)
+    }
+
     // A confirmed reauth rejection latches separately: it can't self-heal, and
     // leaving it unlatched hides the overlay's "Sign in" button on every retry.
     if (shouldLatchRemoteReauthFailure({ attemptedRemote, isReauth: isReauthRequiredError(error) })) {
@@ -10681,9 +10713,14 @@ async function startHermes() {
         // Renderer contract for the self-heal loop (#82679): a transient
         // REMOTE failure (dropped SSH/HTTP registered connection, mint
         // timeout) is retryable — the renderer re-attempts the boot with
-        // bounded backoff. Local failures and confirmed reauth rejections
-        // are not: those end in the recovery overlay / sign-in affordance.
-        retryable: isRetryableRemoteBootFailure({ attemptedRemote, isReauth: isReauthRequiredError(error) }),
+        // bounded backoff. Local failures, confirmed reauth rejections, and
+        // host-key changes are not: those end in the recovery overlay /
+        // sign-in affordance.
+        retryable: isRetryableRemoteBootFailure({
+          attemptedRemote,
+          isReauth: isReauthRequiredError(error),
+          isHostKeyChanged: hostKeyChanged
+        }),
         running: false
       },
       { allowDecrease: true }
@@ -12624,6 +12661,31 @@ async function probeSshProfileInventory(connection) {
 }
 
 async function enumerateRegistryAgentSources(registry = readDesktopConnectionsRegistry()) {
+  // One dead source must not wedge the whole roster: ensureRegistryBackend on
+  // an unreachable remote can block up to the 45s readiness timeout, and the
+  // Bot Mode poll runs every 5s — each poll queued behind the dead dial, so
+  // the renderer painted stale rows for the entire outage (and the roster IPC
+  // hung >30s in live repro). Bound each source's enumeration; a timeout is
+  // reported like any other unreachable source and retried on the next poll.
+  const perSourceTimeoutMs = 10_000
+
+  const withEnumerationDeadline = async <T,>(work: Promise<T>): Promise<T> => {
+    let timer: ReturnType<typeof setTimeout> | null = null
+
+    try {
+      return await Promise.race([
+        work,
+        new Promise<never>((_resolve, reject) => {
+          timer = setTimeout(() => reject(new Error('roster enumeration timed out')), perSourceTimeoutMs)
+        })
+      ])
+    } finally {
+      if (timer !== null) {
+        clearTimeout(timer)
+      }
+    }
+  }
+
   return Promise.all(
     registry.connections.map(async connection => {
       let raw: { connection: typeof connection; error?: string; installId?: string; profiles: null | string[] }
@@ -12657,7 +12719,10 @@ async function enumerateRegistryAgentSources(registry = readDesktopConnectionsRe
             }
           }
 
-          const descriptor: any = await ensureRegistryBackend(connection.id, null)
+          const descriptor: any = await withEnumerationDeadline(
+            Promise.resolve(ensureRegistryBackend(connection.id, null))
+          )
+
           const body: any = await getJsonForBackend(descriptor, '/api/profiles', { timeoutMs: 8_000 })
 
           // Cached with a TTL, so the 5s roster poll usually pays zero extra