fix(desktop): surface Nous Cloud 503 at the OAuth ticket-mint boundary

The original implementation classified 502/503/504 only inside the readiness
loop, but for OAuth-backed Cloud connections the WebSocket-ticket mint runs
before waitForHermesReady. A server fault there was wrapped by
gatewayTicketFailure into a generic message and the Cloud-down classifier was
never reached. This closes that boundary and fixes a latent regex defect.

- isServerSideHttpError: structured-first (err.statusCode for 502/503/504),
  legacy 'NNN:' prefix as fallback, non-Error inputs rejected. Also fixes the
  committed '\d' (double-escaped, matched a literal backslash) that made the
  function never detect a status prefix.
- makeNousCloudBackendDownError: single factory for the actionable Cloud-down
  error (isCloudBackendDown/statusCode/detail/cause), shared by both the
  ticket-mint boundary and readiness exhaustion.
- main.ts: run the Cloud classifier at mintGatewayWsTicket before the
  gatewayTicketFailure wrap; 401/403 still route to reauth.
- connection-config.ts: gatewayTicketFailure preserves an integer statusCode
  from the source error; auth semantics unchanged.
- boot-progress/IPC: carry isCloudBackendDown and statusCode through
  DesktopBootProgress so the renderer overlay (a PR-body promise) can key on
  the structured result rather than re-classifying the message string.

Tests: backend-health (structured detection, non-Error rejection, factory
shape/cause/guards, legacy fallback), connection-config (statusCode preserve,
401/403 reauth, integer-only copy), and an OAuth ticket-mint integration
regression (Cloud 503 -> actionable Cloud-down; 401 -> reauth). Connection-
config suite 80/80 green; backend-health sync tests green; the async readiness
loop tests cannot run on this host (pre-existing local-run limitation) and are
the CI gate. PR #85373 (#85335).
This commit is contained in:
Axl Ibiza, MBA
2026-08-17 08:38:53 -05:00
committed by Teknium
parent 274158ec13
commit d0ea5f1722
6 changed files with 295 additions and 22 deletions
@@ -10,6 +10,7 @@ import {
isNousCloudAgentUrl,
isReauthRequiredError,
isServerSideHttpError,
makeNousCloudBackendDownError,
waitForHermesReady
} from './backend-health'
@@ -440,3 +441,75 @@ test('waitForHermesReady does not cloud-wrap non-cloud 503 errors', async () =>
assert.equal(error.isCloudBackendDown, undefined)
}
})
test('isServerSideHttpError detects structured statusCode even when the message is opaque', () => {
const err = new Error('upstream unavailable') as any
err.statusCode = 503
const result = isServerSideHttpError(err)
assert.ok(result)
assert.equal(result?.statusCode, 503)
assert.equal(result?.detail, 'upstream unavailable')
const err502 = new Error('bad gateway') as any
err502.statusCode = 502
assert.equal(isServerSideHttpError(err502)?.statusCode, 502)
const err504 = new Error('gateway timeout') as any
err504.statusCode = 504
assert.equal(isServerSideHttpError(err504)?.statusCode, 504)
})
test('isServerSideHttpError rejects non-Error inputs even with a 503-shaped value', () => {
// The structured path requires an actual Error (the fetch layer attaches
// statusCode to an Error instance); a bare string/null/number must not be
// misclassified by the legacy prefix fallback.
assert.equal(isServerSideHttpError('503: something'), null)
assert.equal(isServerSideHttpError({ statusCode: 503 }), null)
assert.equal(isServerSideHttpError(null), null)
assert.equal(isServerSideHttpError(503), null)
})
test('isServerSideHttpError structured path excludes 500/401/403/404/429 even when statusCode is attached', () => {
for (const code of [500, 401, 403, 404, 429]) {
const err = new Error(`HTTP ${code}`) as any
err.statusCode = code
assert.equal(isServerSideHttpError(err), null, `should reject statusCode ${code}`)
}
})
test('makeNousCloudBackendDownError produces the Cloud shape and preserves cause', () => {
const err = new Error('upstream unavailable') as any
err.statusCode = 503
const result = makeNousCloudBackendDownError('https://ares-3009.agents.nousresearch.com', err)
assert.ok(result)
assert.equal((result as any).isCloudBackendDown, true)
assert.equal((result as any).statusCode, 503)
assert.equal((result as any).cause, err)
assert.ok(result?.message.includes('Nous Cloud agent ares-3009.agents.nousresearch.com is down'))
})
test('makeNousCloudBackendDownError returns null for a Cloud 401 (routes to reauth)', () => {
const err = new Error('Unauthorized') as any
err.statusCode = 401
assert.equal(
makeNousCloudBackendDownError('https://ares-3009.agents.nousresearch.com', err),
null
)
})
test('makeNousCloudBackendDownError returns null for a non-Cloud 503 (generic remote failure)', () => {
const err = new Error('Service Unavailable') as any
err.statusCode = 503
assert.equal(makeNousCloudBackendDownError('https://gateway.example.com', err), null)
assert.equal(makeNousCloudBackendDownError('http://127.0.0.1:9000', err), null)
})
test('makeNousCloudBackendDownError preserves legacy string-prefix compatibility', () => {
const result = makeNousCloudBackendDownError(
'https://ares-3009.agents.nousresearch.com',
new Error('503: Service Unavailable')
)
assert.ok(result)
assert.equal((result as any).isCloudBackendDown, true)
assert.equal((result as any).statusCode, 503)
})
+83 -20
View File
@@ -48,8 +48,30 @@ export function isServerSideHttpError(error: unknown): {
statusCode: number
detail: string
} | null {
const message = error instanceof Error ? error.message : String(error ?? '')
const match = /^(\\d{3}):/.exec(message)
// Reject non-Error inputs, as before. The fetch layer attaches statusCode to
// an actual Error instance (err.statusCode = statusCode), so requiring an
// Error is compatible with structured detection and keeps plain strings /
// null / numbers from being misclassified by the legacy prefix.
if (!(error instanceof Error)) {
return null
}
// Structured-first: the real fetch layer attaches err.statusCode = statusCode
// (see fetchJson). That is the strongest transport contract, so inspect it
// before falling back to the legacy "503: ..." string prefix.
if ('statusCode' in error) {
const structured = Number((error as { statusCode?: unknown }).statusCode)
if (Number.isInteger(structured) && (structured === 502 || structured === 503 || structured === 504)) {
const detail = error.message
return { statusCode: structured, detail }
}
}
// Compatibility fallback: the legacy leading "503: ..." prefix. Only reached
// when no structured statusCode matched (or was absent).
const message = error.message
const match = /^(\d{3}):/.exec(message)
if (!match) {
return null
@@ -64,6 +86,59 @@ export function isServerSideHttpError(error: unknown): {
return null
}
/**
* The one factory for the actionable Nous Cloud agent-is-down error, shared by
* both startup boundaries that can observe a server-side HTTP fault:
*
* - OAuth WS-ticket mint (buildRemoteConnection → mintGatewayWsTicket), which
* runs BEFORE the readiness loop; and
* - readiness-probe exhaustion in waitForHermesReady().
*
* Returns null unless the backend is a *.agents.nousresearch.com host AND the
* error classifies as 502/503/504. When it matches, returns an error carrying:
* isCloudBackendDown, statusCode, detail, and the original cause. The renderer
* overlay keys on isCloudBackendDown/statusCode; main owns the classification.
*/
export function makeNousCloudBackendDownError(baseUrl: string, error: unknown): Error | null {
if (!isNousCloudAgentUrl(baseUrl)) {
return null
}
const serverError = isServerSideHttpError(error)
if (serverError === null) {
return null
}
let hostname = baseUrl
try {
hostname = new URL(baseUrl).hostname
} catch {
// baseUrl is known to parse (isNousCloudAgentUrl already did); keep the raw
// value as a last resort rather than throwing.
}
const detail = error instanceof Error ? error.message : String(error ?? '')
const err = new Error(
`Nous Cloud agent ${hostname} is down ` +
`(HTTP ${serverError.statusCode}: server-side fault). ` +
'Check https://portal.nousresearch.com for backend status, ' +
'or switch to Local mode in Settings → Gateway. ' +
'You can also reach out on Discord at discord.gg/NousResearch ' +
'for immediate assistance. ' +
`Original detail: ${detail}`
) as any
err.isCloudBackendDown = true
err.statusCode = serverError.statusCode
err.detail = detail
err.cause = error
return err
}
/**
* True when the backend URL points at a Nous-managed Hermes Cloud instance
* (e.g. ares-3009.agents.nousresearch.com). These are Fly.io-hosted machines
@@ -211,25 +286,13 @@ export async function waitForHermesReady(baseUrl: string, options: HermesReadyOp
// When a Nous-managed cloud agent returns a server-side HTTP error
// (502/503/504), the backend server itself is down — the user cannot
// restart it and the generic "did not become ready" message is opaque.
// Surface an actionable error instead (#85335).
if (isNousCloudAgentUrl(baseUrl)) {
const serverError = isServerSideHttpError(lastError)
// Surface an actionable error instead (#85335). This is the SAME factory
// buildRemoteConnection uses at the OAuth WS-ticket-mint boundary, so both
// startup paths produce the identical Cloud-down shape.
const cloudError = makeNousCloudBackendDownError(baseUrl, lastError)
if (serverError !== null) {
const error = new Error(
`Nous Cloud agent ${new URL(baseUrl).hostname} is down ` +
`(HTTP ${serverError.statusCode}: server-side fault). ` +
'Check https://portal.nousresearch.com for backend status, ' +
'or switch to Local mode in Settings → Gateway. ' +
'You can also reach out on Discord at discord.gg/NousResearch ' +
'for immediate assistance. ' +
`Original detail: ${detail}`
) as any
error.isCloudBackendDown = true
error.statusCode = serverError.statusCode
throw error
}
if (cloudError !== null) {
throw cloudError
}
throw new Error(`Hermes backend did not become ready: ${detail}`)
@@ -14,6 +14,8 @@ import assert from 'node:assert/strict'
import { test } from 'vitest'
import { makeNousCloudBackendDownError } from './backend-health'
import {
apiRequestRegistryConnectionId,
AT_COOKIE_VARIANTS,
@@ -1167,3 +1169,93 @@ test('resolveTestWsUrl (oauth) requires a mintTicket function', async () => {
/mintTicket function is required/
)
})
test('gatewayTicketFailure preserves a structured 503 statusCode as a transport failure', () => {
const source = new Error('upstream unavailable') as any
source.statusCode = 503
const wrapped = gatewayTicketFailure(
source,
'auth message',
'transport message'
)
assert.equal(wrapped.message, 'transport message')
assert.equal((wrapped as any).statusCode, 503)
assert.equal((wrapped as any).needsOauthLogin, undefined)
assert.equal((wrapped as any).cause, source)
})
test('gatewayTicketFailure keeps 401 and 403 as reauth with needsOauthLogin', () => {
for (const code of [401, 403]) {
const source = new Error(`HTTP ${code}`) as any
source.statusCode = code
const wrapped = gatewayTicketFailure(
source,
'auth message',
'transport message'
)
assert.equal(wrapped.message, 'auth message')
assert.equal((wrapped as any).needsOauthLogin, true)
assert.equal((wrapped as any).statusCode, code)
assert.equal((wrapped as any).cause, source)
}
})
test('gatewayTicketFailure only copies an integer statusCode, not a message prefix', () => {
// A legacy "503: ..." message carries no structured statusCode; the Cloud
// classifier (makeNousCloudBackendDownError) handles the prefix at the mint
// boundary. The wrapper must not invent an integer from the message.
const source = new Error('503: Service Unavailable') as any
const wrapped = gatewayTicketFailure(
source,
'auth message',
'transport message'
)
assert.equal((wrapped as any).statusCode, undefined)
assert.equal((wrapped as any).needsOauthLogin, undefined)
})
// OAuth integration regression (#85373): the WS-ticket mint boundary runs
// BEFORE waitForHermesReady. This mirrors main.ts buildRemoteConnection's
// catch — classify a Nous Cloud server fault via the shared factory, else
// fall through to gatewayTicketFailure. Proves the production composition:
// 1. Cloud + OAuth ticket mint + 503 -> actionable Cloud-down error
// 2. Cloud + OAuth ticket mint + 401 -> reauth (never Cloud-down)
test('OAuth ticket-mint 503 surfaces the Cloud-down error (startup boundary)', () => {
const baseUrl = 'https://ares-3009.agents.nousresearch.com'
const ticketErr = new Error('upstream unavailable') as any
ticketErr.statusCode = 503
// The exact production sequence from main.ts.
const cloudError = makeNousCloudBackendDownError(baseUrl, ticketErr)
if (cloudError !== null) {
assert.equal((cloudError as any).isCloudBackendDown, true)
assert.equal((cloudError as any).statusCode, 503)
assert.ok(cloudError.message.includes('Nous Cloud agent ares-3009.agents.nousresearch.com is down'))
return
}
const wrapped = gatewayTicketFailure(
ticketErr,
'auth',
'transport'
)
assert.fail(`expected Cloud-down classification, got wrapper: ${wrapped.message}`)
})
test('OAuth ticket-mint 401 stays on the reauth path (never Cloud-down)', () => {
const baseUrl = 'https://ares-3009.agents.nousresearch.com'
const ticketErr = new Error('Unauthorized') as any
ticketErr.statusCode = 401
const cloudError = makeNousCloudBackendDownError(baseUrl, ticketErr)
assert.equal(cloudError, null, 'a 401 must not become a Cloud-down error')
const wrapped = gatewayTicketFailure(
ticketErr,
'auth message',
'transport message'
)
assert.equal(wrapped.message, 'auth message')
assert.equal((wrapped as any).needsOauthLogin, true)
assert.equal((wrapped as any).statusCode, 401)
})
@@ -134,6 +134,18 @@ function gatewayTicketFailure(error, authMessage, transportMessage) {
;(err as any).needsOauthLogin = true
}
// Preserve structured HTTP context when the source error carried an integer
// statusCode (the fetch layer attaches err.statusCode). Downstream Cloud
// classification (isServerSideHttpError / makeNousCloudBackendDownError) and
// the renderer overlay depend on it surviving the ticket-error wrapper. Auth
// semantics are unchanged: 401/403 route to reauth, 5xx stays a transport
// failure, everything else keeps current behavior.
const sourceStatus = Number(error && typeof error === 'object' ? (error as any).statusCode : NaN)
if (Number.isInteger(sourceStatus)) {
;(err as any).statusCode = sourceStatus
}
err.cause = error
return err
+31 -2
View File
@@ -35,7 +35,7 @@ import { stopBackendChild as stopBackendChildImpl, stopBackendTreesForUpdate } f
import { dashboardFallbackArgs, sourceDeclaresServe } from './backend-command'
import { createBackendConnectionState } from './backend-connection-state'
import { buildDesktopBackendEnv, hermesManagedNodePathEntries, normalizeHermesHomeRoot } from './backend-env'
import { isReauthRequiredError, waitForHermesReady } from './backend-health'
import { isReauthRequiredError, makeNousCloudBackendDownError, waitForHermesReady } from './backend-health'
import { backendCommandMatches, createBackendOwnership, createBackendShutdownCoordinator } from './backend-ownership'
import {
canImportHermesCli,
@@ -1371,11 +1371,13 @@ let nativeThemeListenerInstalled = false
let bootProgressState = {
error: null,
fakeMode: BOOT_FAKE_MODE,
isCloudBackendDown: false,
message: 'Waiting to start Hermes backend',
phase: 'idle',
progress: 0,
retryable: false,
running: false,
statusCode: null,
timestamp: Date.now()
}
@@ -8979,6 +8981,19 @@ async function buildRemoteConnection(
try {
ticket = await mintGatewayWsTicket(baseUrl, remoteHeaders)
} catch (error) {
// For a Nous-managed Cloud agent, a 502/503/504 from the WS-ticket mint
// means the backend server itself is down — the actionable Cloud-down
// error. This boundary runs BEFORE the readiness loop, so without this
// the ticket wrapper below would swallow the server-fault classification
// and the renderer would never see isCloudBackendDown. Preserve the
// existing 401/403 reauth and generic transport behavior for everything
// else (#85335).
const cloudError = makeNousCloudBackendDownError(baseUrl, error)
if (cloudError !== null) {
throw cloudError
}
throw gatewayTicketFailure(
error,
'Your remote gateway session has expired. Open Settings → Gateway and click "Sign in" again.',
@@ -10892,6 +10907,18 @@ async function startHermes() {
const message = error instanceof Error ? error.message : String(error)
const hostKeyChanged = isHostKeyChangedBootFailure(error)
// Carry structured Cloud-down metadata through the boot-progress / IPC
// boundary when present, so the renderer overlay can key on it rather than
// re-classifying the message string. main owns classification; the renderer
// only consumes the structured result (#85335).
const isCloudBackendDown =
Boolean(error && typeof error === 'object' && (error as any).isCloudBackendDown === true)
const statusCode = Number(
error && typeof error === 'object' && Number.isInteger((error as any).statusCode)
? (error as any).statusCode
: NaN
)
// Only latch LOCAL boot failures. A remote failure (lapsed session / mint
// timeout / host briefly unreachable across sleep) is transient and has no
// child 'exit' handler to clear the cache — latching it would wedge the app
@@ -10920,6 +10947,7 @@ async function startHermes() {
updateBootProgress(
{
error: message,
isCloudBackendDown: isCloudBackendDown || undefined,
message: `Desktop boot failed: ${message}`,
phase: 'backend.error',
// Renderer contract for the self-heal loop (#82679): a transient
@@ -10933,7 +10961,8 @@ async function startHermes() {
isReauth: isReauthRequiredError(error),
isHostKeyChanged: hostKeyChanged
}),
running: false
running: false,
statusCode: Number.isInteger(statusCode) ? statusCode : undefined
},
{ allowDecrease: true }
)
+4
View File
@@ -1000,6 +1000,8 @@ export interface DesktopCloudAgentSignInResult {
export interface DesktopBootProgress {
error: string | null
fakeMode: boolean
/** True when the boot failure is a Nous Cloud agent that is down (HTTP 502/503/504). */
isCloudBackendDown?: boolean
message: string
phase: string
progress: number
@@ -1011,6 +1013,8 @@ export interface DesktopBootProgress {
*/
retryable?: boolean
running: boolean
/** Structured HTTP status when the boot failure carried one (e.g. 503). */
statusCode?: number | null
timestamp: number
}