fix(desktop): surface Nous Cloud 503 at the OAuth ticket-mint boundary
The original implementation classified 502/503/504 only inside the readiness loop, but for OAuth-backed Cloud connections the WebSocket-ticket mint runs before waitForHermesReady. A server fault there was wrapped by gatewayTicketFailure into a generic message and the Cloud-down classifier was never reached. This closes that boundary and fixes a latent regex defect. - isServerSideHttpError: structured-first (err.statusCode for 502/503/504), legacy 'NNN:' prefix as fallback, non-Error inputs rejected. Also fixes the committed '\d' (double-escaped, matched a literal backslash) that made the function never detect a status prefix. - makeNousCloudBackendDownError: single factory for the actionable Cloud-down error (isCloudBackendDown/statusCode/detail/cause), shared by both the ticket-mint boundary and readiness exhaustion. - main.ts: run the Cloud classifier at mintGatewayWsTicket before the gatewayTicketFailure wrap; 401/403 still route to reauth. - connection-config.ts: gatewayTicketFailure preserves an integer statusCode from the source error; auth semantics unchanged. - boot-progress/IPC: carry isCloudBackendDown and statusCode through DesktopBootProgress so the renderer overlay (a PR-body promise) can key on the structured result rather than re-classifying the message string. Tests: backend-health (structured detection, non-Error rejection, factory shape/cause/guards, legacy fallback), connection-config (statusCode preserve, 401/403 reauth, integer-only copy), and an OAuth ticket-mint integration regression (Cloud 503 -> actionable Cloud-down; 401 -> reauth). Connection- config suite 80/80 green; backend-health sync tests green; the async readiness loop tests cannot run on this host (pre-existing local-run limitation) and are the CI gate. PR #85373 (#85335).
This commit is contained in:
@@ -10,6 +10,7 @@ import {
|
||||
isNousCloudAgentUrl,
|
||||
isReauthRequiredError,
|
||||
isServerSideHttpError,
|
||||
makeNousCloudBackendDownError,
|
||||
waitForHermesReady
|
||||
} from './backend-health'
|
||||
|
||||
@@ -440,3 +441,75 @@ test('waitForHermesReady does not cloud-wrap non-cloud 503 errors', async () =>
|
||||
assert.equal(error.isCloudBackendDown, undefined)
|
||||
}
|
||||
})
|
||||
|
||||
test('isServerSideHttpError detects structured statusCode even when the message is opaque', () => {
|
||||
const err = new Error('upstream unavailable') as any
|
||||
err.statusCode = 503
|
||||
const result = isServerSideHttpError(err)
|
||||
assert.ok(result)
|
||||
assert.equal(result?.statusCode, 503)
|
||||
assert.equal(result?.detail, 'upstream unavailable')
|
||||
|
||||
const err502 = new Error('bad gateway') as any
|
||||
err502.statusCode = 502
|
||||
assert.equal(isServerSideHttpError(err502)?.statusCode, 502)
|
||||
|
||||
const err504 = new Error('gateway timeout') as any
|
||||
err504.statusCode = 504
|
||||
assert.equal(isServerSideHttpError(err504)?.statusCode, 504)
|
||||
})
|
||||
|
||||
test('isServerSideHttpError rejects non-Error inputs even with a 503-shaped value', () => {
|
||||
// The structured path requires an actual Error (the fetch layer attaches
|
||||
// statusCode to an Error instance); a bare string/null/number must not be
|
||||
// misclassified by the legacy prefix fallback.
|
||||
assert.equal(isServerSideHttpError('503: something'), null)
|
||||
assert.equal(isServerSideHttpError({ statusCode: 503 }), null)
|
||||
assert.equal(isServerSideHttpError(null), null)
|
||||
assert.equal(isServerSideHttpError(503), null)
|
||||
})
|
||||
|
||||
test('isServerSideHttpError structured path excludes 500/401/403/404/429 even when statusCode is attached', () => {
|
||||
for (const code of [500, 401, 403, 404, 429]) {
|
||||
const err = new Error(`HTTP ${code}`) as any
|
||||
err.statusCode = code
|
||||
assert.equal(isServerSideHttpError(err), null, `should reject statusCode ${code}`)
|
||||
}
|
||||
})
|
||||
|
||||
test('makeNousCloudBackendDownError produces the Cloud shape and preserves cause', () => {
|
||||
const err = new Error('upstream unavailable') as any
|
||||
err.statusCode = 503
|
||||
const result = makeNousCloudBackendDownError('https://ares-3009.agents.nousresearch.com', err)
|
||||
assert.ok(result)
|
||||
assert.equal((result as any).isCloudBackendDown, true)
|
||||
assert.equal((result as any).statusCode, 503)
|
||||
assert.equal((result as any).cause, err)
|
||||
assert.ok(result?.message.includes('Nous Cloud agent ares-3009.agents.nousresearch.com is down'))
|
||||
})
|
||||
|
||||
test('makeNousCloudBackendDownError returns null for a Cloud 401 (routes to reauth)', () => {
|
||||
const err = new Error('Unauthorized') as any
|
||||
err.statusCode = 401
|
||||
assert.equal(
|
||||
makeNousCloudBackendDownError('https://ares-3009.agents.nousresearch.com', err),
|
||||
null
|
||||
)
|
||||
})
|
||||
|
||||
test('makeNousCloudBackendDownError returns null for a non-Cloud 503 (generic remote failure)', () => {
|
||||
const err = new Error('Service Unavailable') as any
|
||||
err.statusCode = 503
|
||||
assert.equal(makeNousCloudBackendDownError('https://gateway.example.com', err), null)
|
||||
assert.equal(makeNousCloudBackendDownError('http://127.0.0.1:9000', err), null)
|
||||
})
|
||||
|
||||
test('makeNousCloudBackendDownError preserves legacy string-prefix compatibility', () => {
|
||||
const result = makeNousCloudBackendDownError(
|
||||
'https://ares-3009.agents.nousresearch.com',
|
||||
new Error('503: Service Unavailable')
|
||||
)
|
||||
assert.ok(result)
|
||||
assert.equal((result as any).isCloudBackendDown, true)
|
||||
assert.equal((result as any).statusCode, 503)
|
||||
})
|
||||
|
||||
@@ -48,8 +48,30 @@ export function isServerSideHttpError(error: unknown): {
|
||||
statusCode: number
|
||||
detail: string
|
||||
} | null {
|
||||
const message = error instanceof Error ? error.message : String(error ?? '')
|
||||
const match = /^(\\d{3}):/.exec(message)
|
||||
// Reject non-Error inputs, as before. The fetch layer attaches statusCode to
|
||||
// an actual Error instance (err.statusCode = statusCode), so requiring an
|
||||
// Error is compatible with structured detection and keeps plain strings /
|
||||
// null / numbers from being misclassified by the legacy prefix.
|
||||
if (!(error instanceof Error)) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Structured-first: the real fetch layer attaches err.statusCode = statusCode
|
||||
// (see fetchJson). That is the strongest transport contract, so inspect it
|
||||
// before falling back to the legacy "503: ..." string prefix.
|
||||
if ('statusCode' in error) {
|
||||
const structured = Number((error as { statusCode?: unknown }).statusCode)
|
||||
|
||||
if (Number.isInteger(structured) && (structured === 502 || structured === 503 || structured === 504)) {
|
||||
const detail = error.message
|
||||
return { statusCode: structured, detail }
|
||||
}
|
||||
}
|
||||
|
||||
// Compatibility fallback: the legacy leading "503: ..." prefix. Only reached
|
||||
// when no structured statusCode matched (or was absent).
|
||||
const message = error.message
|
||||
const match = /^(\d{3}):/.exec(message)
|
||||
|
||||
if (!match) {
|
||||
return null
|
||||
@@ -64,6 +86,59 @@ export function isServerSideHttpError(error: unknown): {
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* The one factory for the actionable Nous Cloud agent-is-down error, shared by
|
||||
* both startup boundaries that can observe a server-side HTTP fault:
|
||||
*
|
||||
* - OAuth WS-ticket mint (buildRemoteConnection → mintGatewayWsTicket), which
|
||||
* runs BEFORE the readiness loop; and
|
||||
* - readiness-probe exhaustion in waitForHermesReady().
|
||||
*
|
||||
* Returns null unless the backend is a *.agents.nousresearch.com host AND the
|
||||
* error classifies as 502/503/504. When it matches, returns an error carrying:
|
||||
* isCloudBackendDown, statusCode, detail, and the original cause. The renderer
|
||||
* overlay keys on isCloudBackendDown/statusCode; main owns the classification.
|
||||
*/
|
||||
export function makeNousCloudBackendDownError(baseUrl: string, error: unknown): Error | null {
|
||||
if (!isNousCloudAgentUrl(baseUrl)) {
|
||||
return null
|
||||
}
|
||||
|
||||
const serverError = isServerSideHttpError(error)
|
||||
|
||||
if (serverError === null) {
|
||||
return null
|
||||
}
|
||||
|
||||
let hostname = baseUrl
|
||||
|
||||
try {
|
||||
hostname = new URL(baseUrl).hostname
|
||||
} catch {
|
||||
// baseUrl is known to parse (isNousCloudAgentUrl already did); keep the raw
|
||||
// value as a last resort rather than throwing.
|
||||
}
|
||||
|
||||
const detail = error instanceof Error ? error.message : String(error ?? '')
|
||||
|
||||
const err = new Error(
|
||||
`Nous Cloud agent ${hostname} is down ` +
|
||||
`(HTTP ${serverError.statusCode}: server-side fault). ` +
|
||||
'Check https://portal.nousresearch.com for backend status, ' +
|
||||
'or switch to Local mode in Settings → Gateway. ' +
|
||||
'You can also reach out on Discord at discord.gg/NousResearch ' +
|
||||
'for immediate assistance. ' +
|
||||
`Original detail: ${detail}`
|
||||
) as any
|
||||
|
||||
err.isCloudBackendDown = true
|
||||
err.statusCode = serverError.statusCode
|
||||
err.detail = detail
|
||||
err.cause = error
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the backend URL points at a Nous-managed Hermes Cloud instance
|
||||
* (e.g. ares-3009.agents.nousresearch.com). These are Fly.io-hosted machines
|
||||
@@ -211,25 +286,13 @@ export async function waitForHermesReady(baseUrl: string, options: HermesReadyOp
|
||||
// When a Nous-managed cloud agent returns a server-side HTTP error
|
||||
// (502/503/504), the backend server itself is down — the user cannot
|
||||
// restart it and the generic "did not become ready" message is opaque.
|
||||
// Surface an actionable error instead (#85335).
|
||||
if (isNousCloudAgentUrl(baseUrl)) {
|
||||
const serverError = isServerSideHttpError(lastError)
|
||||
// Surface an actionable error instead (#85335). This is the SAME factory
|
||||
// buildRemoteConnection uses at the OAuth WS-ticket-mint boundary, so both
|
||||
// startup paths produce the identical Cloud-down shape.
|
||||
const cloudError = makeNousCloudBackendDownError(baseUrl, lastError)
|
||||
|
||||
if (serverError !== null) {
|
||||
const error = new Error(
|
||||
`Nous Cloud agent ${new URL(baseUrl).hostname} is down ` +
|
||||
`(HTTP ${serverError.statusCode}: server-side fault). ` +
|
||||
'Check https://portal.nousresearch.com for backend status, ' +
|
||||
'or switch to Local mode in Settings → Gateway. ' +
|
||||
'You can also reach out on Discord at discord.gg/NousResearch ' +
|
||||
'for immediate assistance. ' +
|
||||
`Original detail: ${detail}`
|
||||
) as any
|
||||
|
||||
error.isCloudBackendDown = true
|
||||
error.statusCode = serverError.statusCode
|
||||
throw error
|
||||
}
|
||||
if (cloudError !== null) {
|
||||
throw cloudError
|
||||
}
|
||||
|
||||
throw new Error(`Hermes backend did not become ready: ${detail}`)
|
||||
|
||||
@@ -14,6 +14,8 @@ import assert from 'node:assert/strict'
|
||||
|
||||
import { test } from 'vitest'
|
||||
|
||||
import { makeNousCloudBackendDownError } from './backend-health'
|
||||
|
||||
import {
|
||||
apiRequestRegistryConnectionId,
|
||||
AT_COOKIE_VARIANTS,
|
||||
@@ -1167,3 +1169,93 @@ test('resolveTestWsUrl (oauth) requires a mintTicket function', async () => {
|
||||
/mintTicket function is required/
|
||||
)
|
||||
})
|
||||
|
||||
test('gatewayTicketFailure preserves a structured 503 statusCode as a transport failure', () => {
|
||||
const source = new Error('upstream unavailable') as any
|
||||
source.statusCode = 503
|
||||
const wrapped = gatewayTicketFailure(
|
||||
source,
|
||||
'auth message',
|
||||
'transport message'
|
||||
)
|
||||
assert.equal(wrapped.message, 'transport message')
|
||||
assert.equal((wrapped as any).statusCode, 503)
|
||||
assert.equal((wrapped as any).needsOauthLogin, undefined)
|
||||
assert.equal((wrapped as any).cause, source)
|
||||
})
|
||||
|
||||
test('gatewayTicketFailure keeps 401 and 403 as reauth with needsOauthLogin', () => {
|
||||
for (const code of [401, 403]) {
|
||||
const source = new Error(`HTTP ${code}`) as any
|
||||
source.statusCode = code
|
||||
const wrapped = gatewayTicketFailure(
|
||||
source,
|
||||
'auth message',
|
||||
'transport message'
|
||||
)
|
||||
assert.equal(wrapped.message, 'auth message')
|
||||
assert.equal((wrapped as any).needsOauthLogin, true)
|
||||
assert.equal((wrapped as any).statusCode, code)
|
||||
assert.equal((wrapped as any).cause, source)
|
||||
}
|
||||
})
|
||||
|
||||
test('gatewayTicketFailure only copies an integer statusCode, not a message prefix', () => {
|
||||
// A legacy "503: ..." message carries no structured statusCode; the Cloud
|
||||
// classifier (makeNousCloudBackendDownError) handles the prefix at the mint
|
||||
// boundary. The wrapper must not invent an integer from the message.
|
||||
const source = new Error('503: Service Unavailable') as any
|
||||
const wrapped = gatewayTicketFailure(
|
||||
source,
|
||||
'auth message',
|
||||
'transport message'
|
||||
)
|
||||
assert.equal((wrapped as any).statusCode, undefined)
|
||||
assert.equal((wrapped as any).needsOauthLogin, undefined)
|
||||
})
|
||||
|
||||
// OAuth integration regression (#85373): the WS-ticket mint boundary runs
|
||||
// BEFORE waitForHermesReady. This mirrors main.ts buildRemoteConnection's
|
||||
// catch — classify a Nous Cloud server fault via the shared factory, else
|
||||
// fall through to gatewayTicketFailure. Proves the production composition:
|
||||
// 1. Cloud + OAuth ticket mint + 503 -> actionable Cloud-down error
|
||||
// 2. Cloud + OAuth ticket mint + 401 -> reauth (never Cloud-down)
|
||||
test('OAuth ticket-mint 503 surfaces the Cloud-down error (startup boundary)', () => {
|
||||
const baseUrl = 'https://ares-3009.agents.nousresearch.com'
|
||||
const ticketErr = new Error('upstream unavailable') as any
|
||||
ticketErr.statusCode = 503
|
||||
|
||||
// The exact production sequence from main.ts.
|
||||
const cloudError = makeNousCloudBackendDownError(baseUrl, ticketErr)
|
||||
if (cloudError !== null) {
|
||||
assert.equal((cloudError as any).isCloudBackendDown, true)
|
||||
assert.equal((cloudError as any).statusCode, 503)
|
||||
assert.ok(cloudError.message.includes('Nous Cloud agent ares-3009.agents.nousresearch.com is down'))
|
||||
return
|
||||
}
|
||||
|
||||
const wrapped = gatewayTicketFailure(
|
||||
ticketErr,
|
||||
'auth',
|
||||
'transport'
|
||||
)
|
||||
assert.fail(`expected Cloud-down classification, got wrapper: ${wrapped.message}`)
|
||||
})
|
||||
|
||||
test('OAuth ticket-mint 401 stays on the reauth path (never Cloud-down)', () => {
|
||||
const baseUrl = 'https://ares-3009.agents.nousresearch.com'
|
||||
const ticketErr = new Error('Unauthorized') as any
|
||||
ticketErr.statusCode = 401
|
||||
|
||||
const cloudError = makeNousCloudBackendDownError(baseUrl, ticketErr)
|
||||
assert.equal(cloudError, null, 'a 401 must not become a Cloud-down error')
|
||||
|
||||
const wrapped = gatewayTicketFailure(
|
||||
ticketErr,
|
||||
'auth message',
|
||||
'transport message'
|
||||
)
|
||||
assert.equal(wrapped.message, 'auth message')
|
||||
assert.equal((wrapped as any).needsOauthLogin, true)
|
||||
assert.equal((wrapped as any).statusCode, 401)
|
||||
})
|
||||
|
||||
@@ -134,6 +134,18 @@ function gatewayTicketFailure(error, authMessage, transportMessage) {
|
||||
;(err as any).needsOauthLogin = true
|
||||
}
|
||||
|
||||
// Preserve structured HTTP context when the source error carried an integer
|
||||
// statusCode (the fetch layer attaches err.statusCode). Downstream Cloud
|
||||
// classification (isServerSideHttpError / makeNousCloudBackendDownError) and
|
||||
// the renderer overlay depend on it surviving the ticket-error wrapper. Auth
|
||||
// semantics are unchanged: 401/403 route to reauth, 5xx stays a transport
|
||||
// failure, everything else keeps current behavior.
|
||||
const sourceStatus = Number(error && typeof error === 'object' ? (error as any).statusCode : NaN)
|
||||
|
||||
if (Number.isInteger(sourceStatus)) {
|
||||
;(err as any).statusCode = sourceStatus
|
||||
}
|
||||
|
||||
err.cause = error
|
||||
|
||||
return err
|
||||
|
||||
@@ -35,7 +35,7 @@ import { stopBackendChild as stopBackendChildImpl, stopBackendTreesForUpdate } f
|
||||
import { dashboardFallbackArgs, sourceDeclaresServe } from './backend-command'
|
||||
import { createBackendConnectionState } from './backend-connection-state'
|
||||
import { buildDesktopBackendEnv, hermesManagedNodePathEntries, normalizeHermesHomeRoot } from './backend-env'
|
||||
import { isReauthRequiredError, waitForHermesReady } from './backend-health'
|
||||
import { isReauthRequiredError, makeNousCloudBackendDownError, waitForHermesReady } from './backend-health'
|
||||
import { backendCommandMatches, createBackendOwnership, createBackendShutdownCoordinator } from './backend-ownership'
|
||||
import {
|
||||
canImportHermesCli,
|
||||
@@ -1371,11 +1371,13 @@ let nativeThemeListenerInstalled = false
|
||||
let bootProgressState = {
|
||||
error: null,
|
||||
fakeMode: BOOT_FAKE_MODE,
|
||||
isCloudBackendDown: false,
|
||||
message: 'Waiting to start Hermes backend',
|
||||
phase: 'idle',
|
||||
progress: 0,
|
||||
retryable: false,
|
||||
running: false,
|
||||
statusCode: null,
|
||||
timestamp: Date.now()
|
||||
}
|
||||
|
||||
@@ -8979,6 +8981,19 @@ async function buildRemoteConnection(
|
||||
try {
|
||||
ticket = await mintGatewayWsTicket(baseUrl, remoteHeaders)
|
||||
} catch (error) {
|
||||
// For a Nous-managed Cloud agent, a 502/503/504 from the WS-ticket mint
|
||||
// means the backend server itself is down — the actionable Cloud-down
|
||||
// error. This boundary runs BEFORE the readiness loop, so without this
|
||||
// the ticket wrapper below would swallow the server-fault classification
|
||||
// and the renderer would never see isCloudBackendDown. Preserve the
|
||||
// existing 401/403 reauth and generic transport behavior for everything
|
||||
// else (#85335).
|
||||
const cloudError = makeNousCloudBackendDownError(baseUrl, error)
|
||||
|
||||
if (cloudError !== null) {
|
||||
throw cloudError
|
||||
}
|
||||
|
||||
throw gatewayTicketFailure(
|
||||
error,
|
||||
'Your remote gateway session has expired. Open Settings → Gateway and click "Sign in" again.',
|
||||
@@ -10892,6 +10907,18 @@ async function startHermes() {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
const hostKeyChanged = isHostKeyChangedBootFailure(error)
|
||||
|
||||
// Carry structured Cloud-down metadata through the boot-progress / IPC
|
||||
// boundary when present, so the renderer overlay can key on it rather than
|
||||
// re-classifying the message string. main owns classification; the renderer
|
||||
// only consumes the structured result (#85335).
|
||||
const isCloudBackendDown =
|
||||
Boolean(error && typeof error === 'object' && (error as any).isCloudBackendDown === true)
|
||||
const statusCode = Number(
|
||||
error && typeof error === 'object' && Number.isInteger((error as any).statusCode)
|
||||
? (error as any).statusCode
|
||||
: NaN
|
||||
)
|
||||
|
||||
// Only latch LOCAL boot failures. A remote failure (lapsed session / mint
|
||||
// timeout / host briefly unreachable across sleep) is transient and has no
|
||||
// child 'exit' handler to clear the cache — latching it would wedge the app
|
||||
@@ -10920,6 +10947,7 @@ async function startHermes() {
|
||||
updateBootProgress(
|
||||
{
|
||||
error: message,
|
||||
isCloudBackendDown: isCloudBackendDown || undefined,
|
||||
message: `Desktop boot failed: ${message}`,
|
||||
phase: 'backend.error',
|
||||
// Renderer contract for the self-heal loop (#82679): a transient
|
||||
@@ -10933,7 +10961,8 @@ async function startHermes() {
|
||||
isReauth: isReauthRequiredError(error),
|
||||
isHostKeyChanged: hostKeyChanged
|
||||
}),
|
||||
running: false
|
||||
running: false,
|
||||
statusCode: Number.isInteger(statusCode) ? statusCode : undefined
|
||||
},
|
||||
{ allowDecrease: true }
|
||||
)
|
||||
|
||||
Vendored
+4
@@ -1000,6 +1000,8 @@ export interface DesktopCloudAgentSignInResult {
|
||||
export interface DesktopBootProgress {
|
||||
error: string | null
|
||||
fakeMode: boolean
|
||||
/** True when the boot failure is a Nous Cloud agent that is down (HTTP 502/503/504). */
|
||||
isCloudBackendDown?: boolean
|
||||
message: string
|
||||
phase: string
|
||||
progress: number
|
||||
@@ -1011,6 +1013,8 @@ export interface DesktopBootProgress {
|
||||
*/
|
||||
retryable?: boolean
|
||||
running: boolean
|
||||
/** Structured HTTP status when the boot failure carried one (e.g. 503). */
|
||||
statusCode?: number | null
|
||||
timestamp: number
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user