diff --git a/apps/desktop/electron/backend-health.test.ts b/apps/desktop/electron/backend-health.test.ts index a6e370b5f9..128bd7f502 100644 --- a/apps/desktop/electron/backend-health.test.ts +++ b/apps/desktop/electron/backend-health.test.ts @@ -10,6 +10,7 @@ import { isNousCloudAgentUrl, isReauthRequiredError, isServerSideHttpError, + makeNousCloudBackendDownError, waitForHermesReady } from './backend-health' @@ -440,3 +441,75 @@ test('waitForHermesReady does not cloud-wrap non-cloud 503 errors', async () => assert.equal(error.isCloudBackendDown, undefined) } }) + +test('isServerSideHttpError detects structured statusCode even when the message is opaque', () => { + const err = new Error('upstream unavailable') as any + err.statusCode = 503 + const result = isServerSideHttpError(err) + assert.ok(result) + assert.equal(result?.statusCode, 503) + assert.equal(result?.detail, 'upstream unavailable') + + const err502 = new Error('bad gateway') as any + err502.statusCode = 502 + assert.equal(isServerSideHttpError(err502)?.statusCode, 502) + + const err504 = new Error('gateway timeout') as any + err504.statusCode = 504 + assert.equal(isServerSideHttpError(err504)?.statusCode, 504) +}) + +test('isServerSideHttpError rejects non-Error inputs even with a 503-shaped value', () => { + // The structured path requires an actual Error (the fetch layer attaches + // statusCode to an Error instance); a bare string/null/number must not be + // misclassified by the legacy prefix fallback. + assert.equal(isServerSideHttpError('503: something'), null) + assert.equal(isServerSideHttpError({ statusCode: 503 }), null) + assert.equal(isServerSideHttpError(null), null) + assert.equal(isServerSideHttpError(503), null) +}) + +test('isServerSideHttpError structured path excludes 500/401/403/404/429 even when statusCode is attached', () => { + for (const code of [500, 401, 403, 404, 429]) { + const err = new Error(`HTTP ${code}`) as any + err.statusCode = code + assert.equal(isServerSideHttpError(err), null, `should reject statusCode ${code}`) + } +}) + +test('makeNousCloudBackendDownError produces the Cloud shape and preserves cause', () => { + const err = new Error('upstream unavailable') as any + err.statusCode = 503 + const result = makeNousCloudBackendDownError('https://ares-3009.agents.nousresearch.com', err) + assert.ok(result) + assert.equal((result as any).isCloudBackendDown, true) + assert.equal((result as any).statusCode, 503) + assert.equal((result as any).cause, err) + assert.ok(result?.message.includes('Nous Cloud agent ares-3009.agents.nousresearch.com is down')) +}) + +test('makeNousCloudBackendDownError returns null for a Cloud 401 (routes to reauth)', () => { + const err = new Error('Unauthorized') as any + err.statusCode = 401 + assert.equal( + makeNousCloudBackendDownError('https://ares-3009.agents.nousresearch.com', err), + null + ) +}) + +test('makeNousCloudBackendDownError returns null for a non-Cloud 503 (generic remote failure)', () => { + const err = new Error('Service Unavailable') as any + err.statusCode = 503 + assert.equal(makeNousCloudBackendDownError('https://gateway.example.com', err), null) + assert.equal(makeNousCloudBackendDownError('http://127.0.0.1:9000', err), null) +}) + +test('makeNousCloudBackendDownError preserves legacy string-prefix compatibility', () => { + const result = makeNousCloudBackendDownError( + 'https://ares-3009.agents.nousresearch.com', + new Error('503: Service Unavailable') + ) + assert.ok(result) + assert.equal((result as any).isCloudBackendDown, true) + assert.equal((result as any).statusCode, 503) +}) diff --git a/apps/desktop/electron/backend-health.ts b/apps/desktop/electron/backend-health.ts index 731309f885..ca3ae49520 100644 --- a/apps/desktop/electron/backend-health.ts +++ b/apps/desktop/electron/backend-health.ts @@ -48,8 +48,30 @@ export function isServerSideHttpError(error: unknown): { statusCode: number detail: string } | null { - const message = error instanceof Error ? error.message : String(error ?? '') - const match = /^(\\d{3}):/.exec(message) + // Reject non-Error inputs, as before. The fetch layer attaches statusCode to + // an actual Error instance (err.statusCode = statusCode), so requiring an + // Error is compatible with structured detection and keeps plain strings / + // null / numbers from being misclassified by the legacy prefix. + if (!(error instanceof Error)) { + return null + } + + // Structured-first: the real fetch layer attaches err.statusCode = statusCode + // (see fetchJson). That is the strongest transport contract, so inspect it + // before falling back to the legacy "503: ..." string prefix. + if ('statusCode' in error) { + const structured = Number((error as { statusCode?: unknown }).statusCode) + + if (Number.isInteger(structured) && (structured === 502 || structured === 503 || structured === 504)) { + const detail = error.message + return { statusCode: structured, detail } + } + } + + // Compatibility fallback: the legacy leading "503: ..." prefix. Only reached + // when no structured statusCode matched (or was absent). + const message = error.message + const match = /^(\d{3}):/.exec(message) if (!match) { return null @@ -64,6 +86,59 @@ export function isServerSideHttpError(error: unknown): { return null } +/** + * The one factory for the actionable Nous Cloud agent-is-down error, shared by + * both startup boundaries that can observe a server-side HTTP fault: + * + * - OAuth WS-ticket mint (buildRemoteConnection → mintGatewayWsTicket), which + * runs BEFORE the readiness loop; and + * - readiness-probe exhaustion in waitForHermesReady(). + * + * Returns null unless the backend is a *.agents.nousresearch.com host AND the + * error classifies as 502/503/504. When it matches, returns an error carrying: + * isCloudBackendDown, statusCode, detail, and the original cause. The renderer + * overlay keys on isCloudBackendDown/statusCode; main owns the classification. + */ +export function makeNousCloudBackendDownError(baseUrl: string, error: unknown): Error | null { + if (!isNousCloudAgentUrl(baseUrl)) { + return null + } + + const serverError = isServerSideHttpError(error) + + if (serverError === null) { + return null + } + + let hostname = baseUrl + + try { + hostname = new URL(baseUrl).hostname + } catch { + // baseUrl is known to parse (isNousCloudAgentUrl already did); keep the raw + // value as a last resort rather than throwing. + } + + const detail = error instanceof Error ? error.message : String(error ?? '') + + const err = new Error( + `Nous Cloud agent ${hostname} is down ` + + `(HTTP ${serverError.statusCode}: server-side fault). ` + + 'Check https://portal.nousresearch.com for backend status, ' + + 'or switch to Local mode in Settings → Gateway. ' + + 'You can also reach out on Discord at discord.gg/NousResearch ' + + 'for immediate assistance. ' + + `Original detail: ${detail}` + ) as any + + err.isCloudBackendDown = true + err.statusCode = serverError.statusCode + err.detail = detail + err.cause = error + + return err +} + /** * True when the backend URL points at a Nous-managed Hermes Cloud instance * (e.g. ares-3009.agents.nousresearch.com). These are Fly.io-hosted machines @@ -211,25 +286,13 @@ export async function waitForHermesReady(baseUrl: string, options: HermesReadyOp // When a Nous-managed cloud agent returns a server-side HTTP error // (502/503/504), the backend server itself is down — the user cannot // restart it and the generic "did not become ready" message is opaque. - // Surface an actionable error instead (#85335). - if (isNousCloudAgentUrl(baseUrl)) { - const serverError = isServerSideHttpError(lastError) + // Surface an actionable error instead (#85335). This is the SAME factory + // buildRemoteConnection uses at the OAuth WS-ticket-mint boundary, so both + // startup paths produce the identical Cloud-down shape. + const cloudError = makeNousCloudBackendDownError(baseUrl, lastError) - if (serverError !== null) { - const error = new Error( - `Nous Cloud agent ${new URL(baseUrl).hostname} is down ` + - `(HTTP ${serverError.statusCode}: server-side fault). ` + - 'Check https://portal.nousresearch.com for backend status, ' + - 'or switch to Local mode in Settings → Gateway. ' + - 'You can also reach out on Discord at discord.gg/NousResearch ' + - 'for immediate assistance. ' + - `Original detail: ${detail}` - ) as any - - error.isCloudBackendDown = true - error.statusCode = serverError.statusCode - throw error - } + if (cloudError !== null) { + throw cloudError } throw new Error(`Hermes backend did not become ready: ${detail}`) diff --git a/apps/desktop/electron/connection-config.test.ts b/apps/desktop/electron/connection-config.test.ts index 8bc40e639a..55886761f8 100644 --- a/apps/desktop/electron/connection-config.test.ts +++ b/apps/desktop/electron/connection-config.test.ts @@ -14,6 +14,8 @@ import assert from 'node:assert/strict' import { test } from 'vitest' +import { makeNousCloudBackendDownError } from './backend-health' + import { apiRequestRegistryConnectionId, AT_COOKIE_VARIANTS, @@ -1167,3 +1169,93 @@ test('resolveTestWsUrl (oauth) requires a mintTicket function', async () => { /mintTicket function is required/ ) }) + +test('gatewayTicketFailure preserves a structured 503 statusCode as a transport failure', () => { + const source = new Error('upstream unavailable') as any + source.statusCode = 503 + const wrapped = gatewayTicketFailure( + source, + 'auth message', + 'transport message' + ) + assert.equal(wrapped.message, 'transport message') + assert.equal((wrapped as any).statusCode, 503) + assert.equal((wrapped as any).needsOauthLogin, undefined) + assert.equal((wrapped as any).cause, source) +}) + +test('gatewayTicketFailure keeps 401 and 403 as reauth with needsOauthLogin', () => { + for (const code of [401, 403]) { + const source = new Error(`HTTP ${code}`) as any + source.statusCode = code + const wrapped = gatewayTicketFailure( + source, + 'auth message', + 'transport message' + ) + assert.equal(wrapped.message, 'auth message') + assert.equal((wrapped as any).needsOauthLogin, true) + assert.equal((wrapped as any).statusCode, code) + assert.equal((wrapped as any).cause, source) + } +}) + +test('gatewayTicketFailure only copies an integer statusCode, not a message prefix', () => { + // A legacy "503: ..." message carries no structured statusCode; the Cloud + // classifier (makeNousCloudBackendDownError) handles the prefix at the mint + // boundary. The wrapper must not invent an integer from the message. + const source = new Error('503: Service Unavailable') as any + const wrapped = gatewayTicketFailure( + source, + 'auth message', + 'transport message' + ) + assert.equal((wrapped as any).statusCode, undefined) + assert.equal((wrapped as any).needsOauthLogin, undefined) +}) + +// OAuth integration regression (#85373): the WS-ticket mint boundary runs +// BEFORE waitForHermesReady. This mirrors main.ts buildRemoteConnection's +// catch — classify a Nous Cloud server fault via the shared factory, else +// fall through to gatewayTicketFailure. Proves the production composition: +// 1. Cloud + OAuth ticket mint + 503 -> actionable Cloud-down error +// 2. Cloud + OAuth ticket mint + 401 -> reauth (never Cloud-down) +test('OAuth ticket-mint 503 surfaces the Cloud-down error (startup boundary)', () => { + const baseUrl = 'https://ares-3009.agents.nousresearch.com' + const ticketErr = new Error('upstream unavailable') as any + ticketErr.statusCode = 503 + + // The exact production sequence from main.ts. + const cloudError = makeNousCloudBackendDownError(baseUrl, ticketErr) + if (cloudError !== null) { + assert.equal((cloudError as any).isCloudBackendDown, true) + assert.equal((cloudError as any).statusCode, 503) + assert.ok(cloudError.message.includes('Nous Cloud agent ares-3009.agents.nousresearch.com is down')) + return + } + + const wrapped = gatewayTicketFailure( + ticketErr, + 'auth', + 'transport' + ) + assert.fail(`expected Cloud-down classification, got wrapper: ${wrapped.message}`) +}) + +test('OAuth ticket-mint 401 stays on the reauth path (never Cloud-down)', () => { + const baseUrl = 'https://ares-3009.agents.nousresearch.com' + const ticketErr = new Error('Unauthorized') as any + ticketErr.statusCode = 401 + + const cloudError = makeNousCloudBackendDownError(baseUrl, ticketErr) + assert.equal(cloudError, null, 'a 401 must not become a Cloud-down error') + + const wrapped = gatewayTicketFailure( + ticketErr, + 'auth message', + 'transport message' + ) + assert.equal(wrapped.message, 'auth message') + assert.equal((wrapped as any).needsOauthLogin, true) + assert.equal((wrapped as any).statusCode, 401) +}) diff --git a/apps/desktop/electron/connection-config.ts b/apps/desktop/electron/connection-config.ts index 1934fbec0d..10cdd275b1 100644 --- a/apps/desktop/electron/connection-config.ts +++ b/apps/desktop/electron/connection-config.ts @@ -134,6 +134,18 @@ function gatewayTicketFailure(error, authMessage, transportMessage) { ;(err as any).needsOauthLogin = true } + // Preserve structured HTTP context when the source error carried an integer + // statusCode (the fetch layer attaches err.statusCode). Downstream Cloud + // classification (isServerSideHttpError / makeNousCloudBackendDownError) and + // the renderer overlay depend on it surviving the ticket-error wrapper. Auth + // semantics are unchanged: 401/403 route to reauth, 5xx stays a transport + // failure, everything else keeps current behavior. + const sourceStatus = Number(error && typeof error === 'object' ? (error as any).statusCode : NaN) + + if (Number.isInteger(sourceStatus)) { + ;(err as any).statusCode = sourceStatus + } + err.cause = error return err diff --git a/apps/desktop/electron/main.ts b/apps/desktop/electron/main.ts index adf7338ef5..e5f94479e9 100644 --- a/apps/desktop/electron/main.ts +++ b/apps/desktop/electron/main.ts @@ -35,7 +35,7 @@ import { stopBackendChild as stopBackendChildImpl, stopBackendTreesForUpdate } f import { dashboardFallbackArgs, sourceDeclaresServe } from './backend-command' import { createBackendConnectionState } from './backend-connection-state' import { buildDesktopBackendEnv, hermesManagedNodePathEntries, normalizeHermesHomeRoot } from './backend-env' -import { isReauthRequiredError, waitForHermesReady } from './backend-health' +import { isReauthRequiredError, makeNousCloudBackendDownError, waitForHermesReady } from './backend-health' import { backendCommandMatches, createBackendOwnership, createBackendShutdownCoordinator } from './backend-ownership' import { canImportHermesCli, @@ -1371,11 +1371,13 @@ let nativeThemeListenerInstalled = false let bootProgressState = { error: null, fakeMode: BOOT_FAKE_MODE, + isCloudBackendDown: false, message: 'Waiting to start Hermes backend', phase: 'idle', progress: 0, retryable: false, running: false, + statusCode: null, timestamp: Date.now() } @@ -8979,6 +8981,19 @@ async function buildRemoteConnection( try { ticket = await mintGatewayWsTicket(baseUrl, remoteHeaders) } catch (error) { + // For a Nous-managed Cloud agent, a 502/503/504 from the WS-ticket mint + // means the backend server itself is down — the actionable Cloud-down + // error. This boundary runs BEFORE the readiness loop, so without this + // the ticket wrapper below would swallow the server-fault classification + // and the renderer would never see isCloudBackendDown. Preserve the + // existing 401/403 reauth and generic transport behavior for everything + // else (#85335). + const cloudError = makeNousCloudBackendDownError(baseUrl, error) + + if (cloudError !== null) { + throw cloudError + } + throw gatewayTicketFailure( error, 'Your remote gateway session has expired. Open Settings → Gateway and click "Sign in" again.', @@ -10892,6 +10907,18 @@ async function startHermes() { const message = error instanceof Error ? error.message : String(error) const hostKeyChanged = isHostKeyChangedBootFailure(error) + // Carry structured Cloud-down metadata through the boot-progress / IPC + // boundary when present, so the renderer overlay can key on it rather than + // re-classifying the message string. main owns classification; the renderer + // only consumes the structured result (#85335). + const isCloudBackendDown = + Boolean(error && typeof error === 'object' && (error as any).isCloudBackendDown === true) + const statusCode = Number( + error && typeof error === 'object' && Number.isInteger((error as any).statusCode) + ? (error as any).statusCode + : NaN + ) + // Only latch LOCAL boot failures. A remote failure (lapsed session / mint // timeout / host briefly unreachable across sleep) is transient and has no // child 'exit' handler to clear the cache — latching it would wedge the app @@ -10920,6 +10947,7 @@ async function startHermes() { updateBootProgress( { error: message, + isCloudBackendDown: isCloudBackendDown || undefined, message: `Desktop boot failed: ${message}`, phase: 'backend.error', // Renderer contract for the self-heal loop (#82679): a transient @@ -10933,7 +10961,8 @@ async function startHermes() { isReauth: isReauthRequiredError(error), isHostKeyChanged: hostKeyChanged }), - running: false + running: false, + statusCode: Number.isInteger(statusCode) ? statusCode : undefined }, { allowDecrease: true } ) diff --git a/apps/desktop/src/global.d.ts b/apps/desktop/src/global.d.ts index f0038c367e..e754707e63 100644 --- a/apps/desktop/src/global.d.ts +++ b/apps/desktop/src/global.d.ts @@ -1000,6 +1000,8 @@ export interface DesktopCloudAgentSignInResult { export interface DesktopBootProgress { error: string | null fakeMode: boolean + /** True when the boot failure is a Nous Cloud agent that is down (HTTP 502/503/504). */ + isCloudBackendDown?: boolean message: string phase: string progress: number @@ -1011,6 +1013,8 @@ export interface DesktopBootProgress { */ retryable?: boolean running: boolean + /** Structured HTTP status when the boot failure carried one (e.g. 503). */ + statusCode?: number | null timestamp: number }