fix(desktop): make pool slot timeouts actionable
This commit is contained in:
@@ -114,7 +114,7 @@ test('100 real child processes never exceed twelve simultaneous local slots', as
|
||||
const limit = 12
|
||||
const coordinator = new LocalBackendSpawnCoordinator(limit)
|
||||
const livePids = new Set<number>()
|
||||
const seenPids = new Set<number>()
|
||||
let completedChildren = 0
|
||||
let maxLive = 0
|
||||
|
||||
await Promise.all(
|
||||
@@ -128,7 +128,6 @@ test('100 real child processes never exceed twelve simultaneous local slots', as
|
||||
|
||||
assert.ok(child.pid)
|
||||
livePids.add(child.pid)
|
||||
seenPids.add(child.pid)
|
||||
maxLive = Math.max(maxLive, livePids.size)
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
@@ -142,6 +141,7 @@ test('100 real child processes never exceed twelve simultaneous local slots', as
|
||||
})
|
||||
})
|
||||
|
||||
completedChildren += 1
|
||||
livePids.delete(child.pid)
|
||||
} finally {
|
||||
release()
|
||||
@@ -149,7 +149,9 @@ test('100 real child processes never exceed twelve simultaneous local slots', as
|
||||
})
|
||||
)
|
||||
|
||||
assert.equal(seenPids.size, 100)
|
||||
// Windows may recycle a PID after a short-lived child exits; completion
|
||||
// count, not PID uniqueness, is the invariant this concurrency test owns.
|
||||
assert.equal(completedChildren, 100)
|
||||
assert.equal(maxLive, limit)
|
||||
assert.equal(livePids.size, 0)
|
||||
assert.equal(coordinator.activeCount, 0)
|
||||
|
||||
@@ -1,6 +1,15 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { findStoredIdForRuntimeId, resolveRoutingSessionId, resolveSessionRpcOwner } from './wiring-routing'
|
||||
import {
|
||||
findStoredIdForRuntimeId,
|
||||
POOL_LIMITS_SETTINGS_ROUTE,
|
||||
resolveRoutingSessionId,
|
||||
resolveSessionRpcOwner
|
||||
} from './wiring-routing'
|
||||
|
||||
it('targets the Advanced settings surface for pool-cap recovery', () => {
|
||||
expect(POOL_LIMITS_SETTINGS_ROUTE).toBe('/settings?tab=config:advanced')
|
||||
})
|
||||
|
||||
describe('findStoredIdForRuntimeId', () => {
|
||||
it('reverse-resolves a runtime id to its stored id', () => {
|
||||
|
||||
@@ -7,6 +7,11 @@
|
||||
|
||||
import type { SessionOwnerRoute } from '@/store/session-request-router'
|
||||
|
||||
import { SETTINGS_ROUTE } from '../routes'
|
||||
|
||||
/** The in-app destination for pool-cap recovery. */
|
||||
export const POOL_LIMITS_SETTINGS_ROUTE = `${SETTINGS_ROUTE}?tab=config:advanced`
|
||||
|
||||
/**
|
||||
* Resolve a runtime session id back to its stored id by reverse-scanning the
|
||||
* stored->runtime binding map — the same ladder use-session-tile-delegate's
|
||||
|
||||
@@ -51,6 +51,7 @@ import { $cronReviewRequest, setCronFocusJobId } from '@/store/cron'
|
||||
import { requestGatewayForProfile } from '@/store/gateway'
|
||||
import { $pinnedSessionIds, pinSession, restoreWorktree, unpinSession } from '@/store/layout'
|
||||
import { notifyError } from '@/store/notifications'
|
||||
import { $poolLimitsSettingsRequest } from '@/store/pool-limits'
|
||||
import { $previewTarget } from '@/store/preview'
|
||||
import {
|
||||
$activeGatewayProfile,
|
||||
@@ -163,6 +164,7 @@ import { $restartPreviewServer, useTitlebarToolContributions } from './panes'
|
||||
import { type AmbientGatewayRequest, createSessionRpcDispatcher } from './session-rpc-dispatcher'
|
||||
import { ChatRoutesSurface, SidebarSurface, StatusbarSurface, TerminalSurface } from './surfaces'
|
||||
import type { WiringActions, WiringApi } from './types'
|
||||
import { POOL_LIMITS_SETTINGS_ROUTE } from './wiring-routing'
|
||||
|
||||
// Overlay views the controller mounts over the shell — lazy, load on demand.
|
||||
// The workspace-route full-page views (skills/messaging/artifacts) are the
|
||||
@@ -194,6 +196,7 @@ export function ContribWiring({ children }: { children: ReactNode }) {
|
||||
// context (the sticky toast). The shell owns `navigate`, so it consumes the
|
||||
// intent counter here; the ref skips the initial mount value.
|
||||
const billingSettingsSeenRef = useRef(0)
|
||||
const poolLimitsSettingsSeenRef = useRef(0)
|
||||
const cronReviewSeenRef = useRef(0)
|
||||
const activeTranscriptSignatureRef = useRef(new Map<string, string>())
|
||||
const activeTranscriptRequestSequenceRef = useRef(0)
|
||||
@@ -204,6 +207,7 @@ export function ContribWiring({ children }: { children: ReactNode }) {
|
||||
const gatewayState = useStore($gatewayState)
|
||||
const activeSessionId = useStore($activeSessionId)
|
||||
const billingSettingsRequest = useStore($billingSettingsRequest)
|
||||
const poolLimitsSettingsRequest = useStore($poolLimitsSettingsRequest)
|
||||
const cronReviewRequest = useStore($cronReviewRequest)
|
||||
const currentCwd = useStore($currentCwd)
|
||||
|
||||
@@ -220,6 +224,22 @@ export function ContribWiring({ children }: { children: ReactNode }) {
|
||||
}
|
||||
}, [billingSettingsRequest, navigate])
|
||||
|
||||
// Pool-cap recovery is fired by the notification action, which has no router
|
||||
// context. Keep navigation user-initiated: the counter changes only when the
|
||||
// user clicks "Open Advanced Settings" on a pool-slot failure.
|
||||
// eslint-disable-next-line no-restricted-syntax -- one-shot request-seen sentinel, not an atom mirror
|
||||
useEffect(() => {
|
||||
if (poolLimitsSettingsRequest === poolLimitsSettingsSeenRef.current) {
|
||||
return
|
||||
}
|
||||
|
||||
poolLimitsSettingsSeenRef.current = poolLimitsSettingsRequest
|
||||
|
||||
if (poolLimitsSettingsRequest > 0) {
|
||||
navigate(POOL_LIMITS_SETTINGS_ROUTE)
|
||||
}
|
||||
}, [navigate, poolLimitsSettingsRequest])
|
||||
|
||||
// eslint-disable-next-line no-restricted-syntax -- one-shot request-seen sentinel, not an atom mirror
|
||||
useEffect(() => {
|
||||
if (cronReviewRequest === cronReviewSeenRef.current) {
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import { cleanup, render, screen } from '@testing-library/react'
|
||||
import { cleanup, fireEvent, render, screen } from '@testing-library/react'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
|
||||
import { I18nProvider } from '@/i18n'
|
||||
import { clearNotifications, notify } from '@/store/notifications'
|
||||
import { $notifications, clearNotifications, notify, notifyError } from '@/store/notifications'
|
||||
import { $poolLimitsSettingsRequest } from '@/store/pool-limits'
|
||||
|
||||
import { NotificationStack, toastTitleClassName } from './notifications'
|
||||
|
||||
@@ -12,11 +13,13 @@ const DETAIL = 'target user message is no longer in session history'
|
||||
describe('toast titles', () => {
|
||||
beforeEach(() => {
|
||||
clearNotifications()
|
||||
$poolLimitsSettingsRequest.set(0)
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
cleanup()
|
||||
clearNotifications()
|
||||
$poolLimitsSettingsRequest.set(0)
|
||||
})
|
||||
|
||||
it('drops the one-line clamp so a long error title can wrap', () => {
|
||||
@@ -47,4 +50,38 @@ describe('toast titles', () => {
|
||||
expect(title.className).toMatch(/\boverflow-y-auto\b/)
|
||||
expect(screen.getByText(DETAIL)).toBeTruthy()
|
||||
})
|
||||
|
||||
it('makes a local pool-slot timeout actionable without changing ordinary errors', () => {
|
||||
notifyError(
|
||||
new Error(
|
||||
`Error invoking remote method 'hermes:connection': Error: Local backend start for "research" timed out while waiting for a free slot.`
|
||||
),
|
||||
'Failed to switch to profile "research"'
|
||||
)
|
||||
|
||||
render(
|
||||
<I18nProvider configClient={null} initialLocale="en">
|
||||
<NotificationStack />
|
||||
</I18nProvider>
|
||||
)
|
||||
|
||||
expect(screen.getByText(/All local profile backend slots are busy/)).toBeTruthy()
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Open Advanced Settings' }))
|
||||
|
||||
expect($poolLimitsSettingsRequest.get()).toBe(1)
|
||||
expect($notifications.get()).toHaveLength(0)
|
||||
|
||||
notifyError(new Error('gateway unavailable'), 'Failed to switch profile')
|
||||
expect($notifications.get()[0]?.action).toBeUndefined()
|
||||
})
|
||||
|
||||
it('keeps background pool-slot timeouts quiet if they reach the renderer', () => {
|
||||
notifyError(
|
||||
new Error('Local backend start for "background" timed out while waiting for a free slot. (background)'),
|
||||
'Background profile warm-up failed'
|
||||
)
|
||||
|
||||
expect($notifications.get()[0]?.action).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -3273,6 +3273,9 @@ export const ar = defineLocale({
|
||||
resumeStrandedTitle: 'تعذّر تحميل هذه الجلسة',
|
||||
resumeStrandedBody:
|
||||
'فشل الاتصال بهذه الجلسة وتوقفت إعادة المحاولة التلقائية. تأكد من تشغيل البوابة، ثم حاول مجددا.',
|
||||
poolSlotTimeoutBody:
|
||||
'جميع خانات الواجهات الخلفية المحلية للملفات الشخصية مشغولة. زد عدد Warm Bot Backends من الإعدادات ← متقدم، أو أعد المحاولة بعد إزالة واجهة خلفية خاملة.',
|
||||
poolSlotTimeoutOpenSettings: 'فتح الإعدادات المتقدمة',
|
||||
resumeRetry: 'إعادة المحاولة',
|
||||
nothingToBranch: 'لا يوجد ما يمكن تفريعه',
|
||||
branchNeedsChat: 'يحتاج التفريع إلى محادثة',
|
||||
|
||||
@@ -4240,6 +4240,9 @@ export const en: Translations = {
|
||||
resumeStrandedTitle: "Couldn't load this session",
|
||||
resumeStrandedBody:
|
||||
'The connection to this session failed and automatic retries gave up. Check that the gateway is running, then try again.',
|
||||
poolSlotTimeoutBody:
|
||||
'All local profile backend slots are busy. Increase Warm Bot Backends in Settings → Advanced, or retry after an idle backend is evicted.',
|
||||
poolSlotTimeoutOpenSettings: 'Open Advanced Settings',
|
||||
resumeRetry: 'Retry',
|
||||
nothingToBranch: 'Nothing to branch',
|
||||
branchNeedsChat: 'Start or resume a chat before branching.',
|
||||
|
||||
@@ -3710,6 +3710,9 @@ export const ja = defineLocale({
|
||||
resumeStrandedTitle: 'このセッションを読み込めませんでした',
|
||||
resumeStrandedBody:
|
||||
'このセッションへの接続に失敗し、自動再試行も停止しました。ゲートウェイが実行中か確認してから、もう一度お試しください。',
|
||||
poolSlotTimeoutBody:
|
||||
'すべてのローカルプロファイルバックエンドスロットが使用中です。「設定」→「詳細設定」で「Warm Bot Backends」を増やすか、アイドル状態のバックエンドが解放された後に再試行してください。',
|
||||
poolSlotTimeoutOpenSettings: '詳細設定を開く',
|
||||
resumeRetry: '再試行',
|
||||
nothingToBranch: 'ブランチするものがありません',
|
||||
branchNeedsChat: 'ブランチする前にチャットを開始または再開してください。',
|
||||
|
||||
@@ -3825,6 +3825,9 @@ export const ru = defineLocale({
|
||||
resumeStrandedTitle: 'Не удалось загрузить этот сеанс',
|
||||
resumeStrandedBody:
|
||||
'Соединение с этим сеансом оборвалось, и автоматические повторные попытки исчерпаны. Проверьте, что шлюз работает, и попробуйте снова.',
|
||||
poolSlotTimeoutBody:
|
||||
'Все слоты локальных бэкендов профилей заняты. Увеличьте Warm Bot Backends в «Настройки» → «Дополнительно» или повторите попытку после освобождения неактивного бэкенда.',
|
||||
poolSlotTimeoutOpenSettings: 'Открыть расширенные настройки',
|
||||
resumeRetry: 'Повторить',
|
||||
nothingToBranch: 'Нечего ветвить',
|
||||
branchNeedsChat: 'Начните или возобновите чат перед ветвлением.',
|
||||
|
||||
@@ -3700,6 +3700,8 @@ export interface Translations {
|
||||
readOnlyTranscriptSendBlocked: string
|
||||
resumeStrandedTitle: string
|
||||
resumeStrandedBody: string
|
||||
poolSlotTimeoutBody: string
|
||||
poolSlotTimeoutOpenSettings: string
|
||||
resumeRetry: string
|
||||
nothingToBranch: string
|
||||
branchNeedsChat: string
|
||||
|
||||
@@ -3555,6 +3555,9 @@ export const zhHant = defineLocale({
|
||||
readOnlyTranscriptSendBlocked: '此對話目前以唯讀逐字稿方式開啟——傳送已停用。',
|
||||
resumeStrandedTitle: '無法載入此工作階段',
|
||||
resumeStrandedBody: '與此工作階段的連線失敗,自動重試已停止。請確認閘道正在執行,然後重試。',
|
||||
poolSlotTimeoutBody:
|
||||
'所有本機設定檔後端插槽目前都在使用中。請在「設定」→「進階」中增加 Warm Bot Backends,或等待閒置後端被移除後重試。',
|
||||
poolSlotTimeoutOpenSettings: '開啟進階設定',
|
||||
resumeRetry: '重試',
|
||||
nothingToBranch: '沒有可分支的內容',
|
||||
branchNeedsChat: '分支前請先開始或繼續一個聊天。',
|
||||
|
||||
@@ -4349,6 +4349,9 @@ export const zh: Translations = {
|
||||
readOnlyTranscriptSendBlocked: '该会话目前以只读记录方式打开——发送已禁用。',
|
||||
resumeStrandedTitle: '无法加载此会话',
|
||||
resumeStrandedBody: '与此会话的连接失败,自动重试已停止。请确认网关正在运行,然后重试。',
|
||||
poolSlotTimeoutBody:
|
||||
'所有本地配置后端槽位都在使用中。请在“设置”→“高级”中增加 Warm Bot Backends,或等待空闲后端被驱逐后重试。',
|
||||
poolSlotTimeoutOpenSettings: '打开高级设置',
|
||||
resumeRetry: '重试',
|
||||
nothingToBranch: '没有可分支的内容',
|
||||
branchNeedsChat: '分支前请先开始或恢复一个对话。',
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { atom } from 'nanostores'
|
||||
|
||||
import { translateNow } from '@/i18n'
|
||||
import { isLocalBackendSlotWaitTimeout, requestPoolLimitsSettings } from '@/store/pool-limits'
|
||||
|
||||
export type NotificationKind = 'error' | 'warning' | 'info' | 'success'
|
||||
|
||||
@@ -195,12 +196,19 @@ export function notify(input: NotificationInput): string {
|
||||
|
||||
export function notifyError(error: unknown, fallback: string): string {
|
||||
const readable = readableError(error, fallback)
|
||||
const poolSlotTimeout = isLocalBackendSlotWaitTimeout(error)
|
||||
|
||||
return notify({
|
||||
action: poolSlotTimeout
|
||||
? {
|
||||
label: translateNow('desktop.poolSlotTimeoutOpenSettings'),
|
||||
onClick: requestPoolLimitsSettings
|
||||
}
|
||||
: undefined,
|
||||
kind: 'error',
|
||||
title: fallback,
|
||||
message: readable.message,
|
||||
detail: readable.detail
|
||||
message: poolSlotTimeout ? translateNow('desktop.poolSlotTimeoutBody') : readable.message,
|
||||
detail: poolSlotTimeout ? readable.message : readable.detail
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -26,6 +26,30 @@ export const POOL_LIMITS_DEFAULTS: PoolLimits = {
|
||||
|
||||
export const $poolLimits = atom<PoolLimits>({ ...POOL_LIMITS_DEFAULTS })
|
||||
|
||||
// Electron IPC rejects with a plain Error in the renderer, so the stable
|
||||
// coordinator phrase is the cross-process discriminator. Keep it narrow: other
|
||||
// backend failures must retain their existing recovery path.
|
||||
const LOCAL_BACKEND_SLOT_TIMEOUT_PREFIX = 'Local backend start for "'
|
||||
const LOCAL_BACKEND_SLOT_TIMEOUT_SUFFIX = ' timed out while waiting for a free slot.'
|
||||
const LOCAL_BACKEND_SLOT_TIMEOUT_BACKGROUND_SUFFIX = ' (background)'
|
||||
|
||||
export function isLocalBackendSlotWaitTimeout(error: unknown): boolean {
|
||||
const message = error instanceof Error ? error.message : typeof error === 'string' ? error : ''
|
||||
|
||||
return (
|
||||
message.includes(LOCAL_BACKEND_SLOT_TIMEOUT_PREFIX) &&
|
||||
message.includes(LOCAL_BACKEND_SLOT_TIMEOUT_SUFFIX) &&
|
||||
!message.endsWith(LOCAL_BACKEND_SLOT_TIMEOUT_BACKGROUND_SUFFIX)
|
||||
)
|
||||
}
|
||||
|
||||
/** Navigation intent consumed by the shell, which owns React Router. */
|
||||
export const $poolLimitsSettingsRequest = atom(0)
|
||||
|
||||
export function requestPoolLimitsSettings(): void {
|
||||
$poolLimitsSettingsRequest.set($poolLimitsSettingsRequest.get() + 1)
|
||||
}
|
||||
|
||||
/** Seed from main's authoritative state once at startup; no-op without the
|
||||
* bridge (web/older builds just keep the defaults for the UI). */
|
||||
export async function loadPoolLimits(): Promise<void> {
|
||||
|
||||
Reference in New Issue
Block a user