test(cron): retain native Bot Chat delivery reproduction

This commit is contained in:
teknium1
2026-09-14 15:37:53 -07:00
committed by Teknium
parent 5d8390d1a4
commit 6af84ead60
4 changed files with 175 additions and 0 deletions
+2
View File
@@ -701,6 +701,8 @@ def _deliver_to_bot_chat(job: dict, content: str, profile: str, *, deferred: boo
if pending is None and find_canonical_live_owner(home) is None and find_canonical_owner(home):
pending = defer(key, dict(job), content, profile, home)
if pending is not None:
if pending["content"] != content or pending["home"] != str(home):
raise ValueError("delivery id already belongs to a different payload")
status = pending["status"]
target = f"bot-chat:{profile_label}"
job.setdefault("_bot_chat_delivery_receipts", {})[target] = {
+42
View File
@@ -0,0 +1,42 @@
# Native Bot Mode delivery probe
Real Electron, production Python backend and tool execution, disposable HOME/HERMES_HOME,
loopback scripted inference (no paid model). Linux seat fixture; run from repository root:
```sh
npm ci --no-audit --no-fund
cp evals/botmode-dm-delivery/probe-dm-delivery.spec.ts apps/desktop/e2e/
git apply evals/botmode-dm-delivery/mock-trigger.patch
(cd apps/desktop && npm run build)
(cd apps/desktop && DISPLAY=:0 XAUTHORITY=/run/user/1000/xauth_cnpsqU \
XDG_RUNTIME_DIR=/run/user/1000 VIRTUAL_ENV="$VIRTUAL_ENV" \
HERMES_DESKTOP_CDP_PORT=off npx playwright test e2e/probe-dm-delivery.spec.ts --reporter=list)
git apply -R evals/botmode-dm-delivery/mock-trigger.patch
rm apps/desktop/e2e/probe-dm-delivery.spec.ts
```
Use the current seat's actual Xauthority path and an existing runtime venv.
Artifacts are retained in `/tmp/botmode-dm-recovery`; sandbox path is printed. The
fixture's generated hermes shim pins every child to this checkout, not an installed launcher.
## Verified results
Base `cf35e7351e770`: nested beta quiet CLI executes real `message_agent` to the named
alpha Desktop owner. Admission, execution and reply succeed, and the quiet sender
receives its completion. A reload then shows the attributed incoming message. The
live pre-reload view showed the reply but omitted the incoming row in this fixture;
that renderer-refresh behavior is not fixed by this cron change.
Base CLI-owner case: separate cron producer returns exact `SESSION_NOT_OWNED` refusal.
Release owner, run scheduler tick, open Beta in Desktop: no cron output, assertion red.
Fixed case: queued receipt; release owner; real synchronous scheduler tick drains;
Beta Desktop renders `CLI_OWNER_CRON_SENTINEL` and its reply once. Final two-case
native run: **2 passed (1.2m)**. Supported-owner nested case also passed independently
on base (**1 passed (40.0s)**).
This does not establish native Windows parity, general retry after unowned CLI
failure, or correction of the issue #105460 `--in ~` premise. CLI title resolution
is profile-DB-based, not workspace selection; the named live-owner route is positive.
The queue is deliberately at-most-once after claim. A crash before spawning but
after claiming remains inspectable as claimed; it is not retried automatically.
@@ -0,0 +1,26 @@
diff --git a/tests-js/scripts/mock-server.ts b/tests-js/scripts/mock-server.ts
index 6dffd2cf0f388..09c6de07d81b2 100644
--- a/tests-js/scripts/mock-server.ts
+++ b/tests-js/scripts/mock-server.ts
@@ -500,6 +500,21 @@ export function startMockServer(options: MockServerOptions = {}): Promise<MockSe
_receivedUserTexts.push(userText)
}
+ const dmMatch = userText.match(/E2E_DM\(([^)]+)\)\[([\s\S]*)\]/)
+ const lastUserIndex = messages.lastIndexOf(lastUserMsg)
+ if (dmMatch && !messages.slice(lastUserIndex + 1).some(m => m?.role === 'tool')) {
+ const turn: ScriptedTurn = {
+ text: '',
+ toolCalls: [{ name: 'message_agent', args: { target: dmMatch[1], message: dmMatch[2] } }],
+ }
+ if (stream) {
+ streamScriptedTurn(res, model, turn)
+ } else {
+ nonStreamingScriptedTurn(res, model, turn)
+ }
+ return
+ }
+
const isInterimTrigger = userText.includes('E2E_INTERIM_TRIGGER')
const isSidebarTrigger = userText.includes('E2E_SIDEBAR_TRIGGER')
const isSidebarCrossTrigger = userText.includes('E2E_SIDEBAR_CROSS')
@@ -0,0 +1,105 @@
import { execFileSync, spawn } from 'node:child_process'
import fs from 'node:fs'
import path from 'node:path'
import { buildAppEnv, createSandbox, launchDesktop, waitForAppReady, writeEnvFile, writeMockProviderConfig, type MockBackendFixture } from './fixtures'
import { MOCK_REPLY, startMockServer } from '../../../tests-js/scripts/mock-server'
import { expect, test } from './test'
const repo = path.resolve(import.meta.dirname, '../../..')
const python = path.join(process.env.VIRTUAL_ENV || path.join(repo, '.venv'), 'bin', 'python')
let fixture: MockBackendFixture
let env: Record<string, string>
const evidence = '/tmp/botmode-dm-recovery'
test.beforeAll(async () => {
fs.mkdirSync(evidence, { recursive: true })
const sandbox = createSandbox('dm-delivery')
const mock = await startMockServer({ holdFirstCompletionContaining: 'CLI_OWNER_HOLD' })
for (const name of ['default', 'alpha', 'beta']) {
const home = name === 'default' ? sandbox.hermesHome : path.join(sandbox.hermesHome, 'profiles', name)
fs.mkdirSync(home, { recursive: true })
writeMockProviderConfig(home, mock.url)
writeEnvFile(home)
fs.writeFileSync(path.join(home, 'SOUL.md'), `# ${name}\nA Bot Mode teammate.\n`)
fs.writeFileSync(path.join(home, 'profile.yaml'), 'name: ' + name + '\nui_meta:\n hermes-bots: {}\n')
}
const bin = path.join(sandbox.root, 'bin')
fs.mkdirSync(bin)
fs.writeFileSync(path.join(bin, 'hermes'), `#!/bin/sh\ncd ${repo}\nexec ${python} -m hermes_cli.main "$@"\n`, { mode: 0o755 })
env = buildAppEnv(sandbox, { HOME: sandbox.root, HERMES_DESKTOP_PYTHON: python,
HERMES_DESKTOP_HERMES: path.join(bin, 'hermes'), PATH: `${bin}:${process.env.PATH}`,
PYTHONPATH: repo, HERMES_SINGLE_QUERY_LINGER_SECONDS: '30' })
const { app, page } = await launchDesktop(env)
fixture = { app, page, sandbox, mock, mockUrl: mock.url, cleanup: async () => {
await app.close().catch(() => undefined)
await mock.close()
} }
console.log('SANDBOX', sandbox.root)
fs.writeFileSync(path.join(evidence, 'sandbox.txt'), sandbox.root)
await waitForAppReady(fixture, 120_000)
})
test.afterAll(async () => { await fixture?.cleanup() })
test('cron output waits for a CLI-only owner and arrives after owner release', async () => {
test.setTimeout(240_000)
const output = fs.openSync(path.join(evidence, 'cli-owner.log'), 'w')
const child = spawn(python, ['-m', 'hermes_cli.main', '-p', 'beta', 'chat', '--in', '~', '-c', 'Bot Chat', '--create-if-missing', '-Q', '-q', 'CLI_OWNER_HOLD'], { cwd: repo, env, stdio: ['ignore', output, output] })
const cronEnv = { ...env, HERMES_HOME: path.join(fixture.sandbox.hermesHome, 'profiles', 'beta') }
try {
await fixture.mock.waitForHeldCompletion()
const script = 'import json; from cron.scheduler_delivery import _deliver_to_bot_chat; j={"id":"cli-residual","name":"CLI residual","execution_id":"fixed-execution"}; result=_deliver_to_bot_chat(j,"CLI_OWNER_CRON_SENTINEL",""); print(json.dumps({"result":result,"job":j}))'
const result = JSON.parse(execFileSync(python, ['-c', script], { env: cronEnv, cwd: repo, encoding: 'utf8', timeout: 30_000 }))
console.log('CLI_OWNER_CRON_ADMISSION', JSON.stringify(result))
fs.writeFileSync(path.join(evidence, 'cli-owner-admission.json'), JSON.stringify(result, null, 2))
fixture.mock.releaseHeldStream()
await expect.poll(() => child.exitCode, { timeout: 60_000 }).toBe(0)
const ticker = spawn(python, ['-c', 'import time; from cron.scheduler import tick; from cron.bot_chat_delivery import _running; tick(verbose=False);\nwhile _running: time.sleep(0.1)'], { env: cronEnv, cwd: repo, stdio: ['ignore', output, output] })
await expect.poll(() => ticker.exitCode, { timeout: 90_000 }).toBe(0)
await openBot('beta')
expect(dbMessages('beta').filter(([role, text]) => role === 'user' && text.includes('CLI_OWNER_CRON_SENTINEL'))).toHaveLength(1)
await expect(fixture.page.getByText(/CLI_OWNER_CRON_SENTINEL/).filter({ visible: true }).first()).toBeVisible({ timeout: 45_000 })
await fixture.page.screenshot({ path: path.join(evidence, 'cli-owner-after-release.png') })
} finally { fixture.mock.releaseHeldStream(); child.kill(); fs.closeSync(output) }
})
async function openBot(name: string) {
const page = fixture.page
await page.getByRole('button', { name: 'Bots', exact: true }).or(page.getByRole('tab', { name: 'Bots', exact: true })).first().click()
const row = page.getByRole('button', { name: new RegExp(`^${name}\\b`, 'i') }).filter({ visible: true }).first()
await expect(row).toBeVisible({ timeout: 30_000 })
await row.click()
const composer = page.locator('[data-slot="composer-root"] [contenteditable="true"]').filter({ visible: true }).first()
await expect(composer).toBeVisible({ timeout: 120_000 })
return composer
}
function dbMessages(name: string) {
const home = name === 'default' ? fixture.sandbox.hermesHome : path.join(fixture.sandbox.hermesHome, 'profiles', name)
return JSON.parse(execFileSync(python, ['-c', 'import sqlite3,json,sys; c=sqlite3.connect(sys.argv[1]); print(json.dumps(c.execute("select role,content from messages").fetchall()))', path.join(home, 'state.db')], { env, cwd: repo, encoding: 'utf8' })) as string[][]
}
test('named Bot Chat receives a nested one-shot message_agent delivery once', async () => {
test.setTimeout(300_000)
const page = fixture.page
const composer = await openBot('alpha')
await expect(page.getByText('Say something to get started.').filter({ visible: true })).toBeVisible({ timeout: 120_000 })
await composer.fill('initialize alpha owner')
await page.keyboard.press('Enter')
await expect(page.getByText(MOCK_REPLY).filter({ visible: true }).first()).toBeVisible({ timeout: 60_000 })
const output = fs.openSync(path.join(evidence, 'oneshot.log'), 'w')
const child = spawn(python, ['-m', 'hermes_cli.main', '-p', 'beta', 'chat', '--in', '~', '-c', 'Bot Chat', '--create-if-missing', '-Q', '-q', 'E2E_DM(alpha)[nested-one-shot-sentinel]'], { cwd: repo, env, stdio: ['ignore', output, output] })
try {
await expect.poll(() => dbMessages('alpha').filter(([role, text]) => role === 'user' && text.includes('nested-one-shot-sentinel')).length, { timeout: 120_000 }).toBe(1)
console.log('ALPHA_ROWS', JSON.stringify(dbMessages('alpha')))
console.log('BETA_ROWS', JSON.stringify(dbMessages('beta')))
await page.reload()
await waitForAppReady(fixture, 120_000)
await openBot('alpha')
await expect(page.getByText('show message', { exact: true }).first()).toBeVisible({ timeout: 30_000 })
for (const toggle of await page.getByText('show message', { exact: true }).all()) await toggle.click()
await expect(page.getByText(/nested-one-shot-sentinel/).filter({ visible: true }).first()).toBeVisible({ timeout: 30_000 })
await page.screenshot({ path: path.join(evidence, 'nested-delivery-desktop.png') })
fs.writeFileSync(path.join(evidence, 'rows.json'), JSON.stringify({ alpha: dbMessages('alpha'), beta: dbMessages('beta'), prompts: fixture.mock.receivedPrompts }, null, 2))
} finally { child.kill(); fs.closeSync(output) }
})