From 6af84ead606b3bd2ac59f39de97d9ee69d5f8222 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Mon, 14 Sep 2026 15:37:53 -0700 Subject: [PATCH] test(cron): retain native Bot Chat delivery reproduction --- cron/scheduler_delivery.py | 2 + evals/botmode-dm-delivery/README.md | 42 +++++++ evals/botmode-dm-delivery/mock-trigger.patch | 26 +++++ .../probe-dm-delivery.spec.ts | 105 ++++++++++++++++++ 4 files changed, 175 insertions(+) create mode 100644 evals/botmode-dm-delivery/README.md create mode 100644 evals/botmode-dm-delivery/mock-trigger.patch create mode 100644 evals/botmode-dm-delivery/probe-dm-delivery.spec.ts diff --git a/cron/scheduler_delivery.py b/cron/scheduler_delivery.py index c419f8eaa3..0d9d015675 100644 --- a/cron/scheduler_delivery.py +++ b/cron/scheduler_delivery.py @@ -701,6 +701,8 @@ def _deliver_to_bot_chat(job: dict, content: str, profile: str, *, deferred: boo if pending is None and find_canonical_live_owner(home) is None and find_canonical_owner(home): pending = defer(key, dict(job), content, profile, home) if pending is not None: + if pending["content"] != content or pending["home"] != str(home): + raise ValueError("delivery id already belongs to a different payload") status = pending["status"] target = f"bot-chat:{profile_label}" job.setdefault("_bot_chat_delivery_receipts", {})[target] = { diff --git a/evals/botmode-dm-delivery/README.md b/evals/botmode-dm-delivery/README.md new file mode 100644 index 0000000000..3d6cfeda9a --- /dev/null +++ b/evals/botmode-dm-delivery/README.md @@ -0,0 +1,42 @@ +# Native Bot Mode delivery probe + +Real Electron, production Python backend and tool execution, disposable HOME/HERMES_HOME, +loopback scripted inference (no paid model). Linux seat fixture; run from repository root: + +```sh +npm ci --no-audit --no-fund +cp evals/botmode-dm-delivery/probe-dm-delivery.spec.ts apps/desktop/e2e/ +git apply evals/botmode-dm-delivery/mock-trigger.patch +(cd apps/desktop && npm run build) +(cd apps/desktop && DISPLAY=:0 XAUTHORITY=/run/user/1000/xauth_cnpsqU \ + XDG_RUNTIME_DIR=/run/user/1000 VIRTUAL_ENV="$VIRTUAL_ENV" \ + HERMES_DESKTOP_CDP_PORT=off npx playwright test e2e/probe-dm-delivery.spec.ts --reporter=list) +git apply -R evals/botmode-dm-delivery/mock-trigger.patch +rm apps/desktop/e2e/probe-dm-delivery.spec.ts +``` + +Use the current seat's actual Xauthority path and an existing runtime venv. +Artifacts are retained in `/tmp/botmode-dm-recovery`; sandbox path is printed. The +fixture's generated hermes shim pins every child to this checkout, not an installed launcher. + +## Verified results + +Base `cf35e7351e770`: nested beta quiet CLI executes real `message_agent` to the named +alpha Desktop owner. Admission, execution and reply succeed, and the quiet sender +receives its completion. A reload then shows the attributed incoming message. The +live pre-reload view showed the reply but omitted the incoming row in this fixture; +that renderer-refresh behavior is not fixed by this cron change. + +Base CLI-owner case: separate cron producer returns exact `SESSION_NOT_OWNED` refusal. +Release owner, run scheduler tick, open Beta in Desktop: no cron output, assertion red. +Fixed case: queued receipt; release owner; real synchronous scheduler tick drains; +Beta Desktop renders `CLI_OWNER_CRON_SENTINEL` and its reply once. Final two-case +native run: **2 passed (1.2m)**. Supported-owner nested case also passed independently +on base (**1 passed (40.0s)**). + +This does not establish native Windows parity, general retry after unowned CLI +failure, or correction of the issue #105460 `--in ~` premise. CLI title resolution +is profile-DB-based, not workspace selection; the named live-owner route is positive. + +The queue is deliberately at-most-once after claim. A crash before spawning but +after claiming remains inspectable as claimed; it is not retried automatically. diff --git a/evals/botmode-dm-delivery/mock-trigger.patch b/evals/botmode-dm-delivery/mock-trigger.patch new file mode 100644 index 0000000000..bae97952c9 --- /dev/null +++ b/evals/botmode-dm-delivery/mock-trigger.patch @@ -0,0 +1,26 @@ +diff --git a/tests-js/scripts/mock-server.ts b/tests-js/scripts/mock-server.ts +index 6dffd2cf0f388..09c6de07d81b2 100644 +--- a/tests-js/scripts/mock-server.ts ++++ b/tests-js/scripts/mock-server.ts +@@ -500,6 +500,21 @@ export function startMockServer(options: MockServerOptions = {}): Promise m?.role === 'tool')) { ++ const turn: ScriptedTurn = { ++ text: '', ++ toolCalls: [{ name: 'message_agent', args: { target: dmMatch[1], message: dmMatch[2] } }], ++ } ++ if (stream) { ++ streamScriptedTurn(res, model, turn) ++ } else { ++ nonStreamingScriptedTurn(res, model, turn) ++ } ++ return ++ } ++ + const isInterimTrigger = userText.includes('E2E_INTERIM_TRIGGER') + const isSidebarTrigger = userText.includes('E2E_SIDEBAR_TRIGGER') + const isSidebarCrossTrigger = userText.includes('E2E_SIDEBAR_CROSS') diff --git a/evals/botmode-dm-delivery/probe-dm-delivery.spec.ts b/evals/botmode-dm-delivery/probe-dm-delivery.spec.ts new file mode 100644 index 0000000000..49c095433b --- /dev/null +++ b/evals/botmode-dm-delivery/probe-dm-delivery.spec.ts @@ -0,0 +1,105 @@ +import { execFileSync, spawn } from 'node:child_process' +import fs from 'node:fs' +import path from 'node:path' +import { buildAppEnv, createSandbox, launchDesktop, waitForAppReady, writeEnvFile, writeMockProviderConfig, type MockBackendFixture } from './fixtures' +import { MOCK_REPLY, startMockServer } from '../../../tests-js/scripts/mock-server' +import { expect, test } from './test' + +const repo = path.resolve(import.meta.dirname, '../../..') +const python = path.join(process.env.VIRTUAL_ENV || path.join(repo, '.venv'), 'bin', 'python') +let fixture: MockBackendFixture +let env: Record +const evidence = '/tmp/botmode-dm-recovery' + +test.beforeAll(async () => { + fs.mkdirSync(evidence, { recursive: true }) + const sandbox = createSandbox('dm-delivery') + const mock = await startMockServer({ holdFirstCompletionContaining: 'CLI_OWNER_HOLD' }) + for (const name of ['default', 'alpha', 'beta']) { + const home = name === 'default' ? sandbox.hermesHome : path.join(sandbox.hermesHome, 'profiles', name) + fs.mkdirSync(home, { recursive: true }) + writeMockProviderConfig(home, mock.url) + writeEnvFile(home) + fs.writeFileSync(path.join(home, 'SOUL.md'), `# ${name}\nA Bot Mode teammate.\n`) + fs.writeFileSync(path.join(home, 'profile.yaml'), 'name: ' + name + '\nui_meta:\n hermes-bots: {}\n') + } + const bin = path.join(sandbox.root, 'bin') + fs.mkdirSync(bin) + fs.writeFileSync(path.join(bin, 'hermes'), `#!/bin/sh\ncd ${repo}\nexec ${python} -m hermes_cli.main "$@"\n`, { mode: 0o755 }) + env = buildAppEnv(sandbox, { HOME: sandbox.root, HERMES_DESKTOP_PYTHON: python, + HERMES_DESKTOP_HERMES: path.join(bin, 'hermes'), PATH: `${bin}:${process.env.PATH}`, + PYTHONPATH: repo, HERMES_SINGLE_QUERY_LINGER_SECONDS: '30' }) + const { app, page } = await launchDesktop(env) + fixture = { app, page, sandbox, mock, mockUrl: mock.url, cleanup: async () => { + await app.close().catch(() => undefined) + await mock.close() + } } + console.log('SANDBOX', sandbox.root) + fs.writeFileSync(path.join(evidence, 'sandbox.txt'), sandbox.root) + await waitForAppReady(fixture, 120_000) +}) + +test.afterAll(async () => { await fixture?.cleanup() }) + +test('cron output waits for a CLI-only owner and arrives after owner release', async () => { + test.setTimeout(240_000) + const output = fs.openSync(path.join(evidence, 'cli-owner.log'), 'w') + const child = spawn(python, ['-m', 'hermes_cli.main', '-p', 'beta', 'chat', '--in', '~', '-c', 'Bot Chat', '--create-if-missing', '-Q', '-q', 'CLI_OWNER_HOLD'], { cwd: repo, env, stdio: ['ignore', output, output] }) + const cronEnv = { ...env, HERMES_HOME: path.join(fixture.sandbox.hermesHome, 'profiles', 'beta') } + try { + await fixture.mock.waitForHeldCompletion() + const script = 'import json; from cron.scheduler_delivery import _deliver_to_bot_chat; j={"id":"cli-residual","name":"CLI residual","execution_id":"fixed-execution"}; result=_deliver_to_bot_chat(j,"CLI_OWNER_CRON_SENTINEL",""); print(json.dumps({"result":result,"job":j}))' + const result = JSON.parse(execFileSync(python, ['-c', script], { env: cronEnv, cwd: repo, encoding: 'utf8', timeout: 30_000 })) + console.log('CLI_OWNER_CRON_ADMISSION', JSON.stringify(result)) + fs.writeFileSync(path.join(evidence, 'cli-owner-admission.json'), JSON.stringify(result, null, 2)) + fixture.mock.releaseHeldStream() + await expect.poll(() => child.exitCode, { timeout: 60_000 }).toBe(0) + const ticker = spawn(python, ['-c', 'import time; from cron.scheduler import tick; from cron.bot_chat_delivery import _running; tick(verbose=False);\nwhile _running: time.sleep(0.1)'], { env: cronEnv, cwd: repo, stdio: ['ignore', output, output] }) + await expect.poll(() => ticker.exitCode, { timeout: 90_000 }).toBe(0) + await openBot('beta') + expect(dbMessages('beta').filter(([role, text]) => role === 'user' && text.includes('CLI_OWNER_CRON_SENTINEL'))).toHaveLength(1) + await expect(fixture.page.getByText(/CLI_OWNER_CRON_SENTINEL/).filter({ visible: true }).first()).toBeVisible({ timeout: 45_000 }) + await fixture.page.screenshot({ path: path.join(evidence, 'cli-owner-after-release.png') }) + } finally { fixture.mock.releaseHeldStream(); child.kill(); fs.closeSync(output) } +}) + +async function openBot(name: string) { + const page = fixture.page + await page.getByRole('button', { name: 'Bots', exact: true }).or(page.getByRole('tab', { name: 'Bots', exact: true })).first().click() + const row = page.getByRole('button', { name: new RegExp(`^${name}\\b`, 'i') }).filter({ visible: true }).first() + await expect(row).toBeVisible({ timeout: 30_000 }) + await row.click() + const composer = page.locator('[data-slot="composer-root"] [contenteditable="true"]').filter({ visible: true }).first() + await expect(composer).toBeVisible({ timeout: 120_000 }) + return composer +} + +function dbMessages(name: string) { + const home = name === 'default' ? fixture.sandbox.hermesHome : path.join(fixture.sandbox.hermesHome, 'profiles', name) + return JSON.parse(execFileSync(python, ['-c', 'import sqlite3,json,sys; c=sqlite3.connect(sys.argv[1]); print(json.dumps(c.execute("select role,content from messages").fetchall()))', path.join(home, 'state.db')], { env, cwd: repo, encoding: 'utf8' })) as string[][] +} + +test('named Bot Chat receives a nested one-shot message_agent delivery once', async () => { + test.setTimeout(300_000) + const page = fixture.page + const composer = await openBot('alpha') + await expect(page.getByText('Say something to get started.').filter({ visible: true })).toBeVisible({ timeout: 120_000 }) + await composer.fill('initialize alpha owner') + await page.keyboard.press('Enter') + await expect(page.getByText(MOCK_REPLY).filter({ visible: true }).first()).toBeVisible({ timeout: 60_000 }) + const output = fs.openSync(path.join(evidence, 'oneshot.log'), 'w') + const child = spawn(python, ['-m', 'hermes_cli.main', '-p', 'beta', 'chat', '--in', '~', '-c', 'Bot Chat', '--create-if-missing', '-Q', '-q', 'E2E_DM(alpha)[nested-one-shot-sentinel]'], { cwd: repo, env, stdio: ['ignore', output, output] }) + try { + await expect.poll(() => dbMessages('alpha').filter(([role, text]) => role === 'user' && text.includes('nested-one-shot-sentinel')).length, { timeout: 120_000 }).toBe(1) + console.log('ALPHA_ROWS', JSON.stringify(dbMessages('alpha'))) + console.log('BETA_ROWS', JSON.stringify(dbMessages('beta'))) + await page.reload() + await waitForAppReady(fixture, 120_000) + await openBot('alpha') + await expect(page.getByText('show message', { exact: true }).first()).toBeVisible({ timeout: 30_000 }) + for (const toggle of await page.getByText('show message', { exact: true }).all()) await toggle.click() + await expect(page.getByText(/nested-one-shot-sentinel/).filter({ visible: true }).first()).toBeVisible({ timeout: 30_000 }) + await page.screenshot({ path: path.join(evidence, 'nested-delivery-desktop.png') }) + fs.writeFileSync(path.join(evidence, 'rows.json'), JSON.stringify({ alpha: dbMessages('alpha'), beta: dbMessages('beta'), prompts: fixture.mock.receivedPrompts }, null, 2)) + } finally { child.kill(); fs.closeSync(output) } +})