diff --git a/apps/desktop/src/app/chat/right-rail/preview-act.test.ts b/apps/desktop/src/app/chat/right-rail/preview-act.test.ts index b53e046342..2b68b1eb99 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-act.test.ts +++ b/apps/desktop/src/app/chat/right-rail/preview-act.test.ts @@ -4,6 +4,7 @@ import { $rightRailActiveTabId } from '@/store/layout' import { closeRightRail, openPreview, type PreviewTarget } from '@/store/preview' import { actOnActivePreview } from './preview-act' +import { registerPreviewInput } from './preview-input' import { registerPreviewNav } from './preview-nav' import { registerPreviewScriptRunner } from './preview-script-runner' @@ -11,7 +12,7 @@ function urlTarget(url: string): PreviewTarget { return { kind: 'url', label: 'Browser', source: url, url } } -describe('actOnActivePreview (act_preview tool)', () => { +describe('actOnActivePreview (drive_preview tool)', () => { // URL targets share the singleton Browser tab id, so anything a test // registers would answer the next one. let cleanups: Array<() => void> = [] @@ -105,6 +106,192 @@ describe('actOnActivePreview (act_preview tool)', () => { expect((await actOnActivePreview({ kind: 'click', ref: '@e1' })).error).toContain('did not answer') }) + /** A pane that answers the locate trip with a fixed on-screen point, and the + * read-back trip with an empty inventory. Returns the input spy. */ + const withDrivenPane = () => { + const tabId = openBrowserTab() + const send = vi.fn() + + cleanups.push( + registerPreviewScriptRunner(tabId, async code => + code.includes('"kind":"locate"') + ? JSON.stringify({ acted: 'looking at button "Save"', point: { x: 120, y: 80 }, success: true }) + : // `hit` is the page's witness that the real pointerdown arrived. + JSON.stringify({ elements: [], hit: { tag: 'BUTTON', trusted: true }, success: true }) + ) + ) + cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send })) + + return send + } + + const sentTypes = (send: ReturnType) => send.mock.calls.map(([event]) => event.type) + + it('clicks with real input, walking the pointer to where the page said', async () => { + const send = withDrivenPane() + + const result = await actOnActivePreview({ kind: 'click', ref: '@e1' }) + const types = sentTypes(send) + + // Stepped rather than teleported: a page learns it is hovered from a stream + // of moves, and one jump to the target skips everything in between. + expect(types.filter(type => type === 'mouseMove').length).toBeGreaterThan(1) + expect(types).toContain('mouseDown') + expect(types).toContain('mouseUp') + expect(send.mock.calls.map(([event]) => event).find(event => event.type === 'mouseDown')).toMatchObject({ + x: 120, + y: 80 + }) + expect(result.acted).toBe('clicked button "Save"') + }) + + it('types by pressing keys, after selecting whatever the field held', async () => { + const send = withDrivenPane() + + await actOnActivePreview({ kind: 'type', ref: '@e1', submit: true, text: 'hi' }) + + const events = send.mock.calls.map(([event]) => event) + const chars = events.filter(event => event.type === 'char').map(event => event.keyCode) + + // One click to focus, then select-all by keyboard — NOT the triple-click + // this used to do. A triple-click is a pointer gesture, so it grabs the + // paragraph under the cursor whenever the target turns out not to be a + // field, and the agent was leaving pages with their body text highlighted. + expect(events.filter(event => event.type === 'mouseDown').map(event => event.clickCount)).toEqual([1]) + expect(events.filter(event => event.type === 'keyDown' && event.keyCode === 'a')[0]).toMatchObject({ + modifiers: ['control', 'meta'] + }) + // The chord must not send a `char` phase, or select-all types a literal 'a'. + expect(chars).toEqual(['h', 'i', 'Enter']) + }) + + it('hovers by walking the pointer over and leaving it there', async () => { + const send = withDrivenPane() + + const result = await actOnActivePreview({ kind: 'hover', ref: '@e1' }) + const types = sentTypes(send) + + expect(types).toContain('mouseMove') + // The whole request is "be on it" — a click here would open the dropdown the + // agent was trying to reveal, or worse, activate it. + expect(types).not.toContain('mouseDown') + expect(result.acted).toBe('hovered over button "Save"') + }) + + // The witness only speaks for verbs that put the button down. Demanding one + // from a key press reported every press as a failure. + it('does not expect a click witness from a verb that never clicks', async () => { + const tabId = openBrowserTab() + + cleanups.push( + registerPreviewScriptRunner(tabId, async code => + code.includes('"kind":"locate"') + ? JSON.stringify({ acted: 'looking at textbox "Search"', point: { x: 40, y: 20 }, success: true }) + : JSON.stringify({ elements: [], hit: null, success: true }) + ) + ) + cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send: vi.fn() })) + + for (const action of [ + { key: 'Escape', kind: 'press', ref: '@e1' }, + { kind: 'hover', ref: '@e1' } + ]) { + expect(await actOnActivePreview(action)).toMatchObject({ success: true }) + } + }) + + it('scrolls by wheeling for real, not by scripting the page', async () => { + const tabId = openBrowserTab() + const send = vi.fn() + let scripted = false + + cleanups.push( + registerPreviewScriptRunner(tabId, async code => { + scripted ||= code.includes('"kind":"scroll"') + + return code.includes('scrollHeight') + ? JSON.stringify({ page: 700, point: { x: 500, y: 400 }, span: 4_000, success: true }) + : JSON.stringify({ elements: [], success: true }) + }) + ) + cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send })) + + await actOnActivePreview({ kind: 'scroll' }) + + const wheels = send.mock.calls.map(([event]) => event).filter(event => event.type === 'mouseWheel') + + // A stream of notches, not one delta: scroll-linked headers and lazy loaders + // only react to the events, so a scripted scrollBy leaves them asleep. + expect(wheels.length).toBeGreaterThan(1) + // Electron's wheel delta is wheelDelta-signed, so scrolling DOWN is negative. + expect(wheels.every(event => event.deltaY < 0)).toBe(true) + expect(scripted).toBe(false) + }) + + it('says so plainly when the page has nothing to scroll', async () => { + const tabId = openBrowserTab() + + cleanups.push( + registerPreviewScriptRunner(tabId, async () => + JSON.stringify({ page: 700, point: { x: 500, y: 400 }, span: 0, success: true }) + ) + ) + cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send: vi.fn() })) + + expect(await actOnActivePreview({ kind: 'scroll' })).toMatchObject({ note: expect.stringContaining('nothing to scroll') }) + }) + + it('fails loudly when the pointer input never reaches the page', async () => { + const tabId = openBrowserTab() + + cleanups.push( + registerPreviewScriptRunner(tabId, async code => + code.includes('"kind":"locate"') + ? JSON.stringify({ acted: 'looking at button "Save"', point: { x: 12, y: 8 }, success: true }) + : JSON.stringify({ elements: [], hit: null, success: true }) + ) + ) + cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send: vi.fn() })) + + const result = await actOnActivePreview({ kind: 'click', ref: '@e1' }) + + // Everything else about this action travels on the script channel and would + // report success whether or not a single event landed. + expect(result.success).toBe(false) + expect(result.error).toContain('never reached the page') + }) + + it('says so when the overlay itself swallowed the click', async () => { + const tabId = openBrowserTab() + + cleanups.push( + registerPreviewScriptRunner(tabId, async code => + code.includes('"kind":"locate"') + ? JSON.stringify({ acted: 'looking at button "Save"', point: { x: 12, y: 8 }, success: true }) + : JSON.stringify({ elements: [], hit: { tag: 'HERMES-WATCH', trusted: true }, success: true }) + ) + ) + cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send: vi.fn() })) + + expect((await actOnActivePreview({ kind: 'click', ref: '@e1' })).note).toContain('overlay intercepted') + }) + + it('falls back to scripted events when the pane exposes no input channel', async () => { + let injected = '' + + withRunner(async code => { + injected = code + + return JSON.stringify({ acted: 'clicked', success: true }) + }) + + await actOnActivePreview({ kind: 'click', ref: '@e1' }) + + // The one-trip shape: the engine both acts and re-reads, no locate handshake. + expect(injected).toContain('"kind":"click"') + expect(injected).not.toContain('"kind":"locate"') + }) + it('routes history verbs to the pane instead of the guest page', async () => { const back = vi.fn() const runner = vi.fn() diff --git a/apps/desktop/src/app/chat/right-rail/preview-act.ts b/apps/desktop/src/app/chat/right-rail/preview-act.ts index 495ce45f0a..bcef505fbc 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-act.ts +++ b/apps/desktop/src/app/chat/right-rail/preview-act.ts @@ -1,14 +1,23 @@ /** * PREVIEW ACT — performs the agent's interactions inside the preview pane's - * guest page, so `act_preview` drives whatever web app is open in the in-app + * guest page, so `drive_preview` drives whatever web app is open in the in-app * browser. * * The guest page is out-of-process; nothing here can touch its DOM directly. - * Instead each action injects the engine SOURCE over `executeJavaScript` (see - * lib/preview-act/act-in-page.ts's self-containment contract), parked on a - * window global alongside the ref holder so '@e5' still resolves on the next - * call. Injection is idempotent and vanishes with the page — a navigation - * drops the refs, which the engine reports rather than clicking the wrong node. + * Two separate channels reach it, and the split matters: + * + * - `executeJavaScript` injects the engine SOURCE (see + * lib/preview-act/act-in-page.ts's self-containment contract) to RESOLVE + * and READ — turn '@e5' into a node, measure it, inventory the page. It is + * parked on a window global alongside the ref holder so refs survive + * between calls, and it vanishes with the page, so a navigation drops the + * refs and the engine reports that instead of clicking the wrong node. + * - `sendInputEvent` (preview-drive.ts) does the ACTING, as real Chromium + * input. Script can only dispatch synthetic events, which the page can tell + * apart and which never move the browser's own hover or focus target. + * + * Panes with no input channel — a remote HTML preview — fall back to the + * engine's synthetic events, which is worse but still works. * * A mutating action settles briefly and returns a fresh inventory, so the * click → re-read → click loop costs one round trip instead of two. @@ -18,46 +27,461 @@ */ import { actInPage, type PreviewActAction, type PreviewActResult } from '@/lib/preview-act/act-in-page' +import { watchInPage } from '@/lib/preview-act/watch-in-page' +import { clickAt, glideTo, pointerPlaced, pressKey, selectAll, typeText, wheelBy } from './preview-drive' +import { activePreviewInput, type PreviewInputHandle } from './preview-input' import { activePreviewNav, type PreviewNavHandle } from './preview-nav' -import { activePreviewScriptRunner } from './preview-script-runner' +import { activePreviewScriptRunner, type PreviewScriptRunner } from './preview-script-runner' /** Verbs the pane owns; a guest page cannot drive its own history. */ const NAV_ACTIONS: readonly (keyof PreviewNavHandle)[] = ['back', 'forward', 'reload'] /** How long a click/type is given to land before the page is re-inventoried. - * Long enough for a framework re-render, short enough not to stall the turn. */ -const SETTLE_MS = 400 + * Long enough for a framework re-render, short enough not to stall the turn. + * A render that misses this window is not lost — the NEXT action re-reads the + * page anyway, so the cost of being early is a slightly stale inventory, not a + * wrong one. That trade is worth several hundred ms on every single step. */ +const SETTLE_MS = 220 + +/** Cap on one round trip into the guest page. A click that starts a navigation + * tears the document down mid-settle, so the injected promise dies with it and + * `executeJavaScript` never settles — without this the tool eats its whole + * 45s bridge deadline for what was actually a successful click. */ +const ACT_TIMEOUT_MS = 8_000 + +/** Verbs that get the look-then-act treatment: locate the target, walk the + * pointer there, then send real input. Actions with no single target (scroll, + * elements) are absent and skip it. */ +const DRIVEN: readonly string[] = ['click', 'hover', 'press', 'type'] + +/** Verbs that put the mouse button down, and so are the only ones the pointerdown + * witness can speak to. `press` and `hover` never click, so demanding a witness + * from them would report every one of them as a failure. */ +const CLICKS: readonly string[] = ['click', 'type'] const NOTHING_OPEN = 'No live page is open in the in-app browser — open one with open_preview first.' -/** Build the idempotent inject-and-run script for one action. */ -function buildActScript(action: PreviewActAction, settleMs: number): string { +const NAVIGATED = 'The page stopped answering right after — it is probably navigating. Call elements to see where you landed.' + +/** A fingerprint of the overlay's source, so the guest page can tell that the + * code it is running has changed underneath it. + * + * It needs one because the overlay memoizes its own chrome: the cursor, the + * lock frame and the scan line are built ONCE per page and then reused, so an + * edit to any of their styles lands in the injected source, gets injected, and + * changes nothing — the layers it would have styled were built by the previous + * version and are still on the page. In dev that reads as "HMR didn't pick up + * my CSS"; in production it is a long-lived tab pinned to whichever build first + * touched it, across an app upgrade. + * + * Content-addressed rather than a length or a version literal: the change that + * exposed this was one hex colour for another, which is byte-for-byte the same + * length, and a hand-maintained version is a step someone forgets. */ +const WATCH_TAG = (() => { + const source = watchInPage.toString() + let hash = 5381 + + for (let i = 0; i < source.length; i++) { + hash = ((hash << 5) + hash + source.charCodeAt(i)) | 0 + } + + return hash +})() + +/** Injected once per page: the engine, the overlay, and the two helpers every + * script below shares. Idempotent — re-running it on a live page is a no-op. */ +const preamble = () => ` var w = window; + // Reassigned on every trip rather than cached behind a guard: the source is in + // this payload either way, so the guard saved nothing and pinned a long-lived + // tab to whichever build first touched it. The holder is the exception — it + // carries the aimed element from the locate trip to the act trip. + w.__hermesActHolder = w.__hermesActHolder || {}; + w.__hermesAct = (${actInPage.toString()}); + w.__hermesWatch_fn = (${watchInPage.toString()}); + w.__hermesWatchTag = ${WATCH_TAG}; + var holder = w.__hermesActHolder; + var act = function (a) { return w.__hermesAct(document, holder, a); }; + var watch = function (stage, label) { + try { w.__hermesWatch_fn(document, holder, stage, label); } catch (err) {} + }; + var wait = function (ms) { return new Promise(function (resolve) { setTimeout(resolve, ms); }); }; + // Sit out a smooth scroll so the pointer is aimed at a target that has stopped + // moving. A scroll that never started fires no scrollend, so only wait on one + // we can actually see happening; capture catches nested scrollers, whose + // scrollend does not bubble. + var restAfterScroll = function () { + return new Promise(function (resolve) { + var sc = document.scrollingElement || document.documentElement; + var x0 = sc ? sc.scrollLeft : 0; + var y0 = sc ? sc.scrollTop : 0; + requestAnimationFrame(function () { + if (!sc || (sc.scrollLeft === x0 && sc.scrollTop === y0)) { return resolve(); } + var done = false; + var finish = function () { + if (done) { return; } + done = true; + document.removeEventListener('scrollend', finish, true); + clearTimeout(timer); + resolve(); + }; + var timer = setTimeout(finish, 350); + document.addEventListener('scrollend', finish, true); + }); + }); + };` + +/** Bring the target on screen, mark it, and report where it came to rest. */ +function buildLocateScript(action: PreviewActAction, focus: boolean): string { + const locate = { focus, kind: 'locate', ref: action.ref, selector: action.selector } + return `(function () { - var w = window; - if (!w.__hermesAct) { - w.__hermesActHolder = {}; - w.__hermesAct = (${actInPage.toString()}); - } - var act = function (a) { return w.__hermesAct(document, w.__hermesActHolder, a); }; +${preamble()} + var locate = ${JSON.stringify(locate)}; + var found = act(locate); + if (!found.success) { return Promise.resolve(JSON.stringify(found)); } + watch('aim'); + // Arm a witness for the real input that is about to arrive. Without it a + // click that never reached the page is indistinguishable from one the page + // ignored, and the agent would report success either way. + w.__hermesHit = null; + document.addEventListener('pointerdown', function (e) { + w.__hermesHit = { tag: e.target ? e.target.tagName : '?', trusted: e.isTrusted === true }; + }, { capture: true, once: true }); + // Measure again once the scroll has stopped: real input is aimed at a fixed + // viewport coordinate, so it has to be where the target ENDS UP. + return restAfterScroll().then(function () { + var settled = act(locate); + var best = settled.success ? settled : found; + var at = best.point; + // Last line of defence before the pointer is sent somewhere real. An element + // that is still outside the viewport after we scrolled to it is hidden, not + // placed, and aiming at it would drive the cursor off into a corner and + // click whatever happens to be under that coordinate. + if (at && (at.x < 0 || at.y < 0 || at.x > window.innerWidth || at.y > window.innerHeight)) { + return JSON.stringify({ + error: 'That element is still off-screen after scrolling to it, so it is hidden rather than clickable. Call elements again for what is really on the page.', + success: false + }); + } + return JSON.stringify(best); + }); +})()` +} + +/** Put up a mark that outlives the action that made it. Every other cue on the + * overlay retires on a timer, which is right for narrating a click and no use + * at all for holding a finding on screen while the agent keeps working. */ +function buildPinScript(action: PreviewActAction, label: string): string { + const locate = { kind: 'locate', ref: action.ref, selector: action.selector } + + return `(function () { +${preamble()} + var found = act(${JSON.stringify(locate)}); + if (!found.success) { return JSON.stringify(found); } + watch('pin', ${JSON.stringify(label)}); + return JSON.stringify({ acted: 'pinned ' + String(found.acted || 'it').replace(/^looking at /, ''), success: true }); +})()` +} + +/** Freeze the whole visible field as marks. Needs a fresh inventory first, so + * the overlay has both the field to draw and the refs to number it by. */ +function buildHoldScript(): string { + return `(function () { +${preamble()} + var found = act({ kind: 'elements' }); + if (!found.success) { return JSON.stringify(found); } + watch('hold'); + return JSON.stringify({ acted: 'held the field', elements: found.elements, title: found.title, url: found.url, success: true }); +})()` +} + +/** Rattle through the field one box at a time. Needs a fresh inventory for the + * overlay to pick from, and nothing else — the page is never touched. */ +function buildStrobeScript(): string { + return `(function () { +${preamble()} + var found = act({ kind: 'elements' }); + if (!found.success) { return JSON.stringify(found); } + watch('strobe'); + return JSON.stringify({ acted: 'strobed the field', elements: found.elements, title: found.title, url: found.url, success: true }); +})()` +} + +/** Take one mark down, or — with nothing to aim at — all of them. */ +function buildUnpinScript(action: PreviewActAction): string { + const locate = { kind: 'locate', ref: action.ref, selector: action.selector } + const one = !!(action.ref || action.selector) + + return `(function () { +${preamble()} +${ + one + ? ` var found = act(${JSON.stringify(locate)}); + if (!found.success) { return JSON.stringify(found); } + var gone = 'unpinned ' + String(found.acted || 'it').replace(/^looking at /, '');` + : ` holder.aimed = null; + var gone = 'cleared every pin';` +} + watch('unpin'); + return JSON.stringify({ acted: gone, success: true }); +})()` +} + +/** Mark the hit, let the page react, and hand back a fresh inventory. */ +function buildFinishScript(settleMs: number): string { + return `(function () { +${preamble()} + watch('strike'); + return wait(${settleMs}).then(function () { + // A rescan that throws must still answer — an unresolved promise here costs + // the whole bridge deadline. + try { + var out = act({ kind: 'elements' }); + watch('sweep'); + out.hit = w.__hermesHit || null; + return JSON.stringify(out); + } catch (err) { + return JSON.stringify({ note: 'The page changed before it could be re-read: ' + err, success: true }); + } + }); +})()` +} + +/** The no-real-input fallback: the engine both acts and reads, in one trip. + * Both reads mark the field — the explicit `elements` call and the re-read + * every other verb does on its way out — so the page is marked after every + * single action rather than only when the agent asks for an inventory outright. + * They mark it differently, though, and the difference is what each one MEANS: + * an outright `elements` is the agent reading the whole page, which is the + * moment worth showing, so it strobes. The re-read after a click is + * housekeeping, and strobing there would put five seconds of noise between + * every step of a task, so it sweeps. The two never collide: an `elements` call + * settles in 0ms and returns before the re-read. */ +function buildScriptedScript(action: PreviewActAction, settleMs: number): string { + return `(function () { +${preamble()} var result = act(${JSON.stringify(action)}); - if (!result.success || ${settleMs} <= 0) { - return Promise.resolve(JSON.stringify(result)); + ${ + action.kind === 'elements' + ? "if (result.success) { watch('strobe'); }" + : '' } - // Re-inventory after the page has had a moment to react, so the agent's next - // ref is drawn from the DOM its own click produced. - return new Promise(function (resolve) { - setTimeout(function () { + if (!result.success || ${settleMs} <= 0) { return Promise.resolve(JSON.stringify(result)); } + return wait(${settleMs}).then(function () { + try { var after = act({ kind: 'elements' }); + watch('sweep'); result.elements = after.elements; result.url = after.url; result.title = after.title; - resolve(JSON.stringify(result)); - }, ${settleMs}); + } catch (err) { + result.note = 'The page changed before it could be re-read: ' + err; + } + return JSON.stringify(result); }); })()` } +/** The outcome of one round trip into the page. `silent` is its own case on + * purpose: a page that stops answering mid-action is navigating, whereas one + * that answers with nothing is broken, and the agent needs to hear the + * difference. */ +type Trip = { error: string; kind: 'failed' } | { kind: 'answered'; result: PreviewActResult } | { kind: 'silent' } + +async function runJson(run: PreviewScriptRunner, code: string): Promise { + const raw = await Promise.race([ + run(code).catch((error: unknown) => new Error(String(error))), + new Promise(resolve => setTimeout(resolve, ACT_TIMEOUT_MS)) + ]) + + if (raw === undefined) { + return { kind: 'silent' } + } + + if (raw instanceof Error) { + return { error: 'The page rejected the action: ' + raw.message, kind: 'failed' } + } + + if (typeof raw !== 'string' || !raw) { + return { error: 'The page did not answer the action.', kind: 'failed' } + } + + return { kind: 'answered', result: JSON.parse(raw) as PreviewActResult } +} + +/** Past tense of the verb the agent asked for, against what it actually hit. */ +function describeDone(action: PreviewActAction, target: string): string { + if (action.kind === 'type') { + return 'typed into ' + target + (action.submit ? ' and submitted' : '') + } + + if (action.kind === 'press') { + return 'pressed ' + (action.key || '') + ' on ' + target + } + + if (action.kind === 'hover') { + return 'hovered over ' + target + } + + return 'clicked ' + target +} + +/** Look at the target, walk the pointer over, and act on it for real. */ +async function driveAction( + run: PreviewScriptRunner, + input: PreviewInputHandle, + action: PreviewActAction +): Promise { + // A key press must not be preceded by a click — that would activate the + // control rather than type into it — so the page hands it focus instead. + const trip = await runJson(run, buildLocateScript(action, action.kind === 'press')) + + if (trip.kind === 'failed') { + return { error: trip.error, success: false } + } + + if (trip.kind === 'silent') { + return { acted: action.kind, note: NAVIGATED, success: true } + } + + const found = trip.result + + if (!found.success) { + return found + } + + if (!found.point) { + return { error: 'Could not work out where that element is on screen.', success: false } + } + + await glideTo(input, found.point) + + if (action.kind === 'click') { + await clickAt(input) + } else if (action.kind === 'type') { + if (found.typable === false) { + return { + error: `${String(found.acted || 'That').replace(/^looking at /, '')} is not a text field, so typing into it would only select the text under the pointer. Click it if it opens one, then type into that.`, + success: false + } + } + + input.focus() + await clickAt(input) + // Select-all inside the now-focused field, so typing replaces what is there + // the way it would for a person. NOT a triple-click: that is a pointer + // gesture and selects the paragraph under the cursor whenever the target + // turns out not to be a field. + await selectAll(input) + await typeText(input, action.text ?? '') + + if (action.submit) { + await pressKey(input, 'Enter') + } + } else if (action.kind === 'press') { + input.focus() + await pressKey(input, action.key || 'Enter') + } + // hover is the glide and nothing else — the pointer is already sitting on the + // target, which is the whole request. + + const target = String(found.acted || '').replace(/^looking at /, '') + const after = await runJson(run, buildFinishScript(SETTLE_MS)) + const acted = describeDone(action, target) + + // The action itself already happened as real input, so a page that will not + // answer the read-back is a page that navigated — never a failed click. + if (after.kind !== 'answered') { + return { acted, note: NAVIGATED, success: true } + } + + const { hit, ...result } = after.result as PreviewActResult & { hit?: { tag: string; trusted: boolean } | null } + + // The witness the locate trip armed. No record means the input never reached + // the document, which the agent must hear about — every other signal here + // travels on the script channel and would report success regardless. + if (!hit && CLICKS.indexOf(action.kind) !== -1) { + return { + ...result, + error: 'The pointer input never reached the page, so nothing was ' + acted.split(' ')[0] + '.', + success: false + } + } + + return { ...result, acted, note: hitNote(hit), success: true } +} + +/** Flag a click the overlay intercepted, which would otherwise look like a page + * that simply ignored it. */ +function hitNote(hit?: { tag: string; trusted: boolean } | null): string | undefined { + return hit && hit.tag === 'HERMES-WATCH' + ? 'The action overlay intercepted the click instead of the page.' + : undefined +} + +/** How far a screenful is, whether there is anywhere to go, and a spot to wheel + * over — the renderer cannot see the guest viewport to work any of it out. */ +function buildScrollAnchorScript(): string { + return `(function () { +${preamble()} + var sc = document.scrollingElement || document.documentElement; + var track = sc.clientHeight || window.innerHeight; + return Promise.resolve(JSON.stringify({ + page: Math.round(window.innerHeight * 0.9), + point: { x: Math.round(window.innerWidth / 2), y: Math.round(track / 2) }, + span: sc.scrollHeight - track, + success: true + })); +})()` +} + +/** Scroll the page the way a hand does — wheel it. The scripted path calls + * `scrollBy`, which moves the page without the page ever seeing an input + * event, so scroll-linked headers, reveal animations and infinite-scroll + * loaders all stay asleep through a scroll that looked like it happened. */ +async function driveScroll( + run: PreviewScriptRunner, + input: PreviewInputHandle, + action: PreviewActAction, +): Promise { + const far = action.amount ?? 0 + const trip = await runJson( + run, + buildScrollAnchorScript() + ) + + if (trip.kind === 'failed') { + return { error: trip.error, success: false } + } + + if (trip.kind === 'silent') { + return { acted: 'scrolled', note: NAVIGATED, success: true } + } + + const anchor = trip.result as PreviewActResult & { page?: number; span?: number } + + if (!anchor.span) { + return { ...anchor, acted: 'scrolled the page', note: 'The page has nothing to scroll — it all fits already.' } + } + + // A person does not move the mouse to scroll; the wheel turns wherever their + // hand already is. Only send it somewhere if it has never been anywhere. + if (!pointerPlaced() && anchor.point) { + await glideTo(input, anchor.point) + } + + await wheelBy(input, action.amount ?? anchor.page ?? 600) + + const after = await runJson(run, buildFinishScript(SETTLE_MS)) + + if (after.kind !== 'answered') { + return { acted: 'scrolled the page', note: NAVIGATED, success: true } + } + + return { ...after.result, acted: 'scrolled the page', success: true } +} + /** Run one action against the ACTIVE preview tab's page. `kind` is a bare * string: the verb arrives off the wire, and the history ones never reach * the in-page engine. */ @@ -87,12 +511,57 @@ export async function actOnActivePreview( } const typed = action as PreviewActAction - const settle = typed.kind === 'elements' ? 0 : SETTLE_MS - const raw = await run(buildActScript(typed, settle)) - if (typeof raw !== 'string' || !raw) { - return { error: 'The page did not answer the action.', success: false } + // Annotation, not interaction: nothing is clicked, nothing settles, and the + // page is not re-read, so these skip the whole act-then-inventory path. + if (typed.kind === 'pin' || typed.kind === 'unpin' || typed.kind === 'hold' || typed.kind === 'strobe') { + const mark = + typed.kind === 'hold' + ? buildHoldScript() + : typed.kind === 'strobe' + ? buildStrobeScript() + : typed.kind === 'pin' + ? buildPinScript(typed, typed.text || '') + : buildUnpinScript(typed) + const trip = await runJson(run, mark) + + if (trip.kind === 'failed') { + return { error: trip.error, success: false } + } + + return trip.kind === 'answered' ? trip.result : { acted: typed.kind, note: NAVIGATED, success: true } } - return JSON.parse(raw) as PreviewActResult + const input = activePreviewInput() + + if (input && DRIVEN.indexOf(typed.kind) !== -1) { + return driveAction(run, input, typed) + } + + // A plain page scroll is a wheel gesture. Jumping to an end is not — no hand + // wheels to the bottom of a long article — so `to` stays a scripted glide. + const plain = typed.kind === 'scroll' && !typed.to && !typed.ref && !typed.selector + + if (plain && input) { + return driveScroll(run, input, typed) + } + + const settle = typed.kind === 'elements' ? 0 : SETTLE_MS + const scripted = await runJson(run, buildScriptedScript(typed, settle)) + + if (scripted.kind === 'failed') { + return { error: scripted.error, success: false } + } + + // The action almost certainly landed — a page that stops answering right + // after a click is one that navigated. Say so instead of failing it. + return scripted.kind === 'silent' ? { acted: typed.kind, note: NAVIGATED, success: true } : scripted.result +} + +// Self-accept so an edit here, or to the in-page sources this module +// stringifies, doesn't reload the whole renderer out from under a live session. +// The bridge re-requests this module per action in dev, so it picks the change +// up without one (see loadPreviewEngine in gateway-event/desktop-bridge.ts). +if (import.meta.hot) { + import.meta.hot.accept() } diff --git a/apps/desktop/src/app/chat/right-rail/preview-mind.ts b/apps/desktop/src/app/chat/right-rail/preview-mind.ts new file mode 100644 index 0000000000..3f7a1856c4 --- /dev/null +++ b/apps/desktop/src/app/chat/right-rail/preview-mind.ts @@ -0,0 +1,29 @@ +/** + * IDLE PULSE — keeps the preview overlay alive while the model reasons. + * + * Everything else the overlay draws is narration of an action, so the pane goes + * dark for the gap between actions — which is most of the time the agent spends + * on a page, and the part where it is deciding what to do with what it just + * read. A page that stops responding the moment the agent is thinking hardest + * reads as the agent having wandered off. + * + * This is deliberately NOT part of the act pipeline: a turn boundary only has + * to poke the overlay the last action left behind. See preview-nudge.ts. + */ + +import { $busy } from '@/store/session' + +import { nudgeOverlay } from './preview-nudge' + +// Module-level, matching how review.ts and coding-status.ts watch this edge. It +// is inert without a live pane, so there is nothing to mount or tear down. +let running = $busy.get() + +$busy.subscribe(busy => { + if (busy === running) { + return + } + + running = busy + nudgeOverlay(busy ? 'think' : 'rest') +}) diff --git a/apps/desktop/src/app/chat/right-rail/preview-nudge.ts b/apps/desktop/src/app/chat/right-rail/preview-nudge.ts new file mode 100644 index 0000000000..8a8f419987 --- /dev/null +++ b/apps/desktop/src/app/chat/right-rail/preview-nudge.ts @@ -0,0 +1,38 @@ +/** + * OVERLAY NUDGE — the one way to say a stage to an overlay that is ALREADY on + * the page. + * + * The act pipeline (preview-act.ts) injects the whole engine because it has to: + * it needs to resolve a ref, measure a node and read the page back. The stages + * that only narrate — the idle pulse, the read wipe — need none of that. They + * are one function call against something the last action already left on the + * page, so re-shipping the engine to make it would put the payload on the wire + * on every turn boundary. + * + * A page the agent has never acted on has no overlay, and the nudge is a no-op + * there. That is the right answer rather than a gap: chrome on a page the agent + * never touched would be a lie about what it did. + */ + +import type { WatchStage } from '@/lib/preview-act/watch-in-page' + +import { activePreviewScriptRunner } from './preview-script-runner' + +/** Run one stage against the active pane's overlay, if it has one. */ +export function nudgeOverlay(stage: WatchStage): void { + const run = activePreviewScriptRunner() + + if (!run) { + return + } + + void run(`(function () { + var w = window; + var fn = w.__hermesWatch_fn; + if (!fn) { return 'cold'; } + try { fn(document, w.__hermesActHolder || {}, ${JSON.stringify(stage)}); } catch (err) {} + return 'ok'; +})()`).catch(() => { + // The page navigated out from under us. The next action re-injects. + }) +} diff --git a/apps/desktop/src/app/chat/right-rail/preview-pane.tsx b/apps/desktop/src/app/chat/right-rail/preview-pane.tsx index 076fc4a94f..817cec638a 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-pane.tsx +++ b/apps/desktop/src/app/chat/right-rail/preview-pane.tsx @@ -1,3 +1,7 @@ +// Side-effect import: watches the turn edge so the overlay keeps a pulse while +// the model reasons. Lives here because the pane is what makes it reachable. +import './preview-mind' + import { useStore } from '@nanostores/react' import type { PointerEvent as ReactPointerEvent } from 'react' import { useCallback, useEffect, useMemo, useRef, useState } from 'react' @@ -28,7 +32,7 @@ import { import { type ConsoleEntry } from './preview-console-state' import { previewConsoleState } from './preview-console-store' import { LocalFilePreview, PreviewEmptyState } from './preview-file' -import { registerPreviewInput, type PreviewInputEvent } from './preview-input' +import { type PreviewInputEvent, registerPreviewInput } from './preview-input' import { PREVIEW_BROWSER_ATTR, registerPreviewNav } from './preview-nav' import { registerPreviewPageReader } from './preview-reader' import { registerPreviewScriptRunner } from './preview-script-runner' diff --git a/apps/desktop/src/app/chat/right-rail/preview-reader.ts b/apps/desktop/src/app/chat/right-rail/preview-reader.ts index 7f48c8cbb1..97a372db18 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-reader.ts +++ b/apps/desktop/src/app/chat/right-rail/preview-reader.ts @@ -15,6 +15,8 @@ import { $rightRailActiveTabId } from '@/store/layout' import { $previewTabs } from '@/store/preview' +import { nudgeOverlay } from './preview-nudge' + export interface PreviewReadOptions { /** Characters to return from `start` (capped at PREVIEW_READ_MAX_CHARS). */ count?: number @@ -89,6 +91,13 @@ export async function readActivePreview(opts: PreviewReadOptions = {}): Promise< try { const page = await reader() + // Say it on the page. Reading is by far the cheapest thing the agent + // does — a few hundredths of a second against a model round trip either + // side of it — so a run of reads used to leave the pane dark for the + // twenty seconds it took to page through a document, immediately after + // the one moment that showed anything. + nudgeOverlay('read') + return windowText( { kind: target.kind, path: target.path, title: page.title || target.label, url: page.url || target.url }, page.text, diff --git a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/desktop-bridge.ts b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/desktop-bridge.ts index 1a8cee9d02..227a4eb128 100644 --- a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/desktop-bridge.ts +++ b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/desktop-bridge.ts @@ -11,6 +11,34 @@ import { setMessages } from '@/store/session' import type { GatewayEventContext } from './types' +/** The preview engine, loaded on demand so ~25KB of page-injectable source stays + * off the boot path. + * + * In dev that lazy chunk is also a trap. The browser caches a dynamic import by + * URL for the life of the page, so this bridge would hand every action to + * whichever build of the engine loaded first, and no edit to it — or to the + * overlay whose source it stringifies into the page — would reach the guest + * until the whole window reloaded. Asking for a fresh copy is more reliable + * than trusting hot-update propagation to reach a module nothing statically + * imports; the dev server stamps the dependency URLs it has invalidated, so a + * fresh engine pulls a fresh overlay down with it. + * + * The literal path is what a bare specifier can't be here, and it has to track + * this module's real location — hence the fall back to the static import, which + * is also the only branch production keeps, `import.meta.hot` being stripped + * there along with everything it guards. */ +const loadPreviewEngine = () => { + const stable = () => import('@/app/chat/right-rail/preview-act') + + if (!import.meta.hot) { + return stable().then(mod => mod.actOnActivePreview) + } + + return import(/* @vite-ignore */ '/src/app/chat/right-rail/preview-act.ts?hot=' + Date.now()) + .catch(stable) + .then(mod => mod.actOnActivePreview as Awaited>['actOnActivePreview']) +} + /** Desktop-surface bridge events: read-back requests the agent blocks on * (terminal/preview/window), agent terminal streaming, pane reveal, and * message reactions. */ @@ -57,7 +85,7 @@ export function handleDesktopBridgeEvent(ctx: GatewayEventContext): boolean { } if (event.type === 'preview.act.request') { - // act_preview tool: click/type/scroll/press inside the guest page, or + // drive_preview tool: click/type/scroll/press inside the guest page, or // drive the pane's history. Dynamic import keeps the injected engine off // the boot path. Active session only: a background turn must never reach // into the page the user is working in (desktop AGENTS.md: offer, don't @@ -72,9 +100,9 @@ export function handleDesktopBridgeEvent(ctx: GatewayEventContext): boolean { }) if (isActiveEvent) { - void import('@/app/chat/right-rail/preview-act') - .then(({ actOnActivePreview }) => - actOnActivePreview({ + void loadPreviewEngine() + .then(run => + run({ amount: payload?.amount, key: payload?.key, kind: payload?.action ?? '', diff --git a/apps/desktop/src/lib/preview-act/act-in-page.test.ts b/apps/desktop/src/lib/preview-act/act-in-page.test.ts index aa1985fb7a..5fafdb91e8 100644 --- a/apps/desktop/src/lib/preview-act/act-in-page.test.ts +++ b/apps/desktop/src/lib/preview-act/act-in-page.test.ts @@ -3,14 +3,17 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { actInPage, type PreviewActHolder } from './act-in-page' /** jsdom lays nothing out, so every rect is 0×0 and the engine's visibility - * check would reject the whole page. Give elements a plausible box and let + * check would reject the whole page. Give elements a plausible box, honouring + * an explicit width/height so a test can lay out a 1px one, and let * `display: none` (which jsdom DOES compute) carry the hiding. */ function layOutTheDocument() { vi.spyOn(Element.prototype, 'getBoundingClientRect').mockImplementation(function (this: Element) { - const hidden = getComputedStyle(this).display === 'none' - const size = hidden ? 0 : 40 + const style = getComputedStyle(this) + const width = style.display === 'none' ? 0 : parseFloat(style.width) || 40 + const height = style.display === 'none' ? 0 : parseFloat(style.height) || 40 + const left = parseFloat(style.left) || 0 - return { bottom: size, height: size, left: 0, right: size, top: 0, width: size, x: 0, y: 0 } as DOMRect + return { bottom: height, height, left, right: left + width, top: 0, width, x: left, y: 0 } as DOMRect }) } @@ -29,6 +32,33 @@ beforeEach(() => { vi.restoreAllMocks() layOutTheDocument() Element.prototype.scrollIntoView = vi.fn() + // jsdom has no hit-testing at all, so the engine skips the occlusion check + // here by default. Tests that stand one in must not leak it into the next. + delete (document as Document & { elementFromPoint?: unknown }).elementFromPoint +}) + +describe('self-containment', () => { + // The pane injects `actInPage.toString()` into the guest page, where module + // scope does not exist. A single free identifier (a module-level constant, an + // imported helper) is a ReferenceError on every call, and Electron reports it + // only as "Script failed to execute" — so evaluate the source the way the + // guest does and run a real action through it. + it('runs after being stringified and eval’d with no module scope', () => { + const page = document.createElement('div') + page.innerHTML = '' + document.body.replaceChildren(page) + + const injected = new Function('return (' + actInPage.toString() + ')')() as typeof actInPage + const holder: PreviewActHolder = {} + + expect(injected(document, holder, { kind: 'elements' }).elements?.[0].label).toBe('Save') + + const clicked = vi.fn() + document.getElementById('save')!.addEventListener('click', clicked) + + expect(injected(document, holder, { kind: 'click', ref: '@e1' }).success).toBe(true) + expect(clicked).toHaveBeenCalledOnce() + }) }) describe('elements', () => { @@ -73,6 +103,82 @@ describe('elements', () => { expect(inventory(holder).elements?.map(e => e.label)).toEqual(['Real']) }) + // The screen-reader-only recipe is a 1px box parked at the document origin, + // and it passed a `width >= 1` check. Skip links and CSS-only menu toggles are + // built this way and come FIRST in the document, so they landed on @e1 — the + // agent would aim at one and the pointer would fly to the top-left corner and + // click nothing. + it('skips the visually-hidden controls that sit at the document origin', () => { + const holder = page(` + Jump to content + + Skip navigation + + `) + + expect(inventory(holder).elements?.map(e => e.label)).toEqual(['Search']) + }) + + it('skips a control parked off the left edge, which no scroll brings back', () => { + const holder = page(` + + + `) + + expect(inventory(holder).elements?.map(e => e.label)).toEqual(['Search']) + }) + + // Wikipedia and friends are full of these: decorative chrome and collapsed + // menus that are perfectly solid boxes as far as layout is concerned, but + // that the page has already declared are not for anyone to interact with. + it('skips controls the page marked aria-hidden or inert', () => { + const holder = page(` + +
+ + `) + + expect(inventory(holder).elements?.map(e => e.label)).toEqual(['Search']) + }) + + it('skips a control buried under another layer, which would take the click instead', () => { + const holder = page(` + + + `) + const wall = document.createElement('div') + document.body.append(wall) + + // Stand in for the hit-testing jsdom does not do: every element reports + // itself except the buried one, which reports the sheet lying over it. + document.elementFromPoint = (x: number) => (x === 120 ? wall : document.getElementById('real')) + + expect(inventory(holder).elements?.map(e => e.label)).toEqual(['Search']) + }) + + it('keeps a control whose own child is what the hit test lands on', () => { + const holder = page(' Home') + + document.elementFromPoint = () => document.getElementById('icon') + + expect(inventory(holder).elements?.map(e => e.label)).toEqual(['Home']) + }) + + // The overlay draws far more than the agent is told about, so the two lists + // are collected separately: the field is every visible control, the inventory + // is the describable subset that refs point into. + it('parks the whole interactive field for the overlay, not just the inventory', () => { + const holder = page(` + + + + `) + + expect(inventory(holder).elements?.map(e => e.label)).toEqual(['Search']) + expect(holder.nodes).toEqual([document.getElementById('named')]) + expect(holder.field).toEqual([document.getElementById('named'), document.getElementById('mystery')]) + }) + it('prefers an identity selector so the agent can re-find the node later', () => { const holder = page(`
@@ -288,7 +394,23 @@ describe('scroll', () => { const result = actInPage(document, holder, { kind: 'scroll' }) expect(result.success).toBe(true) - expect(scrollBy).toHaveBeenCalledWith({ behavior: 'auto', top: Math.round(window.innerHeight * 0.9) }) + expect(scrollBy).toHaveBeenCalledWith({ behavior: 'smooth', top: Math.round(window.innerHeight * 0.9) }) + }) + + it('drops the animation for a reader who asked for reduced motion', () => { + const holder = page('

long page

') + const scrollBy = vi.spyOn(window, 'scrollBy').mockImplementation(() => {}) + + // Passing 'smooth' explicitly overrides the OS preference, so the engine has + // to opt back out itself rather than leaving it to the browser. + vi.stubGlobal('matchMedia', () => ({ matches: true })) + + actInPage(document, holder, { kind: 'scroll' }) + vi.unstubAllGlobals() + + const [options] = scrollBy.mock.calls[0] as unknown as [ScrollToOptions] + + expect(options.behavior).toBe('auto') }) it('jumps to the bottom', () => { @@ -312,7 +434,7 @@ describe('scroll', () => { const result = actInPage(document, holder, { amount: 200, kind: 'scroll', ref: '@e1' }) - expect(list.scrollBy).toHaveBeenCalledWith({ behavior: 'auto', top: 200 }) + expect(list.scrollBy).toHaveBeenCalledWith({ behavior: 'smooth', top: 200 }) expect(pageScroll).not.toHaveBeenCalled() expect(result.acted).toContain('Results') }) diff --git a/apps/desktop/src/lib/preview-act/act-in-page.ts b/apps/desktop/src/lib/preview-act/act-in-page.ts index f664f58299..0d78fccf2a 100644 --- a/apps/desktop/src/lib/preview-act/act-in-page.ts +++ b/apps/desktop/src/lib/preview-act/act-in-page.ts @@ -35,7 +35,24 @@ export interface PreviewActAction { /** scroll distance in px. Defaults to ~90% of the viewport height. */ amount?: number key?: string - kind: 'click' | 'elements' | 'press' | 'scroll' | 'type' + /** `pin`/`unpin`/`hold` never reach the engine — they resolve their targets + * through `locate`/`elements` and then talk to the overlay — but they arrive + * on the same wire. */ + kind: + | 'click' + | 'elements' + | 'hold' + | 'hover' + | 'locate' + | 'pin' + | 'press' + | 'scroll' + | 'strobe' + | 'type' + | 'unpin' + /** locate: also give the target keyboard focus, for a key press that must not + * be preceded by a click (which would activate the control instead). */ + focus?: boolean /** Cap on the returned inventory. */ max?: number ref?: string @@ -52,8 +69,12 @@ export interface PreviewActResult { elements?: PreviewElement[] error?: string note?: string + /** Viewport centre of a located target, for aiming real pointer input at it. */ + point?: { x: number; y: number } success: boolean title?: string + /** locate: whether the target actually takes typed text. */ + typable?: boolean /** Live document URL after the action — a change means it navigated. */ url?: string } @@ -61,18 +82,34 @@ export interface PreviewActResult { /** Where the surface keeps the last snapshot between actions (a window global * in the preview page), so '@e5' still means something on the next call. */ export interface PreviewActHolder { + /** Target of the action in flight, for the watch overlay to draw onto. */ + aimed?: Element | null + /** The on-screen subset, for the overlay to outline. Diverges from `nodes` in + * both directions: it drops what is below the fold, and it is not capped at + * the inventory's size. */ + field?: Element[] nodes?: Element[] /** URL the snapshot was taken on; a navigation invalidates every ref. */ url?: string } -const ACT_MAX_ELEMENTS = 120 - /** Run one action against `doc`, resolving refs through `holder`. Self-contained. */ export function actInPage(doc: Document, holder: PreviewActHolder, action: PreviewActAction): PreviewActResult { + // Declared inside, not at module scope: this body is stringified and eval'd + // in the guest page, where a module-level constant is simply not defined. + const maxElements = 120 + // How many nodes the OVERLAY may draw, as opposed to how many the agent gets + // told about. Far higher, because an extra mark costs one rect read where an + // extra inventory row costs tokens on every single call. + const maxMarks = 600 const win = doc.defaultView const here = doc.location ? doc.location.href : '' + // Passing 'smooth' explicitly OVERRIDES the user's OS-level reduce-motion + // setting, so ask before animating over someone who opted out. + const still = !!(win && win.matchMedia && win.matchMedia('(prefers-reduced-motion: reduce)').matches) + const glide: ScrollBehavior = still ? 'auto' : 'smooth' + const cssEscape = (value: string) => typeof CSS !== 'undefined' && CSS.escape ? CSS.escape(value) : value.replace(/["\\]/g, '\\$&') @@ -137,17 +174,118 @@ export function actInPage(doc: Document, holder: PreviewActHolder, action: Previ return path.join(' > ') } - const visible = (el: Element): boolean => { + /** On screen right now, and worth drawing a box around. The field is strictly + * viewport-bound: a mark past the fold is one nobody can see, it spends the + * budget that should have gone to what IS on screen, and the ones parked far + * off to the side were landing as stray labels in the corner. */ + const onScreen = (el: Element): boolean => { + if (!win) { + return false + } + + const box = el.getBoundingClientRect() + + if (box.right <= 0 || box.bottom <= 0 || box.left >= win.innerWidth || box.top >= win.innerHeight) { + return false + } + + // Page-sized wrappers. Outlining one draws a rectangle around the whole + // screen, which says nothing and frames everything inside it as if it + // mattered. Full-width banners are fine — it takes both dimensions. + return !(box.width >= win.innerWidth * 0.95 && box.height >= win.innerHeight * 0.9) + } + + /** Painted at all, ancestors included. */ + const shown = (el: Element): boolean => { + // Folds in display/visibility/opacity/content-visibility inherited from an + // ANCESTOR, which this element's own computed style does not report: inside + // a parent at opacity 0, every child still reads back opacity 1. + const seen = (el as Element & { checkVisibility?: (opts: object) => boolean }).checkVisibility + + if (typeof seen === 'function' && !seen.call(el, { checkOpacity: true, checkVisibilityCSS: true })) { + return false + } + const style = win && win.getComputedStyle ? win.getComputedStyle(el) : null - if (style && (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0')) { + if (style) { + if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') { + return false + } + + // The screen-reader-only recipe, both spellings: the deprecated `clip` and + // the modern `clip-path: inset(50%)`. Neither exists for any purpose other + // than hiding something visually while keeping it focusable. + if ((style.clip && style.clip !== 'auto') || (style.clipPath || '').indexOf('inset(50%') !== -1) { + return false + } + } + + return true + } + + const visible = (el: Element): boolean => { + if (!shown(el)) { + return false + } + + // Things the page itself declares are not for interacting with, neither of + // which shows up in a computed style or a bounding box: an aria-hidden + // subtree is invisible to every other assistive client, and `inert` cannot + // be clicked at all. Disabled controls are deliberately NOT filtered here — + // they stay in the inventory so the agent is told a button is disabled + // rather than hunting for one that appears not to exist. + if (el.closest('[aria-hidden="true"], [inert]')) { return false } const rect = el.getBoundingClientRect() - // Off-screen is fine (we scroll to it); collapsed to nothing is not. - return rect.width >= 1 && rect.height >= 1 + // Below the fold is fine — we scroll to it. Off to the LEFT or ABOVE is the + // other half of the sr-only trick (`left: -9999px`, `top: -9999px`) and + // scrolling never brings those back. + if (rect.right <= 0 || rect.bottom <= 0) { + return false + } + + // Bigger than the 1px box sr-only collapses to. Nothing a person can aim at + // is 2px across, and admitting those is what put the cursor in the corner: + // they cluster at the document origin, so they sort to the FRONT of the + // inventory and the agent reaches for one as @e1. + if (rect.width < 3 || rect.height < 3) { + return false + } + + // Occlusion, and the last filter for a reason: everything above is cheap + // and this one forces layout. If the middle of the element is on screen, + // whatever the browser reports at that point has to BE this element — its + // own descendant (an icon inside a link) or the box it paints inside are + // fine, anything else means it is buried under a sticky header, a cookie + // wall, or a transparent layer the page put on top. Those are exactly the + // targets the agent aims at and then misses. Elements below the fold cannot + // be hit-tested from here, so they are taken on trust and land back in this + // function after the scroll. + const midX = rect.left + rect.width / 2 + const midY = rect.top + rect.height / 2 + const under = (doc as Document & { elementFromPoint?: (x: number, y: number) => Element | null }) + .elementFromPoint + + if ( + typeof under === 'function' && + win && + midX >= 0 && + midY >= 0 && + midX < win.innerWidth && + midY < win.innerHeight + ) { + const over = under.call(doc, midX, midY) + + if (!over || !(over === el || el.contains(over) || over.contains(el))) { + return false + } + } + + return true } const valueOf = (el: Element): string => { @@ -166,6 +304,7 @@ export function actInPage(doc: Document, holder: PreviewActHolder, action: Previ const collect = (max: number): PreviewElement[] => { const nodes: Element[] = [] + const field: Element[] = [] const elements: PreviewElement[] = [] const candidates = doc.querySelectorAll( @@ -177,7 +316,22 @@ export function actInPage(doc: Document, holder: PreviewActHolder, action: Previ ) for (const el of candidates) { - if (elements.length >= max || nodes.indexOf(el) !== -1 || !visible(el)) { + if (elements.length >= max && field.length >= maxMarks) { + break + } + + if (!visible(el)) { + continue + } + + // Drawable from here on. The two lists diverge deliberately: the field is + // what is on screen to be outlined, the inventory is what the agent can + // name and reach — which includes things below the fold it will scroll to. + if (field.length < maxMarks && onScreen(el)) { + field.push(el) + } + + if (elements.length >= max) { continue } @@ -212,6 +366,7 @@ export function actInPage(doc: Document, holder: PreviewActHolder, action: Previ } holder.nodes = nodes + holder.field = field holder.url = here return elements @@ -279,7 +434,7 @@ export function actInPage(doc: Document, holder: PreviewActHolder, action: Previ } if (action.kind === 'elements') { - const elements = collect(Math.max(1, Math.min(action.max || ACT_MAX_ELEMENTS, ACT_MAX_ELEMENTS))) + const elements = collect(Math.max(1, Math.min(action.max || maxElements, maxElements))) return answer({ elements, @@ -300,9 +455,9 @@ export function actInPage(doc: Document, holder: PreviewActHolder, action: Previ const by = action.to === 'top' ? -1e7 : action.to === 'bottom' ? 1e7 : (action.amount ?? page) if (scroller) { - scroller.scrollBy({ behavior: 'auto', top: by }) + scroller.scrollBy({ behavior: glide, top: by }) } else if (win) { - win.scrollBy({ behavior: 'auto', top: by }) + win.scrollBy({ behavior: glide, top: by }) } return answer({ acted: scroller ? 'scrolled ' + describe(scroller) : 'scrolled the page', success: true }) @@ -316,12 +471,63 @@ export function actInPage(doc: Document, holder: PreviewActHolder, action: Previ const el = target.el as HTMLElement - if ((el as HTMLInputElement).disabled) { - return fail(describe(el) + ' is disabled.') + // Park the target where the watch overlay can find it, before anything can + // fail — a ring around the thing that turned out to be disabled is exactly + // the feedback someone watching wants. + holder.aimed = el + + // 'locate' is the look-before-you-act half of an action: it brings the target + // on screen and names it, so the overlay has something to draw while the real + // verb is still a beat away. + if (action.kind === 'locate') { + if (el.scrollIntoView) { + el.scrollIntoView({ behavior: 'instant', block: 'center', inline: 'nearest' }) + } + + if (action.focus) { + el.focus() + } + + const spot = pointAt(el) + const tag = el.tagName + + return answer({ + acted: 'looking at ' + describe(el), + point: { x: spot.clientX, y: spot.clientY }, + success: true, + // Real typing starts with a triple-click to clear the field. On anything + // that is not a field that gesture selects the paragraph under it + // instead, which is how the agent ended up highlighting whole pages. + typable: tag === 'TEXTAREA' || tag === 'INPUT' || el.isContentEditable === true + }) } + // Instant, unlike the `scroll` verb above. Getting a target on screen is + // plumbing for the click that follows, not something anyone asked to watch — + // and every millisecond of it is time the caller spends waiting for the page + // to stop moving before it can aim real input at a fixed coordinate. if (el.scrollIntoView) { - el.scrollIntoView({ block: 'center', inline: 'nearest' }) + el.scrollIntoView({ behavior: 'instant', block: 'center', inline: 'nearest' }) + } + + // Hovering a disabled control is allowed — a disabled button with a tooltip + // explaining WHY is exactly the thing worth hovering. + if (action.kind === 'hover') { + const init = { bubbles: true, cancelable: true, ...pointAt(el) } + + if (typeof PointerEvent === 'function') { + el.dispatchEvent(new PointerEvent('pointerover', init)) + el.dispatchEvent(new PointerEvent('pointermove', init)) + } + + el.dispatchEvent(new MouseEvent('mouseover', init)) + el.dispatchEvent(new MouseEvent('mousemove', init)) + + return answer({ acted: 'hovered over ' + describe(el), success: true }) + } + + if ((el as HTMLInputElement).disabled) { + return fail(describe(el) + ' is disabled.') } if (action.kind === 'click') { diff --git a/apps/desktop/src/lib/preview-act/watch-in-page.test.ts b/apps/desktop/src/lib/preview-act/watch-in-page.test.ts new file mode 100644 index 0000000000..b51c62b7f0 --- /dev/null +++ b/apps/desktop/src/lib/preview-act/watch-in-page.test.ts @@ -0,0 +1,363 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { type WatchHolder, watchInPage } from './watch-in-page' + +/** jsdom has no layout engine, so every rect is zero and nothing would be + * positioned. Give elements a plausible box the way the act-engine tests do. */ +beforeEach(() => { + // Never run the callback: the tracking loop re-arms itself every frame, so a + // stub that invokes synchronously recurses until the stack gives out. The + // first pass happens on the direct call, which is the one worth asserting on. + vi.stubGlobal('requestAnimationFrame', () => 1) + vi.stubGlobal('cancelAnimationFrame', () => {}) + + Element.prototype.getBoundingClientRect = function () { + return { bottom: 140, height: 40, left: 100, right: 300, toJSON: () => ({}), top: 100, width: 200, x: 100, y: 100 } + } + + document.body.replaceChildren() + // The host hangs off documentElement, so clearing body does not reach it. + document.querySelector('hermes-watch')?.remove() + delete (window as unknown as Record).__hermesWatch +}) + +/** The overlay lives in a closed shadow root, so a test can only see the host. */ +const host = () => document.querySelector('hermes-watch') + +/** …except through the parts the engine parks on the window for its own reuse, + * which is the only way to inspect what the overlay actually drew. */ +const drawn = () => + (window as unknown as { __hermesWatch: { parts: Record & { shadow: ShadowRoot } } }) + .__hermesWatch.parts + +/** Everything the overlay drew for this action. The cursor is the only fixed + * layer — every box and pin is a mark that comes and goes. */ +const marks = () => [...drawn().shadow.children].filter(child => child !== drawn().pointer) + +describe('watchInPage', () => { + it('mounts one host on documentElement and reuses it across stages', () => { + const target = document.createElement('button') + document.body.append(target) + + const holder: WatchHolder = { aimed: target } + + watchInPage(document, holder, 'aim') + watchInPage(document, holder, 'strike') + + expect(document.querySelectorAll('hermes-watch')).toHaveLength(1) + expect(host()?.parentElement).toBe(document.documentElement) + }) + + it('stays out of the page, out of the way, and out of the a11y tree', () => { + const target = document.createElement('button') + document.body.append(target) + + watchInPage(document, { aimed: target }, 'aim') + + const node = host() as HTMLElement + + // pointer-events keeps real clicks reaching the page; aria-hidden keeps a + // screen reader from announcing decoration. + expect(node.style.pointerEvents).toBe('none') + expect(node.getAttribute('aria-hidden')).toBe('true') + // The closed shadow root is what keeps the overlay out of the act engine's + // own querySelectorAll inventory — there is nothing to exclude by hand. + expect(node.shadowRoot).toBeNull() + }) + + it('does nothing when there is no target to draw on', () => { + watchInPage(document, {}, 'aim') + watchInPage(document, { aimed: document.createElement('div') }, 'strike') + + expect(host()).toBeNull() + }) + + // The cursor is placed, never interpolated. Beyond reading as a machine, a + // transition here was an outright bug on the first paint: an unset transform + // IS the origin, so the browser flew the cursor in from the top-left corner — + // and every navigation is a fresh document, so a fresh first paint. + it('cuts the marks straight to the target instead of travelling to it', () => { + const target = document.createElement('button') + document.body.append(target) + + watchInPage(document, { aimed: target }, 'aim', 'Clicking Save') + + const { pointer } = drawn() + + expect(pointer.style.transition).toBeFalsy() + expect((marks()[0] as HTMLElement).style.transition).toBeFalsy() + // Mocked rect is 200×40 at (100, 100), so the cursor belongs at its centre. + expect(pointer.style.transform).toBe('translate3d(200px,120px,0)') + }) + + // The inventory pass is the one moment the agent is reading the whole page + // rather than aiming at one thing, so the overlay has to be able to hold many + // marks at once — not just the single ring every other stage draws. + it('sweep lights up every catalogued element and holds them well past the sweep', () => { + vi.useFakeTimers() + + const found = [document.createElement('button'), document.createElement('a'), document.createElement('input')] + document.body.append(...found) + + watchInPage(document, { nodes: found }, 'sweep') + expect(marks()).toHaveLength(found.length) + + // The point of the dwell: the grid is still up long after the pass that drew + // it finished, so there is something to actually look at. jsdom has no Web + // Animations, so this exercises the timed fallback, which holds for the same + // beat as the animated path. + vi.advanceTimersByTime(600) + expect(marks()).toHaveLength(found.length) + + // …and then drains scattered rather than blinking off as one block, which is + // the whole reason each box carries its own offset. Stepping through the + // exit must therefore see the count come down in more than one tick. + const drain = new Set() + + for (let t = 0; t < 7_000; t += 100) { + vi.advanceTimersByTime(100) + drain.add(marks().length) + } + + expect(drain.size).toBeGreaterThan(2) + expect(marks()).toHaveLength(0) + + vi.useRealTimers() + }) + + // jsdom cannot run a Web Animation, but it can be handed one and asked what + // it was given — and this particular option is worth pinning, because getting + // it wrong is invisible rather than broken. `easing` in the options bag is the + // ITERATION easing: it rewrites progress before any keyframe is consulted, so + // a step function there holds progress at 0 for the whole duration. Every box + // was created, animated and removed without painting a single frame. + it('does not step the sweep’s iteration easing, which would stop it painting', () => { + const timings: KeyframeAnimationOptions[] = [] + + Element.prototype.animate = vi.fn(function (this: Element, _frames, options) { + timings.push(options as KeyframeAnimationOptions) + + return { cancel: () => {}, finish: () => {} } as unknown as Animation + }) as unknown as Element['animate'] + + const found = [document.createElement('button'), document.createElement('a')] + document.body.append(...found) + + watchInPage(document, { field: found }, 'sweep') + + // The cursor's idle sway is permanent and starts with the overlay, so it is + // not one of the sweep's own animations. + const sweep = timings.filter(timing => timing.iterations !== Infinity) + + expect(sweep).toHaveLength(found.length) + + for (const timing of sweep) { + expect(String(timing.easing)).not.toMatch(/steps/) + } + }) + + it('names each swept box after the element under it, not its ref', () => { + const found = [document.createElement('button'), document.createElement('a'), document.createElement('input')] + + found[0].id = 'buy' + found[1].className = 'nav wide' + found[2].setAttribute('type', 'search') + document.body.append(...found) + + // Every drawable box gets named, addressable or not — a ref number only ever + // meant something for the subset that had one, and said nothing about what + // the element actually is. + watchInPage(document, { field: found, nodes: found.slice(0, 2) }, 'sweep') + + const { shadow } = drawn() + const cells = [...shadow.children].slice(-found.length) + + expect(cells.map(cell => cell.textContent)).toEqual(['button#buy', 'a.nav', 'input[search]']) + }) + + // The case that made positions necessary: a list of links with no id, no + // class and no type. Naming them by tag alone gives a whole page of boxes all + // reading "a", which is what the ref numbers were replaced for. + it('falls back to position so anonymous siblings are still told apart', () => { + const row = document.createElement('td') + + row.className = 'subtext' + row.innerHTML = '' + document.body.append(row) + + const found = [...row.children] + + watchInPage(document, { field: found }, 'sweep') + + const { shadow } = drawn() + const cells = [...shadow.children].slice(-found.length) + + expect(cells.map(cell => cell.textContent)).toEqual(['a[1]', 'a[2]', 'a[3]']) + }) + + it('keeps a name short enough to sit on the box it names', () => { + const found = [document.createElement('div')] + + found[0].id = 'a-genuinely-enormous-generated-identifier' + document.body.append(...found) + + watchInPage(document, { field: found }, 'sweep') + + const name = marks()[0].textContent as string + + expect(name.length).toBeLessThanOrEqual(16) + expect(name.startsWith('div#a-')).toBe(true) + expect(name.endsWith('…')).toBe(true) + }) + + // A field past a couple of dozen boxes is a texture, not a list. Naming every + // one buries the outlines — which ARE the readout at that size — under a wall + // of tags that collide with each other and overhang their own rectangles. + it('stops naming the boxes once the field is too dense to read', () => { + const few = Array.from({ length: 4 }, () => document.createElement('button')) + document.body.append(...few) + + watchInPage(document, { field: few }, 'sweep') + expect(marks().every(mark => mark.textContent)).toBe(true) + + document.querySelector('hermes-watch')?.remove() + delete (window as unknown as Record).__hermesWatch + + const many = Array.from({ length: 40 }, () => document.createElement('button')) + document.body.append(...many) + + watchInPage(document, { field: many }, 'sweep') + expect(marks().some(mark => mark.textContent)).toBe(false) + }) + + // Every other mark on the page wears its label on the top edge. A box against + // the top of the viewport has nowhere to put one, and a tab drawn off-screen + // is a box that looks unlabelled next to boxes that aren't. + it('flips the label under the box when there is no headroom above it', () => { + const roomy = document.createElement('button') + const flush = document.createElement('button') + const huge = document.createElement('button') + + flush.getBoundingClientRect = () => + ({ bottom: 40, height: 40, left: 100, right: 300, toJSON: () => ({}), top: 0, width: 200, x: 100, y: 0 }) as DOMRect + // Taller than the viewport: no room above it and none below it either, so + // the tab has nowhere left to go but inside. + huge.getBoundingClientRect = () => + ({ bottom: 900, height: 900, left: 100, right: 300, toJSON: () => ({}), top: 0, width: 200, x: 100, y: 0 }) as DOMRect + document.body.append(roomy, flush, huge) + + watchInPage(document, { field: [roomy, flush, huge] }, 'sweep') + + const tags = marks().map(mark => mark.querySelector('div:last-child') as HTMLElement) + + expect(tags[0].style.bottom).toBe('100%') + expect(tags[1].style.top).toBe('100%') + expect(tags[2].style.top).toBe('0px') + }) + + // A label is routinely wider than the box it names, so it is the boxes near the + // right edge that push their tab off screen. Whichever end it lands on, the + // tab's edge sits exactly on the outline's — the two are the same colour and + // meet at a corner, so a pixel of overhang reads as a misprint. + it('hangs the label off the right edge when a left-anchored one would run off', () => { + Object.defineProperty(HTMLElement.prototype, 'offsetWidth', { configurable: true, get: () => 80 }) + + const inboard = document.createElement('button') + const edge = document.createElement('button') + + edge.getBoundingClientRect = () => + ({ bottom: 140, height: 40, left: 990, right: 1020, toJSON: () => ({}), top: 100, width: 30, x: 990, y: 100 }) as DOMRect + document.body.append(inboard, edge) + + watchInPage(document, { field: [inboard, edge] }, 'sweep') + + const tags = marks().map(mark => mark.querySelector('div:last-child') as HTMLElement) + + expect(tags[0].style.left).toBe('0px') + expect(tags[0].style.right).toBe('') + expect(tags[1].style.right).toBe('0px') + expect(tags[1].style.left).toBe('') + }) + + it('sweep draws nothing when the inventory came back empty', () => { + watchInPage(document, { nodes: [] }, 'sweep') + + expect(host()).toBeNull() + }) + + // The field is swept after every action, and a sweep retires the lock. The + // cursor must not go with it — a hand that blinks out a fraction of a second + // after every single step reads as the agent having wandered off. + it('keeps the cursor up when a sweep retires the lock', () => { + const target = document.createElement('button') + const found = [document.createElement('a')] + document.body.append(target, ...found) + + watchInPage(document, { aimed: target }, 'aim') + + const { pointer } = drawn() + const lock = marks()[0] + + expect(pointer.style.opacity).toBe('1') + + watchInPage(document, { field: found }, 'sweep') + + expect(lock.isConnected).toBe(false) + expect(pointer.style.opacity).toBe('1') + }) + + // The chrome is memoized for the life of the page, so a style change to it + // would otherwise be injected, run, and restyle nothing — the layers it + // describes were built by the version before. Reads as "HMR isn't picking my + // CSS up" in dev, and as a long-lived tab stuck on an old build in production. + it('rebuilds its chrome when the injected source changes underneath it', () => { + const target = document.createElement('button') + document.body.append(target) + + const stamp = (value: number) => { + ;(window as unknown as Record).__hermesWatchTag = value + } + + stamp(1) + watchInPage(document, { aimed: target }, 'aim') + + const first = drawn().host + + stamp(1) + watchInPage(document, { aimed: target }, 'aim') + + expect(drawn().host).toBe(first) + + stamp(2) + watchInPage(document, { aimed: target }, 'aim') + + expect(drawn().host).not.toBe(first) + // …and the old one goes with it, rather than stacking a second overlay. + expect(document.querySelectorAll('hermes-watch')).toHaveLength(1) + }) + + it('clear releases the target so the tracking loop can stop', () => { + const target = document.createElement('button') + document.body.append(target) + + const holder: WatchHolder = { aimed: target } + + watchInPage(document, holder, 'aim') + watchInPage(document, holder, 'clear') + + expect(holder.aimed).toBeNull() + }) + + // Same contract as the act engine: the pane injects `watchInPage.toString()` + // into the guest page, where module scope does not exist. One free identifier + // is a ReferenceError that Electron reports only as "Script failed to execute". + it('runs after being stringified and eval’d with no module scope', () => { + const target = document.createElement('button') + document.body.append(target) + + const injected = new Function('return (' + watchInPage.toString() + ')')() as typeof watchInPage + + expect(() => injected(document, { aimed: target }, 'aim', 'Clicking Save')).not.toThrow() + expect(host()).not.toBeNull() + }) +}) diff --git a/apps/desktop/src/lib/preview-act/watch-in-page.ts b/apps/desktop/src/lib/preview-act/watch-in-page.ts new file mode 100644 index 0000000000..0395548d0d --- /dev/null +++ b/apps/desktop/src/lib/preview-act/watch-in-page.ts @@ -0,0 +1,1063 @@ +/** + * WATCH OVERLAY — paints the agent's actions onto the live preview page so a + * person can follow along: the field of things it can reach, a box round the + * one it is about to touch, and a cursor that goes there. + * + * There are exactly two things on screen, and everything the agent does is said + * with one of them: + * + * - THE CURSOR — one arrow, built once, never removed. It is the + * collaborative-editing kind, because that is exactly what this is: a second + * party moving around a document you are also looking at. Arrow geometry is + * Liveblocks' (Apache-2.0). Plain black or white against the guest's own + * colour scheme rather than a brand colour: it stands for a hand, and a hand + * is not a feature of this app. + * - THE MARK — a rectangle glued to an element. A candidate, a lock, a hit + * and a pin are all the same object with a different row in `kinds`: how + * heavy the rule is, how long it lives, how it arrives. Marks are a design + * tool's selection blue, deliberately NOT the + * app's accent — this is chrome drawn on top of someone else's page, and + * every design tool has trained people to read that blue as "a tool has + * selected this", where a branded colour would read as part of the page. + * + * Every mark is two elements: a shell the tracker positions, and a skin that + * carries the border and any animation. They are split because both want + * `transform`, and a mark that animates the property the tracker rewrites every + * frame either stops moving or stops animating. + * + * Injected as source (`watchInPage.toString()`) exactly like the act engine, so + * the same rule holds: no imports, no closure references, no module-scope + * constants — everything the body needs is declared inside it. + * + * Three constraints shape the implementation, each one learned from how + * Playwright, Stagehand and browser-use solve the same problem: + * + * - A page can set `z-index: 2147483647` too, and the browser's top layer + * (``, `popover`, fullscreen) outranks every z-index there is. So + * the host joins the top layer via `popover="manual"` instead of trying to + * outbid the page, and keeps the z-index only as a fallback. + * - A strict CSP rejects `style-src` at CSS PARSE time, which kills + * `cssText`, `setAttribute('style', …)` and any `