From f262e684d80119c8aeeaeae087b83bf0e4961f18 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Thu, 20 Aug 2026 02:32:56 -0500 Subject: [PATCH] feat(desktop): an overlay that shows what the agent is doing to the page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Driving someone's browser invisibly is unnerving, and this browser is the one they are signed into. The pane now draws the field the agent can reach, a box round what it is touching, a cursor that goes there, and a wipe over text it just read — one cursor primitive and one mark primitive, in a closed shadow root so the agent's own inventory cannot see them. Marks carry the same handle the agent addresses them by, so the word on screen and the word in the transcript are the same string. The point is supervision rather than decoration: a person glancing at the pane can tell what is about to happen to their live session, and stop it. It also has to cover the waiting. The agent flashes through a click in under a second and then sits idle for the twenty to a hundred seconds the model spends deciding what to do next, which is most of the wall clock of any task — so the surface used to look broken during the part where it was working hardest. A think stage runs off the $busy edge, sparsely flashing elements from the field the last action left behind, and rest stops it. It guards itself: started before there is an overlay or a field, it idles until there is one, so it can be raised on the turn boundary without knowing whether anything has been inventoried yet. read_preview had the same hole from the other side. Reading is the cheapest thing the agent does — hundredths of a second between two model round trips — so paging through a document left the pane dark for twenty seconds immediately after the one moment that showed anything. It draws a top-to-bottom wipe over the text it took, and that is the one stage allowed to be a wipe: reading is the only thing the agent does to a page in an order a person could follow. Both go through preview-nudge, which says a single stage to an overlay the page already has rather than re-shipping the engine to narrate. On a page the agent never acted on it is a no-op, which is the honest answer — chrome there would be a lie about what it did. Everything respects prefers-reduced-motion. --- .../app/chat/right-rail/preview-act.test.ts | 189 ++- .../src/app/chat/right-rail/preview-act.ts | 529 +++++++- .../src/app/chat/right-rail/preview-mind.ts | 29 + .../src/app/chat/right-rail/preview-nudge.ts | 38 + .../src/app/chat/right-rail/preview-pane.tsx | 6 +- .../src/app/chat/right-rail/preview-reader.ts | 9 + .../gateway-event/desktop-bridge.ts | 36 +- .../src/lib/preview-act/act-in-page.test.ts | 134 ++- .../src/lib/preview-act/act-in-page.ts | 234 +++- .../src/lib/preview-act/watch-in-page.test.ts | 363 ++++++ .../src/lib/preview-act/watch-in-page.ts | 1063 +++++++++++++++++ 11 files changed, 2574 insertions(+), 56 deletions(-) create mode 100644 apps/desktop/src/app/chat/right-rail/preview-mind.ts create mode 100644 apps/desktop/src/app/chat/right-rail/preview-nudge.ts create mode 100644 apps/desktop/src/lib/preview-act/watch-in-page.test.ts create mode 100644 apps/desktop/src/lib/preview-act/watch-in-page.ts diff --git a/apps/desktop/src/app/chat/right-rail/preview-act.test.ts b/apps/desktop/src/app/chat/right-rail/preview-act.test.ts index b53e046342..2b68b1eb99 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-act.test.ts +++ b/apps/desktop/src/app/chat/right-rail/preview-act.test.ts @@ -4,6 +4,7 @@ import { $rightRailActiveTabId } from '@/store/layout' import { closeRightRail, openPreview, type PreviewTarget } from '@/store/preview' import { actOnActivePreview } from './preview-act' +import { registerPreviewInput } from './preview-input' import { registerPreviewNav } from './preview-nav' import { registerPreviewScriptRunner } from './preview-script-runner' @@ -11,7 +12,7 @@ function urlTarget(url: string): PreviewTarget { return { kind: 'url', label: 'Browser', source: url, url } } -describe('actOnActivePreview (act_preview tool)', () => { +describe('actOnActivePreview (drive_preview tool)', () => { // URL targets share the singleton Browser tab id, so anything a test // registers would answer the next one. let cleanups: Array<() => void> = [] @@ -105,6 +106,192 @@ describe('actOnActivePreview (act_preview tool)', () => { expect((await actOnActivePreview({ kind: 'click', ref: '@e1' })).error).toContain('did not answer') }) + /** A pane that answers the locate trip with a fixed on-screen point, and the + * read-back trip with an empty inventory. Returns the input spy. */ + const withDrivenPane = () => { + const tabId = openBrowserTab() + const send = vi.fn() + + cleanups.push( + registerPreviewScriptRunner(tabId, async code => + code.includes('"kind":"locate"') + ? JSON.stringify({ acted: 'looking at button "Save"', point: { x: 120, y: 80 }, success: true }) + : // `hit` is the page's witness that the real pointerdown arrived. + JSON.stringify({ elements: [], hit: { tag: 'BUTTON', trusted: true }, success: true }) + ) + ) + cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send })) + + return send + } + + const sentTypes = (send: ReturnType) => send.mock.calls.map(([event]) => event.type) + + it('clicks with real input, walking the pointer to where the page said', async () => { + const send = withDrivenPane() + + const result = await actOnActivePreview({ kind: 'click', ref: '@e1' }) + const types = sentTypes(send) + + // Stepped rather than teleported: a page learns it is hovered from a stream + // of moves, and one jump to the target skips everything in between. + expect(types.filter(type => type === 'mouseMove').length).toBeGreaterThan(1) + expect(types).toContain('mouseDown') + expect(types).toContain('mouseUp') + expect(send.mock.calls.map(([event]) => event).find(event => event.type === 'mouseDown')).toMatchObject({ + x: 120, + y: 80 + }) + expect(result.acted).toBe('clicked button "Save"') + }) + + it('types by pressing keys, after selecting whatever the field held', async () => { + const send = withDrivenPane() + + await actOnActivePreview({ kind: 'type', ref: '@e1', submit: true, text: 'hi' }) + + const events = send.mock.calls.map(([event]) => event) + const chars = events.filter(event => event.type === 'char').map(event => event.keyCode) + + // One click to focus, then select-all by keyboard — NOT the triple-click + // this used to do. A triple-click is a pointer gesture, so it grabs the + // paragraph under the cursor whenever the target turns out not to be a + // field, and the agent was leaving pages with their body text highlighted. + expect(events.filter(event => event.type === 'mouseDown').map(event => event.clickCount)).toEqual([1]) + expect(events.filter(event => event.type === 'keyDown' && event.keyCode === 'a')[0]).toMatchObject({ + modifiers: ['control', 'meta'] + }) + // The chord must not send a `char` phase, or select-all types a literal 'a'. + expect(chars).toEqual(['h', 'i', 'Enter']) + }) + + it('hovers by walking the pointer over and leaving it there', async () => { + const send = withDrivenPane() + + const result = await actOnActivePreview({ kind: 'hover', ref: '@e1' }) + const types = sentTypes(send) + + expect(types).toContain('mouseMove') + // The whole request is "be on it" — a click here would open the dropdown the + // agent was trying to reveal, or worse, activate it. + expect(types).not.toContain('mouseDown') + expect(result.acted).toBe('hovered over button "Save"') + }) + + // The witness only speaks for verbs that put the button down. Demanding one + // from a key press reported every press as a failure. + it('does not expect a click witness from a verb that never clicks', async () => { + const tabId = openBrowserTab() + + cleanups.push( + registerPreviewScriptRunner(tabId, async code => + code.includes('"kind":"locate"') + ? JSON.stringify({ acted: 'looking at textbox "Search"', point: { x: 40, y: 20 }, success: true }) + : JSON.stringify({ elements: [], hit: null, success: true }) + ) + ) + cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send: vi.fn() })) + + for (const action of [ + { key: 'Escape', kind: 'press', ref: '@e1' }, + { kind: 'hover', ref: '@e1' } + ]) { + expect(await actOnActivePreview(action)).toMatchObject({ success: true }) + } + }) + + it('scrolls by wheeling for real, not by scripting the page', async () => { + const tabId = openBrowserTab() + const send = vi.fn() + let scripted = false + + cleanups.push( + registerPreviewScriptRunner(tabId, async code => { + scripted ||= code.includes('"kind":"scroll"') + + return code.includes('scrollHeight') + ? JSON.stringify({ page: 700, point: { x: 500, y: 400 }, span: 4_000, success: true }) + : JSON.stringify({ elements: [], success: true }) + }) + ) + cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send })) + + await actOnActivePreview({ kind: 'scroll' }) + + const wheels = send.mock.calls.map(([event]) => event).filter(event => event.type === 'mouseWheel') + + // A stream of notches, not one delta: scroll-linked headers and lazy loaders + // only react to the events, so a scripted scrollBy leaves them asleep. + expect(wheels.length).toBeGreaterThan(1) + // Electron's wheel delta is wheelDelta-signed, so scrolling DOWN is negative. + expect(wheels.every(event => event.deltaY < 0)).toBe(true) + expect(scripted).toBe(false) + }) + + it('says so plainly when the page has nothing to scroll', async () => { + const tabId = openBrowserTab() + + cleanups.push( + registerPreviewScriptRunner(tabId, async () => + JSON.stringify({ page: 700, point: { x: 500, y: 400 }, span: 0, success: true }) + ) + ) + cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send: vi.fn() })) + + expect(await actOnActivePreview({ kind: 'scroll' })).toMatchObject({ note: expect.stringContaining('nothing to scroll') }) + }) + + it('fails loudly when the pointer input never reaches the page', async () => { + const tabId = openBrowserTab() + + cleanups.push( + registerPreviewScriptRunner(tabId, async code => + code.includes('"kind":"locate"') + ? JSON.stringify({ acted: 'looking at button "Save"', point: { x: 12, y: 8 }, success: true }) + : JSON.stringify({ elements: [], hit: null, success: true }) + ) + ) + cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send: vi.fn() })) + + const result = await actOnActivePreview({ kind: 'click', ref: '@e1' }) + + // Everything else about this action travels on the script channel and would + // report success whether or not a single event landed. + expect(result.success).toBe(false) + expect(result.error).toContain('never reached the page') + }) + + it('says so when the overlay itself swallowed the click', async () => { + const tabId = openBrowserTab() + + cleanups.push( + registerPreviewScriptRunner(tabId, async code => + code.includes('"kind":"locate"') + ? JSON.stringify({ acted: 'looking at button "Save"', point: { x: 12, y: 8 }, success: true }) + : JSON.stringify({ elements: [], hit: { tag: 'HERMES-WATCH', trusted: true }, success: true }) + ) + ) + cleanups.push(registerPreviewInput(tabId, { focus: vi.fn(), send: vi.fn() })) + + expect((await actOnActivePreview({ kind: 'click', ref: '@e1' })).note).toContain('overlay intercepted') + }) + + it('falls back to scripted events when the pane exposes no input channel', async () => { + let injected = '' + + withRunner(async code => { + injected = code + + return JSON.stringify({ acted: 'clicked', success: true }) + }) + + await actOnActivePreview({ kind: 'click', ref: '@e1' }) + + // The one-trip shape: the engine both acts and re-reads, no locate handshake. + expect(injected).toContain('"kind":"click"') + expect(injected).not.toContain('"kind":"locate"') + }) + it('routes history verbs to the pane instead of the guest page', async () => { const back = vi.fn() const runner = vi.fn() diff --git a/apps/desktop/src/app/chat/right-rail/preview-act.ts b/apps/desktop/src/app/chat/right-rail/preview-act.ts index 495ce45f0a..bcef505fbc 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-act.ts +++ b/apps/desktop/src/app/chat/right-rail/preview-act.ts @@ -1,14 +1,23 @@ /** * PREVIEW ACT — performs the agent's interactions inside the preview pane's - * guest page, so `act_preview` drives whatever web app is open in the in-app + * guest page, so `drive_preview` drives whatever web app is open in the in-app * browser. * * The guest page is out-of-process; nothing here can touch its DOM directly. - * Instead each action injects the engine SOURCE over `executeJavaScript` (see - * lib/preview-act/act-in-page.ts's self-containment contract), parked on a - * window global alongside the ref holder so '@e5' still resolves on the next - * call. Injection is idempotent and vanishes with the page — a navigation - * drops the refs, which the engine reports rather than clicking the wrong node. + * Two separate channels reach it, and the split matters: + * + * - `executeJavaScript` injects the engine SOURCE (see + * lib/preview-act/act-in-page.ts's self-containment contract) to RESOLVE + * and READ — turn '@e5' into a node, measure it, inventory the page. It is + * parked on a window global alongside the ref holder so refs survive + * between calls, and it vanishes with the page, so a navigation drops the + * refs and the engine reports that instead of clicking the wrong node. + * - `sendInputEvent` (preview-drive.ts) does the ACTING, as real Chromium + * input. Script can only dispatch synthetic events, which the page can tell + * apart and which never move the browser's own hover or focus target. + * + * Panes with no input channel — a remote HTML preview — fall back to the + * engine's synthetic events, which is worse but still works. * * A mutating action settles briefly and returns a fresh inventory, so the * click → re-read → click loop costs one round trip instead of two. @@ -18,46 +27,461 @@ */ import { actInPage, type PreviewActAction, type PreviewActResult } from '@/lib/preview-act/act-in-page' +import { watchInPage } from '@/lib/preview-act/watch-in-page' +import { clickAt, glideTo, pointerPlaced, pressKey, selectAll, typeText, wheelBy } from './preview-drive' +import { activePreviewInput, type PreviewInputHandle } from './preview-input' import { activePreviewNav, type PreviewNavHandle } from './preview-nav' -import { activePreviewScriptRunner } from './preview-script-runner' +import { activePreviewScriptRunner, type PreviewScriptRunner } from './preview-script-runner' /** Verbs the pane owns; a guest page cannot drive its own history. */ const NAV_ACTIONS: readonly (keyof PreviewNavHandle)[] = ['back', 'forward', 'reload'] /** How long a click/type is given to land before the page is re-inventoried. - * Long enough for a framework re-render, short enough not to stall the turn. */ -const SETTLE_MS = 400 + * Long enough for a framework re-render, short enough not to stall the turn. + * A render that misses this window is not lost — the NEXT action re-reads the + * page anyway, so the cost of being early is a slightly stale inventory, not a + * wrong one. That trade is worth several hundred ms on every single step. */ +const SETTLE_MS = 220 + +/** Cap on one round trip into the guest page. A click that starts a navigation + * tears the document down mid-settle, so the injected promise dies with it and + * `executeJavaScript` never settles — without this the tool eats its whole + * 45s bridge deadline for what was actually a successful click. */ +const ACT_TIMEOUT_MS = 8_000 + +/** Verbs that get the look-then-act treatment: locate the target, walk the + * pointer there, then send real input. Actions with no single target (scroll, + * elements) are absent and skip it. */ +const DRIVEN: readonly string[] = ['click', 'hover', 'press', 'type'] + +/** Verbs that put the mouse button down, and so are the only ones the pointerdown + * witness can speak to. `press` and `hover` never click, so demanding a witness + * from them would report every one of them as a failure. */ +const CLICKS: readonly string[] = ['click', 'type'] const NOTHING_OPEN = 'No live page is open in the in-app browser — open one with open_preview first.' -/** Build the idempotent inject-and-run script for one action. */ -function buildActScript(action: PreviewActAction, settleMs: number): string { +const NAVIGATED = 'The page stopped answering right after — it is probably navigating. Call elements to see where you landed.' + +/** A fingerprint of the overlay's source, so the guest page can tell that the + * code it is running has changed underneath it. + * + * It needs one because the overlay memoizes its own chrome: the cursor, the + * lock frame and the scan line are built ONCE per page and then reused, so an + * edit to any of their styles lands in the injected source, gets injected, and + * changes nothing — the layers it would have styled were built by the previous + * version and are still on the page. In dev that reads as "HMR didn't pick up + * my CSS"; in production it is a long-lived tab pinned to whichever build first + * touched it, across an app upgrade. + * + * Content-addressed rather than a length or a version literal: the change that + * exposed this was one hex colour for another, which is byte-for-byte the same + * length, and a hand-maintained version is a step someone forgets. */ +const WATCH_TAG = (() => { + const source = watchInPage.toString() + let hash = 5381 + + for (let i = 0; i < source.length; i++) { + hash = ((hash << 5) + hash + source.charCodeAt(i)) | 0 + } + + return hash +})() + +/** Injected once per page: the engine, the overlay, and the two helpers every + * script below shares. Idempotent — re-running it on a live page is a no-op. */ +const preamble = () => ` var w = window; + // Reassigned on every trip rather than cached behind a guard: the source is in + // this payload either way, so the guard saved nothing and pinned a long-lived + // tab to whichever build first touched it. The holder is the exception — it + // carries the aimed element from the locate trip to the act trip. + w.__hermesActHolder = w.__hermesActHolder || {}; + w.__hermesAct = (${actInPage.toString()}); + w.__hermesWatch_fn = (${watchInPage.toString()}); + w.__hermesWatchTag = ${WATCH_TAG}; + var holder = w.__hermesActHolder; + var act = function (a) { return w.__hermesAct(document, holder, a); }; + var watch = function (stage, label) { + try { w.__hermesWatch_fn(document, holder, stage, label); } catch (err) {} + }; + var wait = function (ms) { return new Promise(function (resolve) { setTimeout(resolve, ms); }); }; + // Sit out a smooth scroll so the pointer is aimed at a target that has stopped + // moving. A scroll that never started fires no scrollend, so only wait on one + // we can actually see happening; capture catches nested scrollers, whose + // scrollend does not bubble. + var restAfterScroll = function () { + return new Promise(function (resolve) { + var sc = document.scrollingElement || document.documentElement; + var x0 = sc ? sc.scrollLeft : 0; + var y0 = sc ? sc.scrollTop : 0; + requestAnimationFrame(function () { + if (!sc || (sc.scrollLeft === x0 && sc.scrollTop === y0)) { return resolve(); } + var done = false; + var finish = function () { + if (done) { return; } + done = true; + document.removeEventListener('scrollend', finish, true); + clearTimeout(timer); + resolve(); + }; + var timer = setTimeout(finish, 350); + document.addEventListener('scrollend', finish, true); + }); + }); + };` + +/** Bring the target on screen, mark it, and report where it came to rest. */ +function buildLocateScript(action: PreviewActAction, focus: boolean): string { + const locate = { focus, kind: 'locate', ref: action.ref, selector: action.selector } + return `(function () { - var w = window; - if (!w.__hermesAct) { - w.__hermesActHolder = {}; - w.__hermesAct = (${actInPage.toString()}); - } - var act = function (a) { return w.__hermesAct(document, w.__hermesActHolder, a); }; +${preamble()} + var locate = ${JSON.stringify(locate)}; + var found = act(locate); + if (!found.success) { return Promise.resolve(JSON.stringify(found)); } + watch('aim'); + // Arm a witness for the real input that is about to arrive. Without it a + // click that never reached the page is indistinguishable from one the page + // ignored, and the agent would report success either way. + w.__hermesHit = null; + document.addEventListener('pointerdown', function (e) { + w.__hermesHit = { tag: e.target ? e.target.tagName : '?', trusted: e.isTrusted === true }; + }, { capture: true, once: true }); + // Measure again once the scroll has stopped: real input is aimed at a fixed + // viewport coordinate, so it has to be where the target ENDS UP. + return restAfterScroll().then(function () { + var settled = act(locate); + var best = settled.success ? settled : found; + var at = best.point; + // Last line of defence before the pointer is sent somewhere real. An element + // that is still outside the viewport after we scrolled to it is hidden, not + // placed, and aiming at it would drive the cursor off into a corner and + // click whatever happens to be under that coordinate. + if (at && (at.x < 0 || at.y < 0 || at.x > window.innerWidth || at.y > window.innerHeight)) { + return JSON.stringify({ + error: 'That element is still off-screen after scrolling to it, so it is hidden rather than clickable. Call elements again for what is really on the page.', + success: false + }); + } + return JSON.stringify(best); + }); +})()` +} + +/** Put up a mark that outlives the action that made it. Every other cue on the + * overlay retires on a timer, which is right for narrating a click and no use + * at all for holding a finding on screen while the agent keeps working. */ +function buildPinScript(action: PreviewActAction, label: string): string { + const locate = { kind: 'locate', ref: action.ref, selector: action.selector } + + return `(function () { +${preamble()} + var found = act(${JSON.stringify(locate)}); + if (!found.success) { return JSON.stringify(found); } + watch('pin', ${JSON.stringify(label)}); + return JSON.stringify({ acted: 'pinned ' + String(found.acted || 'it').replace(/^looking at /, ''), success: true }); +})()` +} + +/** Freeze the whole visible field as marks. Needs a fresh inventory first, so + * the overlay has both the field to draw and the refs to number it by. */ +function buildHoldScript(): string { + return `(function () { +${preamble()} + var found = act({ kind: 'elements' }); + if (!found.success) { return JSON.stringify(found); } + watch('hold'); + return JSON.stringify({ acted: 'held the field', elements: found.elements, title: found.title, url: found.url, success: true }); +})()` +} + +/** Rattle through the field one box at a time. Needs a fresh inventory for the + * overlay to pick from, and nothing else — the page is never touched. */ +function buildStrobeScript(): string { + return `(function () { +${preamble()} + var found = act({ kind: 'elements' }); + if (!found.success) { return JSON.stringify(found); } + watch('strobe'); + return JSON.stringify({ acted: 'strobed the field', elements: found.elements, title: found.title, url: found.url, success: true }); +})()` +} + +/** Take one mark down, or — with nothing to aim at — all of them. */ +function buildUnpinScript(action: PreviewActAction): string { + const locate = { kind: 'locate', ref: action.ref, selector: action.selector } + const one = !!(action.ref || action.selector) + + return `(function () { +${preamble()} +${ + one + ? ` var found = act(${JSON.stringify(locate)}); + if (!found.success) { return JSON.stringify(found); } + var gone = 'unpinned ' + String(found.acted || 'it').replace(/^looking at /, '');` + : ` holder.aimed = null; + var gone = 'cleared every pin';` +} + watch('unpin'); + return JSON.stringify({ acted: gone, success: true }); +})()` +} + +/** Mark the hit, let the page react, and hand back a fresh inventory. */ +function buildFinishScript(settleMs: number): string { + return `(function () { +${preamble()} + watch('strike'); + return wait(${settleMs}).then(function () { + // A rescan that throws must still answer — an unresolved promise here costs + // the whole bridge deadline. + try { + var out = act({ kind: 'elements' }); + watch('sweep'); + out.hit = w.__hermesHit || null; + return JSON.stringify(out); + } catch (err) { + return JSON.stringify({ note: 'The page changed before it could be re-read: ' + err, success: true }); + } + }); +})()` +} + +/** The no-real-input fallback: the engine both acts and reads, in one trip. + * Both reads mark the field — the explicit `elements` call and the re-read + * every other verb does on its way out — so the page is marked after every + * single action rather than only when the agent asks for an inventory outright. + * They mark it differently, though, and the difference is what each one MEANS: + * an outright `elements` is the agent reading the whole page, which is the + * moment worth showing, so it strobes. The re-read after a click is + * housekeeping, and strobing there would put five seconds of noise between + * every step of a task, so it sweeps. The two never collide: an `elements` call + * settles in 0ms and returns before the re-read. */ +function buildScriptedScript(action: PreviewActAction, settleMs: number): string { + return `(function () { +${preamble()} var result = act(${JSON.stringify(action)}); - if (!result.success || ${settleMs} <= 0) { - return Promise.resolve(JSON.stringify(result)); + ${ + action.kind === 'elements' + ? "if (result.success) { watch('strobe'); }" + : '' } - // Re-inventory after the page has had a moment to react, so the agent's next - // ref is drawn from the DOM its own click produced. - return new Promise(function (resolve) { - setTimeout(function () { + if (!result.success || ${settleMs} <= 0) { return Promise.resolve(JSON.stringify(result)); } + return wait(${settleMs}).then(function () { + try { var after = act({ kind: 'elements' }); + watch('sweep'); result.elements = after.elements; result.url = after.url; result.title = after.title; - resolve(JSON.stringify(result)); - }, ${settleMs}); + } catch (err) { + result.note = 'The page changed before it could be re-read: ' + err; + } + return JSON.stringify(result); }); })()` } +/** The outcome of one round trip into the page. `silent` is its own case on + * purpose: a page that stops answering mid-action is navigating, whereas one + * that answers with nothing is broken, and the agent needs to hear the + * difference. */ +type Trip = { error: string; kind: 'failed' } | { kind: 'answered'; result: PreviewActResult } | { kind: 'silent' } + +async function runJson(run: PreviewScriptRunner, code: string): Promise { + const raw = await Promise.race([ + run(code).catch((error: unknown) => new Error(String(error))), + new Promise(resolve => setTimeout(resolve, ACT_TIMEOUT_MS)) + ]) + + if (raw === undefined) { + return { kind: 'silent' } + } + + if (raw instanceof Error) { + return { error: 'The page rejected the action: ' + raw.message, kind: 'failed' } + } + + if (typeof raw !== 'string' || !raw) { + return { error: 'The page did not answer the action.', kind: 'failed' } + } + + return { kind: 'answered', result: JSON.parse(raw) as PreviewActResult } +} + +/** Past tense of the verb the agent asked for, against what it actually hit. */ +function describeDone(action: PreviewActAction, target: string): string { + if (action.kind === 'type') { + return 'typed into ' + target + (action.submit ? ' and submitted' : '') + } + + if (action.kind === 'press') { + return 'pressed ' + (action.key || '') + ' on ' + target + } + + if (action.kind === 'hover') { + return 'hovered over ' + target + } + + return 'clicked ' + target +} + +/** Look at the target, walk the pointer over, and act on it for real. */ +async function driveAction( + run: PreviewScriptRunner, + input: PreviewInputHandle, + action: PreviewActAction +): Promise { + // A key press must not be preceded by a click — that would activate the + // control rather than type into it — so the page hands it focus instead. + const trip = await runJson(run, buildLocateScript(action, action.kind === 'press')) + + if (trip.kind === 'failed') { + return { error: trip.error, success: false } + } + + if (trip.kind === 'silent') { + return { acted: action.kind, note: NAVIGATED, success: true } + } + + const found = trip.result + + if (!found.success) { + return found + } + + if (!found.point) { + return { error: 'Could not work out where that element is on screen.', success: false } + } + + await glideTo(input, found.point) + + if (action.kind === 'click') { + await clickAt(input) + } else if (action.kind === 'type') { + if (found.typable === false) { + return { + error: `${String(found.acted || 'That').replace(/^looking at /, '')} is not a text field, so typing into it would only select the text under the pointer. Click it if it opens one, then type into that.`, + success: false + } + } + + input.focus() + await clickAt(input) + // Select-all inside the now-focused field, so typing replaces what is there + // the way it would for a person. NOT a triple-click: that is a pointer + // gesture and selects the paragraph under the cursor whenever the target + // turns out not to be a field. + await selectAll(input) + await typeText(input, action.text ?? '') + + if (action.submit) { + await pressKey(input, 'Enter') + } + } else if (action.kind === 'press') { + input.focus() + await pressKey(input, action.key || 'Enter') + } + // hover is the glide and nothing else — the pointer is already sitting on the + // target, which is the whole request. + + const target = String(found.acted || '').replace(/^looking at /, '') + const after = await runJson(run, buildFinishScript(SETTLE_MS)) + const acted = describeDone(action, target) + + // The action itself already happened as real input, so a page that will not + // answer the read-back is a page that navigated — never a failed click. + if (after.kind !== 'answered') { + return { acted, note: NAVIGATED, success: true } + } + + const { hit, ...result } = after.result as PreviewActResult & { hit?: { tag: string; trusted: boolean } | null } + + // The witness the locate trip armed. No record means the input never reached + // the document, which the agent must hear about — every other signal here + // travels on the script channel and would report success regardless. + if (!hit && CLICKS.indexOf(action.kind) !== -1) { + return { + ...result, + error: 'The pointer input never reached the page, so nothing was ' + acted.split(' ')[0] + '.', + success: false + } + } + + return { ...result, acted, note: hitNote(hit), success: true } +} + +/** Flag a click the overlay intercepted, which would otherwise look like a page + * that simply ignored it. */ +function hitNote(hit?: { tag: string; trusted: boolean } | null): string | undefined { + return hit && hit.tag === 'HERMES-WATCH' + ? 'The action overlay intercepted the click instead of the page.' + : undefined +} + +/** How far a screenful is, whether there is anywhere to go, and a spot to wheel + * over — the renderer cannot see the guest viewport to work any of it out. */ +function buildScrollAnchorScript(): string { + return `(function () { +${preamble()} + var sc = document.scrollingElement || document.documentElement; + var track = sc.clientHeight || window.innerHeight; + return Promise.resolve(JSON.stringify({ + page: Math.round(window.innerHeight * 0.9), + point: { x: Math.round(window.innerWidth / 2), y: Math.round(track / 2) }, + span: sc.scrollHeight - track, + success: true + })); +})()` +} + +/** Scroll the page the way a hand does — wheel it. The scripted path calls + * `scrollBy`, which moves the page without the page ever seeing an input + * event, so scroll-linked headers, reveal animations and infinite-scroll + * loaders all stay asleep through a scroll that looked like it happened. */ +async function driveScroll( + run: PreviewScriptRunner, + input: PreviewInputHandle, + action: PreviewActAction, +): Promise { + const far = action.amount ?? 0 + const trip = await runJson( + run, + buildScrollAnchorScript() + ) + + if (trip.kind === 'failed') { + return { error: trip.error, success: false } + } + + if (trip.kind === 'silent') { + return { acted: 'scrolled', note: NAVIGATED, success: true } + } + + const anchor = trip.result as PreviewActResult & { page?: number; span?: number } + + if (!anchor.span) { + return { ...anchor, acted: 'scrolled the page', note: 'The page has nothing to scroll — it all fits already.' } + } + + // A person does not move the mouse to scroll; the wheel turns wherever their + // hand already is. Only send it somewhere if it has never been anywhere. + if (!pointerPlaced() && anchor.point) { + await glideTo(input, anchor.point) + } + + await wheelBy(input, action.amount ?? anchor.page ?? 600) + + const after = await runJson(run, buildFinishScript(SETTLE_MS)) + + if (after.kind !== 'answered') { + return { acted: 'scrolled the page', note: NAVIGATED, success: true } + } + + return { ...after.result, acted: 'scrolled the page', success: true } +} + /** Run one action against the ACTIVE preview tab's page. `kind` is a bare * string: the verb arrives off the wire, and the history ones never reach * the in-page engine. */ @@ -87,12 +511,57 @@ export async function actOnActivePreview( } const typed = action as PreviewActAction - const settle = typed.kind === 'elements' ? 0 : SETTLE_MS - const raw = await run(buildActScript(typed, settle)) - if (typeof raw !== 'string' || !raw) { - return { error: 'The page did not answer the action.', success: false } + // Annotation, not interaction: nothing is clicked, nothing settles, and the + // page is not re-read, so these skip the whole act-then-inventory path. + if (typed.kind === 'pin' || typed.kind === 'unpin' || typed.kind === 'hold' || typed.kind === 'strobe') { + const mark = + typed.kind === 'hold' + ? buildHoldScript() + : typed.kind === 'strobe' + ? buildStrobeScript() + : typed.kind === 'pin' + ? buildPinScript(typed, typed.text || '') + : buildUnpinScript(typed) + const trip = await runJson(run, mark) + + if (trip.kind === 'failed') { + return { error: trip.error, success: false } + } + + return trip.kind === 'answered' ? trip.result : { acted: typed.kind, note: NAVIGATED, success: true } } - return JSON.parse(raw) as PreviewActResult + const input = activePreviewInput() + + if (input && DRIVEN.indexOf(typed.kind) !== -1) { + return driveAction(run, input, typed) + } + + // A plain page scroll is a wheel gesture. Jumping to an end is not — no hand + // wheels to the bottom of a long article — so `to` stays a scripted glide. + const plain = typed.kind === 'scroll' && !typed.to && !typed.ref && !typed.selector + + if (plain && input) { + return driveScroll(run, input, typed) + } + + const settle = typed.kind === 'elements' ? 0 : SETTLE_MS + const scripted = await runJson(run, buildScriptedScript(typed, settle)) + + if (scripted.kind === 'failed') { + return { error: scripted.error, success: false } + } + + // The action almost certainly landed — a page that stops answering right + // after a click is one that navigated. Say so instead of failing it. + return scripted.kind === 'silent' ? { acted: typed.kind, note: NAVIGATED, success: true } : scripted.result +} + +// Self-accept so an edit here, or to the in-page sources this module +// stringifies, doesn't reload the whole renderer out from under a live session. +// The bridge re-requests this module per action in dev, so it picks the change +// up without one (see loadPreviewEngine in gateway-event/desktop-bridge.ts). +if (import.meta.hot) { + import.meta.hot.accept() } diff --git a/apps/desktop/src/app/chat/right-rail/preview-mind.ts b/apps/desktop/src/app/chat/right-rail/preview-mind.ts new file mode 100644 index 0000000000..3f7a1856c4 --- /dev/null +++ b/apps/desktop/src/app/chat/right-rail/preview-mind.ts @@ -0,0 +1,29 @@ +/** + * IDLE PULSE — keeps the preview overlay alive while the model reasons. + * + * Everything else the overlay draws is narration of an action, so the pane goes + * dark for the gap between actions — which is most of the time the agent spends + * on a page, and the part where it is deciding what to do with what it just + * read. A page that stops responding the moment the agent is thinking hardest + * reads as the agent having wandered off. + * + * This is deliberately NOT part of the act pipeline: a turn boundary only has + * to poke the overlay the last action left behind. See preview-nudge.ts. + */ + +import { $busy } from '@/store/session' + +import { nudgeOverlay } from './preview-nudge' + +// Module-level, matching how review.ts and coding-status.ts watch this edge. It +// is inert without a live pane, so there is nothing to mount or tear down. +let running = $busy.get() + +$busy.subscribe(busy => { + if (busy === running) { + return + } + + running = busy + nudgeOverlay(busy ? 'think' : 'rest') +}) diff --git a/apps/desktop/src/app/chat/right-rail/preview-nudge.ts b/apps/desktop/src/app/chat/right-rail/preview-nudge.ts new file mode 100644 index 0000000000..8a8f419987 --- /dev/null +++ b/apps/desktop/src/app/chat/right-rail/preview-nudge.ts @@ -0,0 +1,38 @@ +/** + * OVERLAY NUDGE — the one way to say a stage to an overlay that is ALREADY on + * the page. + * + * The act pipeline (preview-act.ts) injects the whole engine because it has to: + * it needs to resolve a ref, measure a node and read the page back. The stages + * that only narrate — the idle pulse, the read wipe — need none of that. They + * are one function call against something the last action already left on the + * page, so re-shipping the engine to make it would put the payload on the wire + * on every turn boundary. + * + * A page the agent has never acted on has no overlay, and the nudge is a no-op + * there. That is the right answer rather than a gap: chrome on a page the agent + * never touched would be a lie about what it did. + */ + +import type { WatchStage } from '@/lib/preview-act/watch-in-page' + +import { activePreviewScriptRunner } from './preview-script-runner' + +/** Run one stage against the active pane's overlay, if it has one. */ +export function nudgeOverlay(stage: WatchStage): void { + const run = activePreviewScriptRunner() + + if (!run) { + return + } + + void run(`(function () { + var w = window; + var fn = w.__hermesWatch_fn; + if (!fn) { return 'cold'; } + try { fn(document, w.__hermesActHolder || {}, ${JSON.stringify(stage)}); } catch (err) {} + return 'ok'; +})()`).catch(() => { + // The page navigated out from under us. The next action re-injects. + }) +} diff --git a/apps/desktop/src/app/chat/right-rail/preview-pane.tsx b/apps/desktop/src/app/chat/right-rail/preview-pane.tsx index 076fc4a94f..817cec638a 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-pane.tsx +++ b/apps/desktop/src/app/chat/right-rail/preview-pane.tsx @@ -1,3 +1,7 @@ +// Side-effect import: watches the turn edge so the overlay keeps a pulse while +// the model reasons. Lives here because the pane is what makes it reachable. +import './preview-mind' + import { useStore } from '@nanostores/react' import type { PointerEvent as ReactPointerEvent } from 'react' import { useCallback, useEffect, useMemo, useRef, useState } from 'react' @@ -28,7 +32,7 @@ import { import { type ConsoleEntry } from './preview-console-state' import { previewConsoleState } from './preview-console-store' import { LocalFilePreview, PreviewEmptyState } from './preview-file' -import { registerPreviewInput, type PreviewInputEvent } from './preview-input' +import { type PreviewInputEvent, registerPreviewInput } from './preview-input' import { PREVIEW_BROWSER_ATTR, registerPreviewNav } from './preview-nav' import { registerPreviewPageReader } from './preview-reader' import { registerPreviewScriptRunner } from './preview-script-runner' diff --git a/apps/desktop/src/app/chat/right-rail/preview-reader.ts b/apps/desktop/src/app/chat/right-rail/preview-reader.ts index 7f48c8cbb1..97a372db18 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-reader.ts +++ b/apps/desktop/src/app/chat/right-rail/preview-reader.ts @@ -15,6 +15,8 @@ import { $rightRailActiveTabId } from '@/store/layout' import { $previewTabs } from '@/store/preview' +import { nudgeOverlay } from './preview-nudge' + export interface PreviewReadOptions { /** Characters to return from `start` (capped at PREVIEW_READ_MAX_CHARS). */ count?: number @@ -89,6 +91,13 @@ export async function readActivePreview(opts: PreviewReadOptions = {}): Promise< try { const page = await reader() + // Say it on the page. Reading is by far the cheapest thing the agent + // does — a few hundredths of a second against a model round trip either + // side of it — so a run of reads used to leave the pane dark for the + // twenty seconds it took to page through a document, immediately after + // the one moment that showed anything. + nudgeOverlay('read') + return windowText( { kind: target.kind, path: target.path, title: page.title || target.label, url: page.url || target.url }, page.text, diff --git a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/desktop-bridge.ts b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/desktop-bridge.ts index 1a8cee9d02..227a4eb128 100644 --- a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/desktop-bridge.ts +++ b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event/desktop-bridge.ts @@ -11,6 +11,34 @@ import { setMessages } from '@/store/session' import type { GatewayEventContext } from './types' +/** The preview engine, loaded on demand so ~25KB of page-injectable source stays + * off the boot path. + * + * In dev that lazy chunk is also a trap. The browser caches a dynamic import by + * URL for the life of the page, so this bridge would hand every action to + * whichever build of the engine loaded first, and no edit to it — or to the + * overlay whose source it stringifies into the page — would reach the guest + * until the whole window reloaded. Asking for a fresh copy is more reliable + * than trusting hot-update propagation to reach a module nothing statically + * imports; the dev server stamps the dependency URLs it has invalidated, so a + * fresh engine pulls a fresh overlay down with it. + * + * The literal path is what a bare specifier can't be here, and it has to track + * this module's real location — hence the fall back to the static import, which + * is also the only branch production keeps, `import.meta.hot` being stripped + * there along with everything it guards. */ +const loadPreviewEngine = () => { + const stable = () => import('@/app/chat/right-rail/preview-act') + + if (!import.meta.hot) { + return stable().then(mod => mod.actOnActivePreview) + } + + return import(/* @vite-ignore */ '/src/app/chat/right-rail/preview-act.ts?hot=' + Date.now()) + .catch(stable) + .then(mod => mod.actOnActivePreview as Awaited>['actOnActivePreview']) +} + /** Desktop-surface bridge events: read-back requests the agent blocks on * (terminal/preview/window), agent terminal streaming, pane reveal, and * message reactions. */ @@ -57,7 +85,7 @@ export function handleDesktopBridgeEvent(ctx: GatewayEventContext): boolean { } if (event.type === 'preview.act.request') { - // act_preview tool: click/type/scroll/press inside the guest page, or + // drive_preview tool: click/type/scroll/press inside the guest page, or // drive the pane's history. Dynamic import keeps the injected engine off // the boot path. Active session only: a background turn must never reach // into the page the user is working in (desktop AGENTS.md: offer, don't @@ -72,9 +100,9 @@ export function handleDesktopBridgeEvent(ctx: GatewayEventContext): boolean { }) if (isActiveEvent) { - void import('@/app/chat/right-rail/preview-act') - .then(({ actOnActivePreview }) => - actOnActivePreview({ + void loadPreviewEngine() + .then(run => + run({ amount: payload?.amount, key: payload?.key, kind: payload?.action ?? '', diff --git a/apps/desktop/src/lib/preview-act/act-in-page.test.ts b/apps/desktop/src/lib/preview-act/act-in-page.test.ts index aa1985fb7a..5fafdb91e8 100644 --- a/apps/desktop/src/lib/preview-act/act-in-page.test.ts +++ b/apps/desktop/src/lib/preview-act/act-in-page.test.ts @@ -3,14 +3,17 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { actInPage, type PreviewActHolder } from './act-in-page' /** jsdom lays nothing out, so every rect is 0×0 and the engine's visibility - * check would reject the whole page. Give elements a plausible box and let + * check would reject the whole page. Give elements a plausible box, honouring + * an explicit width/height so a test can lay out a 1px one, and let * `display: none` (which jsdom DOES compute) carry the hiding. */ function layOutTheDocument() { vi.spyOn(Element.prototype, 'getBoundingClientRect').mockImplementation(function (this: Element) { - const hidden = getComputedStyle(this).display === 'none' - const size = hidden ? 0 : 40 + const style = getComputedStyle(this) + const width = style.display === 'none' ? 0 : parseFloat(style.width) || 40 + const height = style.display === 'none' ? 0 : parseFloat(style.height) || 40 + const left = parseFloat(style.left) || 0 - return { bottom: size, height: size, left: 0, right: size, top: 0, width: size, x: 0, y: 0 } as DOMRect + return { bottom: height, height, left, right: left + width, top: 0, width, x: left, y: 0 } as DOMRect }) } @@ -29,6 +32,33 @@ beforeEach(() => { vi.restoreAllMocks() layOutTheDocument() Element.prototype.scrollIntoView = vi.fn() + // jsdom has no hit-testing at all, so the engine skips the occlusion check + // here by default. Tests that stand one in must not leak it into the next. + delete (document as Document & { elementFromPoint?: unknown }).elementFromPoint +}) + +describe('self-containment', () => { + // The pane injects `actInPage.toString()` into the guest page, where module + // scope does not exist. A single free identifier (a module-level constant, an + // imported helper) is a ReferenceError on every call, and Electron reports it + // only as "Script failed to execute" — so evaluate the source the way the + // guest does and run a real action through it. + it('runs after being stringified and eval’d with no module scope', () => { + const page = document.createElement('div') + page.innerHTML = '' + document.body.replaceChildren(page) + + const injected = new Function('return (' + actInPage.toString() + ')')() as typeof actInPage + const holder: PreviewActHolder = {} + + expect(injected(document, holder, { kind: 'elements' }).elements?.[0].label).toBe('Save') + + const clicked = vi.fn() + document.getElementById('save')!.addEventListener('click', clicked) + + expect(injected(document, holder, { kind: 'click', ref: '@e1' }).success).toBe(true) + expect(clicked).toHaveBeenCalledOnce() + }) }) describe('elements', () => { @@ -73,6 +103,82 @@ describe('elements', () => { expect(inventory(holder).elements?.map(e => e.label)).toEqual(['Real']) }) + // The screen-reader-only recipe is a 1px box parked at the document origin, + // and it passed a `width >= 1` check. Skip links and CSS-only menu toggles are + // built this way and come FIRST in the document, so they landed on @e1 — the + // agent would aim at one and the pointer would fly to the top-left corner and + // click nothing. + it('skips the visually-hidden controls that sit at the document origin', () => { + const holder = page(` + Jump to content + + Skip navigation + + `) + + expect(inventory(holder).elements?.map(e => e.label)).toEqual(['Search']) + }) + + it('skips a control parked off the left edge, which no scroll brings back', () => { + const holder = page(` + + + `) + + expect(inventory(holder).elements?.map(e => e.label)).toEqual(['Search']) + }) + + // Wikipedia and friends are full of these: decorative chrome and collapsed + // menus that are perfectly solid boxes as far as layout is concerned, but + // that the page has already declared are not for anyone to interact with. + it('skips controls the page marked aria-hidden or inert', () => { + const holder = page(` + +
+ + `) + + expect(inventory(holder).elements?.map(e => e.label)).toEqual(['Search']) + }) + + it('skips a control buried under another layer, which would take the click instead', () => { + const holder = page(` + + + `) + const wall = document.createElement('div') + document.body.append(wall) + + // Stand in for the hit-testing jsdom does not do: every element reports + // itself except the buried one, which reports the sheet lying over it. + document.elementFromPoint = (x: number) => (x === 120 ? wall : document.getElementById('real')) + + expect(inventory(holder).elements?.map(e => e.label)).toEqual(['Search']) + }) + + it('keeps a control whose own child is what the hit test lands on', () => { + const holder = page(' Home') + + document.elementFromPoint = () => document.getElementById('icon') + + expect(inventory(holder).elements?.map(e => e.label)).toEqual(['Home']) + }) + + // The overlay draws far more than the agent is told about, so the two lists + // are collected separately: the field is every visible control, the inventory + // is the describable subset that refs point into. + it('parks the whole interactive field for the overlay, not just the inventory', () => { + const holder = page(` + + + + `) + + expect(inventory(holder).elements?.map(e => e.label)).toEqual(['Search']) + expect(holder.nodes).toEqual([document.getElementById('named')]) + expect(holder.field).toEqual([document.getElementById('named'), document.getElementById('mystery')]) + }) + it('prefers an identity selector so the agent can re-find the node later', () => { const holder = page(`
@@ -288,7 +394,23 @@ describe('scroll', () => { const result = actInPage(document, holder, { kind: 'scroll' }) expect(result.success).toBe(true) - expect(scrollBy).toHaveBeenCalledWith({ behavior: 'auto', top: Math.round(window.innerHeight * 0.9) }) + expect(scrollBy).toHaveBeenCalledWith({ behavior: 'smooth', top: Math.round(window.innerHeight * 0.9) }) + }) + + it('drops the animation for a reader who asked for reduced motion', () => { + const holder = page('

long page

') + const scrollBy = vi.spyOn(window, 'scrollBy').mockImplementation(() => {}) + + // Passing 'smooth' explicitly overrides the OS preference, so the engine has + // to opt back out itself rather than leaving it to the browser. + vi.stubGlobal('matchMedia', () => ({ matches: true })) + + actInPage(document, holder, { kind: 'scroll' }) + vi.unstubAllGlobals() + + const [options] = scrollBy.mock.calls[0] as unknown as [ScrollToOptions] + + expect(options.behavior).toBe('auto') }) it('jumps to the bottom', () => { @@ -312,7 +434,7 @@ describe('scroll', () => { const result = actInPage(document, holder, { amount: 200, kind: 'scroll', ref: '@e1' }) - expect(list.scrollBy).toHaveBeenCalledWith({ behavior: 'auto', top: 200 }) + expect(list.scrollBy).toHaveBeenCalledWith({ behavior: 'smooth', top: 200 }) expect(pageScroll).not.toHaveBeenCalled() expect(result.acted).toContain('Results') }) diff --git a/apps/desktop/src/lib/preview-act/act-in-page.ts b/apps/desktop/src/lib/preview-act/act-in-page.ts index f664f58299..0d78fccf2a 100644 --- a/apps/desktop/src/lib/preview-act/act-in-page.ts +++ b/apps/desktop/src/lib/preview-act/act-in-page.ts @@ -35,7 +35,24 @@ export interface PreviewActAction { /** scroll distance in px. Defaults to ~90% of the viewport height. */ amount?: number key?: string - kind: 'click' | 'elements' | 'press' | 'scroll' | 'type' + /** `pin`/`unpin`/`hold` never reach the engine — they resolve their targets + * through `locate`/`elements` and then talk to the overlay — but they arrive + * on the same wire. */ + kind: + | 'click' + | 'elements' + | 'hold' + | 'hover' + | 'locate' + | 'pin' + | 'press' + | 'scroll' + | 'strobe' + | 'type' + | 'unpin' + /** locate: also give the target keyboard focus, for a key press that must not + * be preceded by a click (which would activate the control instead). */ + focus?: boolean /** Cap on the returned inventory. */ max?: number ref?: string @@ -52,8 +69,12 @@ export interface PreviewActResult { elements?: PreviewElement[] error?: string note?: string + /** Viewport centre of a located target, for aiming real pointer input at it. */ + point?: { x: number; y: number } success: boolean title?: string + /** locate: whether the target actually takes typed text. */ + typable?: boolean /** Live document URL after the action — a change means it navigated. */ url?: string } @@ -61,18 +82,34 @@ export interface PreviewActResult { /** Where the surface keeps the last snapshot between actions (a window global * in the preview page), so '@e5' still means something on the next call. */ export interface PreviewActHolder { + /** Target of the action in flight, for the watch overlay to draw onto. */ + aimed?: Element | null + /** The on-screen subset, for the overlay to outline. Diverges from `nodes` in + * both directions: it drops what is below the fold, and it is not capped at + * the inventory's size. */ + field?: Element[] nodes?: Element[] /** URL the snapshot was taken on; a navigation invalidates every ref. */ url?: string } -const ACT_MAX_ELEMENTS = 120 - /** Run one action against `doc`, resolving refs through `holder`. Self-contained. */ export function actInPage(doc: Document, holder: PreviewActHolder, action: PreviewActAction): PreviewActResult { + // Declared inside, not at module scope: this body is stringified and eval'd + // in the guest page, where a module-level constant is simply not defined. + const maxElements = 120 + // How many nodes the OVERLAY may draw, as opposed to how many the agent gets + // told about. Far higher, because an extra mark costs one rect read where an + // extra inventory row costs tokens on every single call. + const maxMarks = 600 const win = doc.defaultView const here = doc.location ? doc.location.href : '' + // Passing 'smooth' explicitly OVERRIDES the user's OS-level reduce-motion + // setting, so ask before animating over someone who opted out. + const still = !!(win && win.matchMedia && win.matchMedia('(prefers-reduced-motion: reduce)').matches) + const glide: ScrollBehavior = still ? 'auto' : 'smooth' + const cssEscape = (value: string) => typeof CSS !== 'undefined' && CSS.escape ? CSS.escape(value) : value.replace(/["\\]/g, '\\$&') @@ -137,17 +174,118 @@ export function actInPage(doc: Document, holder: PreviewActHolder, action: Previ return path.join(' > ') } - const visible = (el: Element): boolean => { + /** On screen right now, and worth drawing a box around. The field is strictly + * viewport-bound: a mark past the fold is one nobody can see, it spends the + * budget that should have gone to what IS on screen, and the ones parked far + * off to the side were landing as stray labels in the corner. */ + const onScreen = (el: Element): boolean => { + if (!win) { + return false + } + + const box = el.getBoundingClientRect() + + if (box.right <= 0 || box.bottom <= 0 || box.left >= win.innerWidth || box.top >= win.innerHeight) { + return false + } + + // Page-sized wrappers. Outlining one draws a rectangle around the whole + // screen, which says nothing and frames everything inside it as if it + // mattered. Full-width banners are fine — it takes both dimensions. + return !(box.width >= win.innerWidth * 0.95 && box.height >= win.innerHeight * 0.9) + } + + /** Painted at all, ancestors included. */ + const shown = (el: Element): boolean => { + // Folds in display/visibility/opacity/content-visibility inherited from an + // ANCESTOR, which this element's own computed style does not report: inside + // a parent at opacity 0, every child still reads back opacity 1. + const seen = (el as Element & { checkVisibility?: (opts: object) => boolean }).checkVisibility + + if (typeof seen === 'function' && !seen.call(el, { checkOpacity: true, checkVisibilityCSS: true })) { + return false + } + const style = win && win.getComputedStyle ? win.getComputedStyle(el) : null - if (style && (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0')) { + if (style) { + if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') { + return false + } + + // The screen-reader-only recipe, both spellings: the deprecated `clip` and + // the modern `clip-path: inset(50%)`. Neither exists for any purpose other + // than hiding something visually while keeping it focusable. + if ((style.clip && style.clip !== 'auto') || (style.clipPath || '').indexOf('inset(50%') !== -1) { + return false + } + } + + return true + } + + const visible = (el: Element): boolean => { + if (!shown(el)) { + return false + } + + // Things the page itself declares are not for interacting with, neither of + // which shows up in a computed style or a bounding box: an aria-hidden + // subtree is invisible to every other assistive client, and `inert` cannot + // be clicked at all. Disabled controls are deliberately NOT filtered here — + // they stay in the inventory so the agent is told a button is disabled + // rather than hunting for one that appears not to exist. + if (el.closest('[aria-hidden="true"], [inert]')) { return false } const rect = el.getBoundingClientRect() - // Off-screen is fine (we scroll to it); collapsed to nothing is not. - return rect.width >= 1 && rect.height >= 1 + // Below the fold is fine — we scroll to it. Off to the LEFT or ABOVE is the + // other half of the sr-only trick (`left: -9999px`, `top: -9999px`) and + // scrolling never brings those back. + if (rect.right <= 0 || rect.bottom <= 0) { + return false + } + + // Bigger than the 1px box sr-only collapses to. Nothing a person can aim at + // is 2px across, and admitting those is what put the cursor in the corner: + // they cluster at the document origin, so they sort to the FRONT of the + // inventory and the agent reaches for one as @e1. + if (rect.width < 3 || rect.height < 3) { + return false + } + + // Occlusion, and the last filter for a reason: everything above is cheap + // and this one forces layout. If the middle of the element is on screen, + // whatever the browser reports at that point has to BE this element — its + // own descendant (an icon inside a link) or the box it paints inside are + // fine, anything else means it is buried under a sticky header, a cookie + // wall, or a transparent layer the page put on top. Those are exactly the + // targets the agent aims at and then misses. Elements below the fold cannot + // be hit-tested from here, so they are taken on trust and land back in this + // function after the scroll. + const midX = rect.left + rect.width / 2 + const midY = rect.top + rect.height / 2 + const under = (doc as Document & { elementFromPoint?: (x: number, y: number) => Element | null }) + .elementFromPoint + + if ( + typeof under === 'function' && + win && + midX >= 0 && + midY >= 0 && + midX < win.innerWidth && + midY < win.innerHeight + ) { + const over = under.call(doc, midX, midY) + + if (!over || !(over === el || el.contains(over) || over.contains(el))) { + return false + } + } + + return true } const valueOf = (el: Element): string => { @@ -166,6 +304,7 @@ export function actInPage(doc: Document, holder: PreviewActHolder, action: Previ const collect = (max: number): PreviewElement[] => { const nodes: Element[] = [] + const field: Element[] = [] const elements: PreviewElement[] = [] const candidates = doc.querySelectorAll( @@ -177,7 +316,22 @@ export function actInPage(doc: Document, holder: PreviewActHolder, action: Previ ) for (const el of candidates) { - if (elements.length >= max || nodes.indexOf(el) !== -1 || !visible(el)) { + if (elements.length >= max && field.length >= maxMarks) { + break + } + + if (!visible(el)) { + continue + } + + // Drawable from here on. The two lists diverge deliberately: the field is + // what is on screen to be outlined, the inventory is what the agent can + // name and reach — which includes things below the fold it will scroll to. + if (field.length < maxMarks && onScreen(el)) { + field.push(el) + } + + if (elements.length >= max) { continue } @@ -212,6 +366,7 @@ export function actInPage(doc: Document, holder: PreviewActHolder, action: Previ } holder.nodes = nodes + holder.field = field holder.url = here return elements @@ -279,7 +434,7 @@ export function actInPage(doc: Document, holder: PreviewActHolder, action: Previ } if (action.kind === 'elements') { - const elements = collect(Math.max(1, Math.min(action.max || ACT_MAX_ELEMENTS, ACT_MAX_ELEMENTS))) + const elements = collect(Math.max(1, Math.min(action.max || maxElements, maxElements))) return answer({ elements, @@ -300,9 +455,9 @@ export function actInPage(doc: Document, holder: PreviewActHolder, action: Previ const by = action.to === 'top' ? -1e7 : action.to === 'bottom' ? 1e7 : (action.amount ?? page) if (scroller) { - scroller.scrollBy({ behavior: 'auto', top: by }) + scroller.scrollBy({ behavior: glide, top: by }) } else if (win) { - win.scrollBy({ behavior: 'auto', top: by }) + win.scrollBy({ behavior: glide, top: by }) } return answer({ acted: scroller ? 'scrolled ' + describe(scroller) : 'scrolled the page', success: true }) @@ -316,12 +471,63 @@ export function actInPage(doc: Document, holder: PreviewActHolder, action: Previ const el = target.el as HTMLElement - if ((el as HTMLInputElement).disabled) { - return fail(describe(el) + ' is disabled.') + // Park the target where the watch overlay can find it, before anything can + // fail — a ring around the thing that turned out to be disabled is exactly + // the feedback someone watching wants. + holder.aimed = el + + // 'locate' is the look-before-you-act half of an action: it brings the target + // on screen and names it, so the overlay has something to draw while the real + // verb is still a beat away. + if (action.kind === 'locate') { + if (el.scrollIntoView) { + el.scrollIntoView({ behavior: 'instant', block: 'center', inline: 'nearest' }) + } + + if (action.focus) { + el.focus() + } + + const spot = pointAt(el) + const tag = el.tagName + + return answer({ + acted: 'looking at ' + describe(el), + point: { x: spot.clientX, y: spot.clientY }, + success: true, + // Real typing starts with a triple-click to clear the field. On anything + // that is not a field that gesture selects the paragraph under it + // instead, which is how the agent ended up highlighting whole pages. + typable: tag === 'TEXTAREA' || tag === 'INPUT' || el.isContentEditable === true + }) } + // Instant, unlike the `scroll` verb above. Getting a target on screen is + // plumbing for the click that follows, not something anyone asked to watch — + // and every millisecond of it is time the caller spends waiting for the page + // to stop moving before it can aim real input at a fixed coordinate. if (el.scrollIntoView) { - el.scrollIntoView({ block: 'center', inline: 'nearest' }) + el.scrollIntoView({ behavior: 'instant', block: 'center', inline: 'nearest' }) + } + + // Hovering a disabled control is allowed — a disabled button with a tooltip + // explaining WHY is exactly the thing worth hovering. + if (action.kind === 'hover') { + const init = { bubbles: true, cancelable: true, ...pointAt(el) } + + if (typeof PointerEvent === 'function') { + el.dispatchEvent(new PointerEvent('pointerover', init)) + el.dispatchEvent(new PointerEvent('pointermove', init)) + } + + el.dispatchEvent(new MouseEvent('mouseover', init)) + el.dispatchEvent(new MouseEvent('mousemove', init)) + + return answer({ acted: 'hovered over ' + describe(el), success: true }) + } + + if ((el as HTMLInputElement).disabled) { + return fail(describe(el) + ' is disabled.') } if (action.kind === 'click') { diff --git a/apps/desktop/src/lib/preview-act/watch-in-page.test.ts b/apps/desktop/src/lib/preview-act/watch-in-page.test.ts new file mode 100644 index 0000000000..b51c62b7f0 --- /dev/null +++ b/apps/desktop/src/lib/preview-act/watch-in-page.test.ts @@ -0,0 +1,363 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { type WatchHolder, watchInPage } from './watch-in-page' + +/** jsdom has no layout engine, so every rect is zero and nothing would be + * positioned. Give elements a plausible box the way the act-engine tests do. */ +beforeEach(() => { + // Never run the callback: the tracking loop re-arms itself every frame, so a + // stub that invokes synchronously recurses until the stack gives out. The + // first pass happens on the direct call, which is the one worth asserting on. + vi.stubGlobal('requestAnimationFrame', () => 1) + vi.stubGlobal('cancelAnimationFrame', () => {}) + + Element.prototype.getBoundingClientRect = function () { + return { bottom: 140, height: 40, left: 100, right: 300, toJSON: () => ({}), top: 100, width: 200, x: 100, y: 100 } + } + + document.body.replaceChildren() + // The host hangs off documentElement, so clearing body does not reach it. + document.querySelector('hermes-watch')?.remove() + delete (window as unknown as Record).__hermesWatch +}) + +/** The overlay lives in a closed shadow root, so a test can only see the host. */ +const host = () => document.querySelector('hermes-watch') + +/** …except through the parts the engine parks on the window for its own reuse, + * which is the only way to inspect what the overlay actually drew. */ +const drawn = () => + (window as unknown as { __hermesWatch: { parts: Record & { shadow: ShadowRoot } } }) + .__hermesWatch.parts + +/** Everything the overlay drew for this action. The cursor is the only fixed + * layer — every box and pin is a mark that comes and goes. */ +const marks = () => [...drawn().shadow.children].filter(child => child !== drawn().pointer) + +describe('watchInPage', () => { + it('mounts one host on documentElement and reuses it across stages', () => { + const target = document.createElement('button') + document.body.append(target) + + const holder: WatchHolder = { aimed: target } + + watchInPage(document, holder, 'aim') + watchInPage(document, holder, 'strike') + + expect(document.querySelectorAll('hermes-watch')).toHaveLength(1) + expect(host()?.parentElement).toBe(document.documentElement) + }) + + it('stays out of the page, out of the way, and out of the a11y tree', () => { + const target = document.createElement('button') + document.body.append(target) + + watchInPage(document, { aimed: target }, 'aim') + + const node = host() as HTMLElement + + // pointer-events keeps real clicks reaching the page; aria-hidden keeps a + // screen reader from announcing decoration. + expect(node.style.pointerEvents).toBe('none') + expect(node.getAttribute('aria-hidden')).toBe('true') + // The closed shadow root is what keeps the overlay out of the act engine's + // own querySelectorAll inventory — there is nothing to exclude by hand. + expect(node.shadowRoot).toBeNull() + }) + + it('does nothing when there is no target to draw on', () => { + watchInPage(document, {}, 'aim') + watchInPage(document, { aimed: document.createElement('div') }, 'strike') + + expect(host()).toBeNull() + }) + + // The cursor is placed, never interpolated. Beyond reading as a machine, a + // transition here was an outright bug on the first paint: an unset transform + // IS the origin, so the browser flew the cursor in from the top-left corner — + // and every navigation is a fresh document, so a fresh first paint. + it('cuts the marks straight to the target instead of travelling to it', () => { + const target = document.createElement('button') + document.body.append(target) + + watchInPage(document, { aimed: target }, 'aim', 'Clicking Save') + + const { pointer } = drawn() + + expect(pointer.style.transition).toBeFalsy() + expect((marks()[0] as HTMLElement).style.transition).toBeFalsy() + // Mocked rect is 200×40 at (100, 100), so the cursor belongs at its centre. + expect(pointer.style.transform).toBe('translate3d(200px,120px,0)') + }) + + // The inventory pass is the one moment the agent is reading the whole page + // rather than aiming at one thing, so the overlay has to be able to hold many + // marks at once — not just the single ring every other stage draws. + it('sweep lights up every catalogued element and holds them well past the sweep', () => { + vi.useFakeTimers() + + const found = [document.createElement('button'), document.createElement('a'), document.createElement('input')] + document.body.append(...found) + + watchInPage(document, { nodes: found }, 'sweep') + expect(marks()).toHaveLength(found.length) + + // The point of the dwell: the grid is still up long after the pass that drew + // it finished, so there is something to actually look at. jsdom has no Web + // Animations, so this exercises the timed fallback, which holds for the same + // beat as the animated path. + vi.advanceTimersByTime(600) + expect(marks()).toHaveLength(found.length) + + // …and then drains scattered rather than blinking off as one block, which is + // the whole reason each box carries its own offset. Stepping through the + // exit must therefore see the count come down in more than one tick. + const drain = new Set() + + for (let t = 0; t < 7_000; t += 100) { + vi.advanceTimersByTime(100) + drain.add(marks().length) + } + + expect(drain.size).toBeGreaterThan(2) + expect(marks()).toHaveLength(0) + + vi.useRealTimers() + }) + + // jsdom cannot run a Web Animation, but it can be handed one and asked what + // it was given — and this particular option is worth pinning, because getting + // it wrong is invisible rather than broken. `easing` in the options bag is the + // ITERATION easing: it rewrites progress before any keyframe is consulted, so + // a step function there holds progress at 0 for the whole duration. Every box + // was created, animated and removed without painting a single frame. + it('does not step the sweep’s iteration easing, which would stop it painting', () => { + const timings: KeyframeAnimationOptions[] = [] + + Element.prototype.animate = vi.fn(function (this: Element, _frames, options) { + timings.push(options as KeyframeAnimationOptions) + + return { cancel: () => {}, finish: () => {} } as unknown as Animation + }) as unknown as Element['animate'] + + const found = [document.createElement('button'), document.createElement('a')] + document.body.append(...found) + + watchInPage(document, { field: found }, 'sweep') + + // The cursor's idle sway is permanent and starts with the overlay, so it is + // not one of the sweep's own animations. + const sweep = timings.filter(timing => timing.iterations !== Infinity) + + expect(sweep).toHaveLength(found.length) + + for (const timing of sweep) { + expect(String(timing.easing)).not.toMatch(/steps/) + } + }) + + it('names each swept box after the element under it, not its ref', () => { + const found = [document.createElement('button'), document.createElement('a'), document.createElement('input')] + + found[0].id = 'buy' + found[1].className = 'nav wide' + found[2].setAttribute('type', 'search') + document.body.append(...found) + + // Every drawable box gets named, addressable or not — a ref number only ever + // meant something for the subset that had one, and said nothing about what + // the element actually is. + watchInPage(document, { field: found, nodes: found.slice(0, 2) }, 'sweep') + + const { shadow } = drawn() + const cells = [...shadow.children].slice(-found.length) + + expect(cells.map(cell => cell.textContent)).toEqual(['button#buy', 'a.nav', 'input[search]']) + }) + + // The case that made positions necessary: a list of links with no id, no + // class and no type. Naming them by tag alone gives a whole page of boxes all + // reading "a", which is what the ref numbers were replaced for. + it('falls back to position so anonymous siblings are still told apart', () => { + const row = document.createElement('td') + + row.className = 'subtext' + row.innerHTML = '' + document.body.append(row) + + const found = [...row.children] + + watchInPage(document, { field: found }, 'sweep') + + const { shadow } = drawn() + const cells = [...shadow.children].slice(-found.length) + + expect(cells.map(cell => cell.textContent)).toEqual(['a[1]', 'a[2]', 'a[3]']) + }) + + it('keeps a name short enough to sit on the box it names', () => { + const found = [document.createElement('div')] + + found[0].id = 'a-genuinely-enormous-generated-identifier' + document.body.append(...found) + + watchInPage(document, { field: found }, 'sweep') + + const name = marks()[0].textContent as string + + expect(name.length).toBeLessThanOrEqual(16) + expect(name.startsWith('div#a-')).toBe(true) + expect(name.endsWith('…')).toBe(true) + }) + + // A field past a couple of dozen boxes is a texture, not a list. Naming every + // one buries the outlines — which ARE the readout at that size — under a wall + // of tags that collide with each other and overhang their own rectangles. + it('stops naming the boxes once the field is too dense to read', () => { + const few = Array.from({ length: 4 }, () => document.createElement('button')) + document.body.append(...few) + + watchInPage(document, { field: few }, 'sweep') + expect(marks().every(mark => mark.textContent)).toBe(true) + + document.querySelector('hermes-watch')?.remove() + delete (window as unknown as Record).__hermesWatch + + const many = Array.from({ length: 40 }, () => document.createElement('button')) + document.body.append(...many) + + watchInPage(document, { field: many }, 'sweep') + expect(marks().some(mark => mark.textContent)).toBe(false) + }) + + // Every other mark on the page wears its label on the top edge. A box against + // the top of the viewport has nowhere to put one, and a tab drawn off-screen + // is a box that looks unlabelled next to boxes that aren't. + it('flips the label under the box when there is no headroom above it', () => { + const roomy = document.createElement('button') + const flush = document.createElement('button') + const huge = document.createElement('button') + + flush.getBoundingClientRect = () => + ({ bottom: 40, height: 40, left: 100, right: 300, toJSON: () => ({}), top: 0, width: 200, x: 100, y: 0 }) as DOMRect + // Taller than the viewport: no room above it and none below it either, so + // the tab has nowhere left to go but inside. + huge.getBoundingClientRect = () => + ({ bottom: 900, height: 900, left: 100, right: 300, toJSON: () => ({}), top: 0, width: 200, x: 100, y: 0 }) as DOMRect + document.body.append(roomy, flush, huge) + + watchInPage(document, { field: [roomy, flush, huge] }, 'sweep') + + const tags = marks().map(mark => mark.querySelector('div:last-child') as HTMLElement) + + expect(tags[0].style.bottom).toBe('100%') + expect(tags[1].style.top).toBe('100%') + expect(tags[2].style.top).toBe('0px') + }) + + // A label is routinely wider than the box it names, so it is the boxes near the + // right edge that push their tab off screen. Whichever end it lands on, the + // tab's edge sits exactly on the outline's — the two are the same colour and + // meet at a corner, so a pixel of overhang reads as a misprint. + it('hangs the label off the right edge when a left-anchored one would run off', () => { + Object.defineProperty(HTMLElement.prototype, 'offsetWidth', { configurable: true, get: () => 80 }) + + const inboard = document.createElement('button') + const edge = document.createElement('button') + + edge.getBoundingClientRect = () => + ({ bottom: 140, height: 40, left: 990, right: 1020, toJSON: () => ({}), top: 100, width: 30, x: 990, y: 100 }) as DOMRect + document.body.append(inboard, edge) + + watchInPage(document, { field: [inboard, edge] }, 'sweep') + + const tags = marks().map(mark => mark.querySelector('div:last-child') as HTMLElement) + + expect(tags[0].style.left).toBe('0px') + expect(tags[0].style.right).toBe('') + expect(tags[1].style.right).toBe('0px') + expect(tags[1].style.left).toBe('') + }) + + it('sweep draws nothing when the inventory came back empty', () => { + watchInPage(document, { nodes: [] }, 'sweep') + + expect(host()).toBeNull() + }) + + // The field is swept after every action, and a sweep retires the lock. The + // cursor must not go with it — a hand that blinks out a fraction of a second + // after every single step reads as the agent having wandered off. + it('keeps the cursor up when a sweep retires the lock', () => { + const target = document.createElement('button') + const found = [document.createElement('a')] + document.body.append(target, ...found) + + watchInPage(document, { aimed: target }, 'aim') + + const { pointer } = drawn() + const lock = marks()[0] + + expect(pointer.style.opacity).toBe('1') + + watchInPage(document, { field: found }, 'sweep') + + expect(lock.isConnected).toBe(false) + expect(pointer.style.opacity).toBe('1') + }) + + // The chrome is memoized for the life of the page, so a style change to it + // would otherwise be injected, run, and restyle nothing — the layers it + // describes were built by the version before. Reads as "HMR isn't picking my + // CSS up" in dev, and as a long-lived tab stuck on an old build in production. + it('rebuilds its chrome when the injected source changes underneath it', () => { + const target = document.createElement('button') + document.body.append(target) + + const stamp = (value: number) => { + ;(window as unknown as Record).__hermesWatchTag = value + } + + stamp(1) + watchInPage(document, { aimed: target }, 'aim') + + const first = drawn().host + + stamp(1) + watchInPage(document, { aimed: target }, 'aim') + + expect(drawn().host).toBe(first) + + stamp(2) + watchInPage(document, { aimed: target }, 'aim') + + expect(drawn().host).not.toBe(first) + // …and the old one goes with it, rather than stacking a second overlay. + expect(document.querySelectorAll('hermes-watch')).toHaveLength(1) + }) + + it('clear releases the target so the tracking loop can stop', () => { + const target = document.createElement('button') + document.body.append(target) + + const holder: WatchHolder = { aimed: target } + + watchInPage(document, holder, 'aim') + watchInPage(document, holder, 'clear') + + expect(holder.aimed).toBeNull() + }) + + // Same contract as the act engine: the pane injects `watchInPage.toString()` + // into the guest page, where module scope does not exist. One free identifier + // is a ReferenceError that Electron reports only as "Script failed to execute". + it('runs after being stringified and eval’d with no module scope', () => { + const target = document.createElement('button') + document.body.append(target) + + const injected = new Function('return (' + watchInPage.toString() + ')')() as typeof watchInPage + + expect(() => injected(document, { aimed: target }, 'aim', 'Clicking Save')).not.toThrow() + expect(host()).not.toBeNull() + }) +}) diff --git a/apps/desktop/src/lib/preview-act/watch-in-page.ts b/apps/desktop/src/lib/preview-act/watch-in-page.ts new file mode 100644 index 0000000000..0395548d0d --- /dev/null +++ b/apps/desktop/src/lib/preview-act/watch-in-page.ts @@ -0,0 +1,1063 @@ +/** + * WATCH OVERLAY — paints the agent's actions onto the live preview page so a + * person can follow along: the field of things it can reach, a box round the + * one it is about to touch, and a cursor that goes there. + * + * There are exactly two things on screen, and everything the agent does is said + * with one of them: + * + * - THE CURSOR — one arrow, built once, never removed. It is the + * collaborative-editing kind, because that is exactly what this is: a second + * party moving around a document you are also looking at. Arrow geometry is + * Liveblocks' (Apache-2.0). Plain black or white against the guest's own + * colour scheme rather than a brand colour: it stands for a hand, and a hand + * is not a feature of this app. + * - THE MARK — a rectangle glued to an element. A candidate, a lock, a hit + * and a pin are all the same object with a different row in `kinds`: how + * heavy the rule is, how long it lives, how it arrives. Marks are a design + * tool's selection blue, deliberately NOT the + * app's accent — this is chrome drawn on top of someone else's page, and + * every design tool has trained people to read that blue as "a tool has + * selected this", where a branded colour would read as part of the page. + * + * Every mark is two elements: a shell the tracker positions, and a skin that + * carries the border and any animation. They are split because both want + * `transform`, and a mark that animates the property the tracker rewrites every + * frame either stops moving or stops animating. + * + * Injected as source (`watchInPage.toString()`) exactly like the act engine, so + * the same rule holds: no imports, no closure references, no module-scope + * constants — everything the body needs is declared inside it. + * + * Three constraints shape the implementation, each one learned from how + * Playwright, Stagehand and browser-use solve the same problem: + * + * - A page can set `z-index: 2147483647` too, and the browser's top layer + * (``, `popover`, fullscreen) outranks every z-index there is. So + * the host joins the top layer via `popover="manual"` instead of trying to + * outbid the page, and keeps the z-index only as a fallback. + * - A strict CSP rejects `style-src` at CSS PARSE time, which kills + * `cssText`, `setAttribute('style', …)` and any `