diff --git a/.github/workflows/install-e2e-run.yml b/.github/workflows/install-e2e-run.yml index 36553831c0..f45848a5e6 100644 --- a/.github/workflows/install-e2e-run.yml +++ b/.github/workflows/install-e2e-run.yml @@ -42,7 +42,7 @@ on: required: true type: string update-method: - description: 'How the install updates to HEAD. Supported: hermes-update (the updater), installer-script (re-run the one-liner), installer-script+desktop (re-run with --include-desktop). Declared-but-TODO methods (open-app-update, hermes-desktop-app-update) skip.' + description: 'How the install updates to HEAD. Supported: hermes-update (the updater), installer-script (re-run the one-liner), installer-script+desktop (re-run with --include-desktop), hermes-desktop-app-update (launch via hermes desktop under Playwright, click Update now). Declared-but-TODO methods (open-app-update) skip.' required: true type: string install-ref: @@ -61,10 +61,10 @@ on: type: string default: ubuntu-latest timeout-minutes: - description: 'Job timeout. A cold run installs real toolchains twice.' + description: 'Job timeout. A cold run installs real toolchains twice, and app-update legs add a full Electron build + launch.' required: false type: number - default: 45 + default: 75 permissions: contents: read @@ -73,13 +73,14 @@ jobs: e2e: name: install & update # The pairs the driver can run today; anything else is a declared TODO - # and natively skips. The +desktop variants also need the starting tag - # to ship apps/desktop (the --include-desktop flag shipped with it). + # and natively skips. Desktop-surface methods (+desktop installs, + # hermes-desktop-app-update) also need the starting tag to ship + # apps/desktop (their flags shipped with it). if: >- (inputs.install-method == 'installer-script' || (inputs.install-method == 'installer-script+desktop' && inputs.tag-has-desktop)) && (contains(fromJSON('["hermes-update", "installer-script"]'), inputs.update-method) - || (inputs.update-method == 'installer-script+desktop' && inputs.tag-has-desktop)) + || (contains(fromJSON('["installer-script+desktop", "hermes-desktop-app-update"]'), inputs.update-method) && inputs.tag-has-desktop)) runs-on: ${{ inputs.runner }} timeout-minutes: ${{ inputs.timeout-minutes }} diff --git a/scripts/launch_capture_probe.sh b/scripts/launch_capture_probe.sh new file mode 100755 index 0000000000..45dc5ee06f --- /dev/null +++ b/scripts/launch_capture_probe.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Probe-the-probe for launch-capture/sitecustomize.py, no real install needed. +# Control rows FIRST: without the opt-in env var, and for non-launch argv +# shapes, subprocess.run must behave untouched. Then treatment rows: both +# launch shapes must be captured without spawning. +set -euo pipefail +CAP_DIR="$(cd "$(dirname "$0")" && pwd)/../tests/install/e2e-assets/launch-capture" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +run_py() { + # $1: with_var (yes/no); rest: python -c payload + local with_var="$1"; shift + if [ "$with_var" = yes ]; then + PYTHONPATH="$CAP_DIR${PYTHONPATH:+:$PYTHONPATH}" \ + HERMES_E2E_CAPTURE_LAUNCH="$WORK/spec.json" python3 "$@" + else + PYTHONPATH="$CAP_DIR${PYTHONPATH:+:$PYTHONPATH}" python3 "$@" + fi +} + +fail() { echo "PROBE FAILED: $*" >&2; exit 1; } + +echo "--- control 1: no env var -> run() untouched, real spawn happens" +out="$(run_py no -c 'import subprocess; print(subprocess.run(["echo","real-spawn"],capture_output=True,text=True).stdout.strip())')" +[ "$out" = "real-spawn" ] || fail "control 1: expected real-spawn, got '$out'" +[ ! -e "$WORK/spec.json" ] || fail "control 1: spec written without opt-in" +echo "OK" + +echo "--- control 2: env var set, NON-launch argv (npm run pack) -> passthrough" +rm -f "$WORK"/spec.json* +out="$(run_py yes -c 'import subprocess; r=subprocess.run(["echo","npm-build-ran"],capture_output=True,text=True); print(r.stdout.strip())')" +[ "$out" = "npm-build-ran" ] || fail "control 2: echo did not run" +# and an actual npm-shaped BUILD argv (argv[0]=npm but no electron token): +run_py yes -c 'import subprocess,sys; r=subprocess.run(["npm","run","pack"],capture_output=True); sys.exit(0)' 2>/dev/null || true +[ ! -e "$WORK/spec.json" ] || fail "control 2: build argv was wrongly captured" +echo "OK" + +echo "--- treatment 1: source shape (npm exec -- electron .) captured, not spawned" +rm -f "$WORK"/spec.json* +run_py yes -c ' +import subprocess +r = subprocess.run(["npm", "exec", "--", "electron", "."], cwd="/tmp", env={"HERMES_DESKTOP_CWD": "/tmp", "PATH": "/usr/bin"}) +assert r.returncode == 0, r +' +[ -e "$WORK/spec.json" ] || fail "treatment 1: no spec written" +[ "$(cat "$WORK/spec.json.captured")" = "source" ] || fail "treatment 1: wrong shape" +python3 - "$WORK/spec.json" <<'EOF' +import json, sys +spec = json.load(open(sys.argv[1])) +assert spec["argv"] == ["npm", "exec", "--", "electron", "."], spec["argv"] +assert spec["cwd"] == "/tmp", spec["cwd"] +assert spec["env"]["HERMES_DESKTOP_CWD"] == "/tmp", "env= kwarg not captured" +assert spec["matchedShape"] == "source" +print("spec contents OK") +EOF +echo "OK" + +echo "--- treatment 2: packaged shape captured, not spawned" +rm -f "$WORK"/spec.json* +run_py yes -c ' +import subprocess +exe = "/x/apps/desktop/release/linux-unpacked/Hermes" +r = subprocess.run([exe, "--no-sandbox"], cwd="/tmp", env={"PATH": "/usr/bin"}) +assert r.returncode == 0, r # a real spawn of this path would ENOENT +' +[ "$(cat "$WORK/spec.json.captured")" = "packaged" ] || fail "treatment 2: wrong shape" +echo "OK" + +echo "--- treatment 3: windows-style packaged argv matches too" +rm -f "$WORK"/spec.json* +run_py yes -c ' +import subprocess +r = subprocess.run(["C:\\x\\apps\\desktop\\release\\win-unpacked\\Hermes.exe"], env={}) +assert r.returncode == 0 +' +[ "$(cat "$WORK/spec.json.captured")" = "packaged" ] || fail "treatment 3: wrong shape" +echo "OK" + +echo "ALL PROBES PASSED" diff --git a/tests/install/README.md b/tests/install/README.md index e9de9fb35b..54cb7f8957 100644 --- a/tests/install/README.md +++ b/tests/install/README.md @@ -54,7 +54,7 @@ A leg can install a release from months back. The driver must not assume that th The desktop app has two launch paths, so the matrix has two app-update methods. Both click "Update now" in the running app. They differ in how the app starts: - `open-app-update`: the app starts from the OS entry point that the install created. On windows these are the Start Menu and Desktop shortcuts to the installed `Hermes.exe`; the desktop installer always creates them. The installer scripts do not create entry points: their opt-in desktop stage (`--include-desktop` / `-IncludeDesktop`) builds the app inside the checkout but does not register it with the OS. So `open-app-update` legs pair with a `desktop-installer` install. -- `hermes-desktop-app-update`: the app starts with the `hermes desktop` command. Every install method provides this command, on each OS that ships the desktop app. On linux this is the only app surface: no desktop installer and no packaged desktop artifact exist for linux. +- `hermes-desktop-app-update`: the app starts with the `hermes desktop` command. Every install method provides this command, on each OS that ships the desktop app. On linux this is the only app surface: no desktop installer and no packaged desktop artifact exist for linux. The driver captures the product's own launch call (argv, cwd, environment) with `e2e-assets/launch-capture/sitecustomize.py` and re-executes it under Playwright, which owns the app and clicks the update flow. ## Skips diff --git a/tests/install/e2e-assets/launch-capture/sitecustomize.py b/tests/install/e2e-assets/launch-capture/sitecustomize.py new file mode 100644 index 0000000000..bf7b391169 --- /dev/null +++ b/tests/install/e2e-assets/launch-capture/sitecustomize.py @@ -0,0 +1,102 @@ +"""Driver-side spawn interception for hermes desktop E2E legs. + +The installed ``hermes`` is a venv console script, so its interpreter +imports ``sitecustomize`` at startup when this directory is on +``PYTHONPATH``. Behind an explicit env-var opt-in the module wraps +``subprocess.run`` so the FINAL electron launch call of ``hermes +desktop`` is captured -- argv, cwd, and the fully-constructed ``env`` +kwarg written to a JSON spec -- and replaced with a fake success instead +of spawning. Everything before the spawn (build, stamps, integrity gate, +sandbox fixup) runs for real, in the REAL installed code of whatever +version is under test; Playwright's ``_electron.launch`` then owns the +app from spawn using exactly the spec the product would have used. + +This ships with the DRIVER, never with the product, so it works +identically on every sampled OLD ref -- version drift in the launch +shapes is the matcher's problem, which lives here, next to the driver +(the same maintenance model as ``installer_supports()``). + +Opt-in: ``HERMES_E2E_CAPTURE_LAUNCH=`` -- the spec is written +there, and the marker file ``.captured`` distinguishes "hermes +desktop exited 0 and we captured" from "exited 0 without reaching a +launch" (a version that errors out earlier must FAIL the leg, loudly). + +Launch shapes across sampled desktop-era tags (verified against each +tag's own hermes_cli/main.py): + + v2026.6.5 subprocess.run([npm, "exec", "--", "electron", "."], ...) + v0.20.0+ the npm-exec form AND subprocess.run(launch_command, ...) + where launch_command[0] is the packaged app executable + under apps/desktop/release/ + +Both go through ``subprocess.run`` with an explicit ``env=`` kwarg. npm +BUILD calls (``npm run build`` / ``npm run pack``) carry no ``electron`` +token in argv and pass through untouched -- they must run for real. +""" + +import os + +_SPEC_PATH = os.environ.get("HERMES_E2E_CAPTURE_LAUNCH") + +if _SPEC_PATH: + import json + import subprocess + + _SPEC: str = _SPEC_PATH + _real_run = subprocess.run + + def _basename_noext(token: str) -> str: + base = os.path.basename(str(token)) + for ext in (".exe", ".cmd", ".bat"): + if base.lower().endswith(ext): + base = base[: -len(ext)] + return base.lower() + + def _match_shape(argv: "list[str]") -> str: + """Return the launch shape for argv, or '' when it is not a launch.""" + if not argv: + return "" + tokens = [str(t) for t in argv] + head = _basename_noext(tokens[0]) + # Source shape: npm/npx invoking electron ("npm exec -- electron ."). + # Membership, not position: absorb argv drift across versions. Build + # calls ("npm run pack") carry no bare "electron" token. + if head in ("npm", "npx"): + if any(_basename_noext(t) == "electron" for t in tokens[1:]): + return "source" + return "" + # Packaged shape: argv[0] is the packaged app executable under + # apps/desktop/release/ (win-unpacked/Hermes.exe, linux-unpacked/..., + # mac*/Hermes.app/Contents/MacOS/...). + first = tokens[0].replace("\\", "/") + if "apps/desktop/release/" in first: + return "packaged" + return "" + + def _capturing_run(*args, **kwargs): + argv = args[0] if args else kwargs.get("args") + if not isinstance(argv, (list, tuple)): + return _real_run(*args, **kwargs) + tokens = [str(t) for t in argv] + shape = _match_shape(tokens) + if not shape: + return _real_run(*args, **kwargs) + env = kwargs.get("env") + spec = { + "argv": tokens, + "cwd": str(kwargs.get("cwd") or os.getcwd()), + # Capture what the child would ACTUALLY get: the constructed + # env= when present, the ambient environment when not. + "env": dict(env) if env is not None else dict(os.environ), + "matchedShape": shape, + } + tmp = _SPEC + ".tmp" + with open(tmp, "w", encoding="utf-8") as fh: + json.dump(spec, fh, indent=2) + os.replace(tmp, _SPEC) + with open(_SPEC + ".captured", "w", encoding="utf-8") as fh: + fh.write(shape) + print(f"[e2e launch-capture] captured {shape} launch -> {_SPEC} (not spawning)") + return subprocess.CompletedProcess(tokens, 0, stdout=None, stderr=None) + + subprocess.run = _capturing_run diff --git a/tests/install/e2e-assets/launch-from-spec.mjs b/tests/install/e2e-assets/launch-from-spec.mjs new file mode 100644 index 0000000000..e203ba5714 --- /dev/null +++ b/tests/install/e2e-assets/launch-from-spec.mjs @@ -0,0 +1,175 @@ +// @ts-check +/** + * Launch the Hermes desktop app from a captured launch spec and click the + * real update flow: Settings -> About -> "Update now". + * + * The spec is written by launch-capture/sitecustomize.py at `hermes + * desktop`'s own spawn site, so argv, cwd, and the fully-constructed env + * are the product's own -- this launcher only translates the npm-exec + * source shape into a direct electron binary path (Playwright needs a + * real executable, and the electron npm shim would re-spawn out of our + * control). + * + * Usage (from the scratch dir where the driver installed @playwright/test): + * node launch-from-spec.mjs --spec /path/launch-spec.json \ + * [--result $HERMES_HOME/.hermes-update-result.json] \ + * [--expect-sha --repo-dir ] [--no-update] + * + * --no-update: launch + wait for the window + close. The smoke arm. + * Otherwise: click Update now, then poll for completion. Two signals, + * either satisfies (poll whichever are given, first hit wins): + * --result the windows hand-off's result file + * (HERMES_HOME/.hermes-update-result.json) + * --expect-sha the installed checkout reaching the expected commit - + * the source-install signal, where the About pane's update + * runs `hermes update` and no result file exists. + * The Playwright close event is unreliable across the update handoff, so + * neither signal is an app event. + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { parseArgs } from 'node:util'; +import { _electron } from '@playwright/test'; + +/** + * @typedef {{argv: string[], cwd: string, env: Record, + * matchedShape: 'source' | 'packaged'}} LaunchSpec + */ + +/** + * Resolve what _electron.launch needs from a captured spec. + * @param {LaunchSpec} spec + * @returns {{executablePath: string, args: string[], cwd: string, + * env: Record}} + */ +export function resolveLaunch(spec) { + if (spec.matchedShape === 'packaged') { + return { + executablePath: spec.argv[0], + args: spec.argv.slice(1), + cwd: spec.cwd, + env: spec.env, + }; + } + // Source shape: ["npm", "exec", "--", "electron", ".", ...extra] running + // in apps/desktop. Electron's real binary lives in the workspace-hoisted + // node_modules; `electron/index.js` exports its path but requires the + // module -- cheaper here to read the path file it derives from. + const desktopDir = spec.cwd; + const idx = spec.argv.findIndex((t) => t === 'electron'); + const extra = idx >= 0 ? spec.argv.slice(idx + 1).filter((t) => t !== '.') : []; + const candidates = [ + path.join(desktopDir, 'node_modules', 'electron'), + path.join(desktopDir, '..', '..', 'node_modules', 'electron'), + ]; + for (const moduleDir of candidates) { + const pathTxt = path.join(moduleDir, 'path.txt'); + if (!fs.existsSync(pathTxt)) continue; + const rel = fs.readFileSync(pathTxt, 'utf8').trim(); + const exe = path.join(moduleDir, 'dist', rel); + if (fs.existsSync(exe)) { + return { executablePath: exe, args: ['.', ...extra], cwd: desktopDir, env: spec.env }; + } + } + throw new Error(`no electron binary found under ${candidates.join(' or ')}`); +} + +/** @param {string} msg */ +function log(msg) { + console.log(`[launch-from-spec] ${msg}`); +} + +async function main() { + const { values } = parseArgs({ + options: { + spec: { type: 'string' }, + result: { type: 'string' }, + 'expect-sha': { type: 'string' }, + 'repo-dir': { type: 'string' }, + 'no-update': { type: 'boolean', default: false }, + 'timeout-ms': { type: 'string', default: '600000' }, + }, + }); + if (!values.spec) throw new Error('--spec is required'); + /** @type {LaunchSpec} */ + const spec = JSON.parse(fs.readFileSync(values.spec, 'utf8')); + const launch = resolveLaunch(spec); + log(`launching ${launch.executablePath} (shape: ${spec.matchedShape})`); + + const app = await _electron.launch({ + executablePath: launch.executablePath, + args: launch.args, + cwd: launch.cwd, + env: launch.env, + }); + const window = await app.firstWindow({ timeout: 120_000 }); + await window.waitForLoadState('domcontentloaded'); + log(`window up: ${await window.title()}`); + await window.screenshot({ path: `${values.spec}.window.png` }).catch(() => {}); + + if (values['no-update']) { + log('smoke mode: window proven, closing'); + await app.close().catch(() => {}); + return; + } + + if (!values.result && !(values['expect-sha'] && values['repo-dir'])) { + throw new Error('need --result and/or --expect-sha + --repo-dir unless --no-update'); + } + const deadline = Date.now() + Number(values['timeout-ms']); + + // Dismiss the onboarding overlay when present (fresh HERMES_HOME). + const skip = window.getByRole('button', { name: /skip|get started|continue/i }).first(); + if (await skip.isVisible({ timeout: 5_000 }).catch(() => false)) { + await skip.click().catch(() => {}); + } + + // Settings -> About -> Update now. Selectors favor accessible names over + // DOM structure so renderer refactors don't break the leg. + await window.getByRole('button', { name: /settings/i }).first().click(); + await window.getByRole('tab', { name: /about/i }).or( + window.getByRole('button', { name: /about/i })).first().click(); + const updateNow = window.getByRole('button', { name: /update now/i }).first(); + await updateNow.waitFor({ state: 'visible', timeout: 60_000 }); + await updateNow.click(); + log('clicked Update now; polling for result file'); + + // The app may relaunch/exit during the update; completion signals are + // product state, not Playwright events. + const resultPath = values.result; + const expectSha = values['expect-sha']; + const repoDir = values['repo-dir']; + /** @returns {string} */ + const headSha = () => { + try { + return execFileSync('git', ['-C', /** @type {string} */ (repoDir), 'rev-parse', 'HEAD'], { + encoding: 'utf8', + }).trim(); + } catch { + return ''; + } + }; + for (;;) { + if (resultPath && fs.existsSync(resultPath)) { + log(`update result present: ${fs.readFileSync(resultPath, 'utf8').slice(0, 200)}`); + break; + } + if (expectSha && repoDir && headSha() === expectSha) { + log(`checkout reached expected sha ${expectSha}`); + break; + } + if (Date.now() > deadline) { + await window.screenshot({ path: `${values.spec}.timeout.png` }).catch(() => {}); + throw new Error('update completion signal never appeared (result file / expected sha)'); + } + await new Promise((r) => setTimeout(r, 2_000)); + } + await app.close().catch(() => {}); +} + +const invoked = process.argv[1] && path.resolve(process.argv[1]) === (await import('node:url')).fileURLToPath(import.meta.url); +if (invoked) { + await main(); +} diff --git a/tests/install/installer-script-e2e.sh b/tests/install/installer-script-e2e.sh index d2e1a0a009..bc6078f3be 100755 --- a/tests/install/installer-script-e2e.sh +++ b/tests/install/installer-script-e2e.sh @@ -35,6 +35,9 @@ # --update-method hermes-update `hermes update` # installer-script re-run install.sh (HEAD's copy) # installer-script+desktop re-run with --include-desktop +# hermes-desktop-app-update launch the app via `hermes +# desktop` (spawn captured, Playwright +# drives it) and click Update now # --install-ref what to install first; anything git resolves. Default: # the newest release tag in the checkout. # @@ -65,8 +68,8 @@ case "$INSTALL_METHOD" in *) echo "error: --install-method must be installer-script or installer-script+desktop, got '$INSTALL_METHOD'" >&2; exit 1 ;; esac case "$UPDATE_METHOD" in - hermes-update|installer-script|installer-script+desktop) ;; - *) echo "error: --update-method must be hermes-update, installer-script or installer-script+desktop, got '$UPDATE_METHOD'" >&2; exit 1 ;; + hermes-update|installer-script|installer-script+desktop|hermes-desktop-app-update) ;; + *) echo "error: --update-method must be hermes-update, installer-script, installer-script+desktop or hermes-desktop-app-update, got '$UPDATE_METHOD'" >&2; exit 1 ;; esac REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" @@ -290,6 +293,51 @@ case "$UPDATE_METHOD" in run_installer "$HEAD_SHA" head desktop assert_desktop_artifact HEAD ;; + hermes-desktop-app-update) + # The real user surface: `hermes desktop` launches the app, the user + # clicks Settings -> About -> Update now. Playwright must OWN the spawn + # (it needs the inspection pipe), so the driver intercepts the product's + # own launch call - argv/cwd/env captured at the spawn site by + # e2e-assets/launch-capture/sitecustomize.py - and re-executes it under + # _electron.launch. Everything before the spawn (build, stamps, sandbox + # fixup) runs for real in the installed code. + HERMES="$INSTALL_DIR/venv/bin/hermes" + ASSETS="$REPO_ROOT/tests/install/e2e-assets" + SPEC="$WORK_ROOT/launch-spec.json" + + step "capturing the hermes desktop launch spec (build runs for real)" + rc=0 + (cd "$INSTALL_DIR" && \ + PYTHONPATH="$ASSETS/launch-capture${PYTHONPATH:+:$PYTHONPATH}" \ + HERMES_E2E_CAPTURE_LAUNCH="$SPEC" \ + "$HERMES" desktop < /dev/null > "$LOG_DIR/desktop-launch-capture.log" 2>&1) || rc=$? + log_group "hermes desktop (launch capture) transcript" "$LOG_DIR/desktop-launch-capture.log" + [ "$rc" -eq 0 ] || fail "hermes desktop exited $rc during launch capture; transcript above" + # Exit 0 without a capture means a version that never reached its + # launch - that must fail loudly, not pass as a no-op. + [ -f "$SPEC.captured" ] || fail "hermes desktop exited 0 but no launch was captured at $SPEC" + ok "captured $(cat "$SPEC.captured") launch spec" + + step "driving the app under Playwright: Settings -> About -> Update now" + # Driver tooling comes from the driver: a scratch dir with our own + # pinned @playwright/test, never resolved from the installed tree + # (older OLD refs predate the dependency; hoisting moves it around). + PW_DIR="$WORK_ROOT/playwright" + mkdir -p "$PW_DIR" + (cd "$PW_DIR" && npm install --no-save --no-audit --no-fund \ + "@playwright/test@1.58.2" > "$LOG_DIR/playwright-install.log" 2>&1) \ + || { log_group "playwright install transcript" "$LOG_DIR/playwright-install.log"; fail "playwright install failed"; } + cp "$ASSETS/launch-from-spec.mjs" "$PW_DIR/" + rc=0 + (cd "$PW_DIR" && node launch-from-spec.mjs \ + --spec "$SPEC" \ + --result "$HERMES_HOME/.hermes-update-result.json" \ + --expect-sha "$HEAD_SHA" \ + --repo-dir "$INSTALL_DIR" \ + > "$LOG_DIR/app-update.log" 2>&1) || rc=$? + log_group "app update (Playwright) transcript" "$LOG_DIR/app-update.log" + [ "$rc" -eq 0 ] || fail "app-driven update exited $rc; transcript above" + ;; esac assert_checkout "$HEAD_SHA" HEAD smoke_desktop head