From 1bf0a1c95be2df7d4bb66d8f8489b8503fbd9daa Mon Sep 17 00:00:00 2001 From: ethernet Date: Tue, 11 Aug 2026 15:21:24 -0400 Subject: [PATCH] ci(install-e2e): generate every install/update combination - one job per combo Replace the hand-enumerated update/installer/windows-desktop jobs with a support-matrix generator (scripts/sandbox/generate-e2e-matrix.mjs). The spec declares every {os, install-method, update-method} combination a user could be on; generate-matrix expands it and fans out ONE JOB PER COMBINATION: * linux combos (curl-bash install x hermes-update/curl-bash rerun) drive install-e2e-run.yml, still multiplied by the sampled release tags from pick-releases; * the windows combo (desktop-installer@latest -> desktop-app) drives install-e2e-windows-run.yml - the real GUI flow; * every declared-but-unimplemented combo (all of macOS, the remaining windows methods) becomes its own visible skipped job, so the coverage gap is enumerable from the Checks tab and implementing one is a one-line move into IMPLEMENTED. Strictness carried into the generator: method ids validate against a closed set, installer 'versions' arrays only allow 'latest' until a versioned archive exists, secondUpdate must stay empty until a chained second-update leg is implemented, and unknown spec keys/routes throw. Expansion, route filtering, empty-matrix gating, and all seven error paths verified locally; the dispatch route choice keeps its exact previous semantics (all/both/update/installer/windows-desktop). --- .github/workflows/install-e2e.yml | 133 ++++++++----- scripts/sandbox/generate-e2e-matrix.mjs | 238 ++++++++++++++++++++++++ 2 files changed, 326 insertions(+), 45 deletions(-) create mode 100644 scripts/sandbox/generate-e2e-matrix.mjs diff --git a/.github/workflows/install-e2e.yml b/.github/workflows/install-e2e.yml index 08d46582d0..29d169a005 100644 --- a/.github/workflows/install-e2e.yml +++ b/.github/workflows/install-e2e.yml @@ -2,22 +2,28 @@ name: Install & Update E2E # Can a user on a released version get to this commit? # -# For each release we sample, a leg installs that release through the real -# `curl | install.sh` one-liner (uv, a managed Python, Node, the venv) inside -# scripts/dev-sandbox.sh, then applies one update route and requires the -# checkout to land on this commit with a working `hermes`. +# The support matrix -- every {os, install-method, update-method} combination +# a user could be on -- lives in scripts/sandbox/generate-e2e-matrix.mjs. +# The generate-matrix job expands it into ONE JOB PER COMBINATION: # -# The starting versions are chosen at runtime from the repo's release tags -# (scripts/sandbox/pick-release-tags.sh): newest, oldest, and a spread between. -# A hardcoded list would stop covering the newest release the day after it -# ships, and would pin an "oldest" that nobody still runs. +# * linux legs install a sampled release through the real +# `curl | install.sh` one-liner (uv, a managed Python, Node, the venv) +# inside scripts/dev-sandbox.sh, then apply the combo's update route and +# require the checkout to land on this commit with a working `hermes`; +# * windows legs run the real desktop user flow (install-e2e-windows-run +# .yml): the website's Hermes-Setup.exe clicked by AutoHotkey, then the +# update driven through the app itself, Playwright clicking "Update now"; +# * combinations the spec declares but CI can't drive yet (macOS, the +# remaining windows install/update methods) surface as skipped jobs so +# the coverage gap is enumerable instead of buried in comments. # -# A separate Windows axis (install-e2e-windows-run.yml) covers the desktop -# user flow: install OLD with the website's published bootstrap installer -# (headed, AutoHotkey-clicked), then update to HEAD through a real update -# surface — the app's own Update button clicked by Playwright. No tag -# matrix there: the published exe's own build pin is the starting point -# users actually have. +# The linux starting versions are chosen at runtime from the repo's release +# tags (scripts/sandbox/pick-release-tags.sh): newest, oldest, and a spread +# between. A hardcoded list would stop covering the newest release the day +# after it ships, and would pin an "oldest" that nobody still runs. The +# windows starting version is the newest release tag (the published +# installer has no commit pin -- it installs what `main` serves, which the +# harness stages to that tag). # # Triggers: # * every 12 hours, so upstream drift (a new uv, a Node bump, a PyPI change) @@ -62,7 +68,7 @@ concurrency: jobs: # Which released versions do we test updating FROM? Resolved once and shared - # by both route matrices, so the two routes cover the same set. + # by every linux leg, so all routes cover the same set. pick-releases: name: Pick release tags runs-on: ubuntu-latest @@ -86,44 +92,81 @@ jobs: echo "Testing updates from: $tags" echo "tags=$tags" >> "$GITHUB_OUTPUT" - # `hermes update` -- the route most users take. - update: - if: github.event_name != 'workflow_dispatch' || contains(fromJSON('["all", "both", "update"]'), inputs.route) + # Expand the {os, install-method, update-method} support matrix + # (scripts/sandbox/generate-e2e-matrix.mjs) into one job per combination. + # Implemented combos land in the linux/windows matrices below; everything + # the spec declares but CI can't drive yet lands in `skipped`, so the TODO + # surface is visible in every run instead of buried in comments. + generate-matrix: + name: Generate combination matrix needs: pick-releases + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + linux: ${{ steps.gen.outputs.linux }} + windows: ${{ steps.gen.outputs.windows }} + skipped: ${{ steps.gen.outputs.skipped }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: scripts/sandbox/generate-e2e-matrix.mjs + sparse-checkout-cone-mode: false + - id: gen + run: | + set -euo pipefail + matrices="$(node scripts/sandbox/generate-e2e-matrix.mjs \ + --tags '${{ needs.pick-releases.outputs.tags }}' \ + --route '${{ inputs.route || 'all' }}')" + echo "$matrices" + for key in linux windows skipped; do + echo "$key=$(echo "$matrices" | node -e 'let d="";process.stdin.on("data",c=>d+=c).on("end",()=>console.log(JSON.stringify(JSON.parse(d)[process.argv[1]])))' "$key")" >> "$GITHUB_OUTPUT" + done + + # Linux sandbox legs: one job per {install-method -> update-method, release + # tag} combination, each a real curl|bash install in bubblewrap followed by + # the combo's update route. + linux: + name: "linux: ${{ matrix.name }}" + if: fromJSON(needs.generate-matrix.outputs.linux).include[0] != null + needs: generate-matrix strategy: - # One release breaking is worth knowing about even if another already + # One combo breaking is worth knowing about even if another already # failed, so let every leg report. fail-fast: false - matrix: - install-ref: ${{ fromJSON(needs.pick-releases.outputs.tags) }} + max-parallel: 4 + matrix: ${{ fromJSON(needs.generate-matrix.outputs.linux) }} uses: ./.github/workflows/install-e2e-run.yml with: - route: update - install-ref: ${{ matrix.install-ref }} + route: ${{ matrix.route }} + install-ref: ${{ matrix.install_ref }} - # Re-running the curl one-liner over an existing checkout: autostash + pull - # rather than the updater's own git handling. - installer: - if: github.event_name != 'workflow_dispatch' || contains(fromJSON('["all", "both", "installer"]'), inputs.route) - needs: pick-releases + # Windows GUI legs: one job per combination of the real desktop user flow + # (website installer clicked by AutoHotkey, update driven through the app). + windows: + name: "windows: ${{ matrix.name }}" + if: fromJSON(needs.generate-matrix.outputs.windows).include[0] != null + needs: generate-matrix strategy: fail-fast: false - max-parallel: 3 - matrix: - install-ref: ${{ fromJSON(needs.pick-releases.outputs.tags) }} - uses: ./.github/workflows/install-e2e-run.yml - with: - route: installer - install-ref: ${{ matrix.install-ref }} - - # Windows desktop: install OLD with the website's published bootstrap - # installer (the exact bits a user downloads today, AutoHotkey-clicked), - # then update to HEAD by clicking the app's own Update button under - # Playwright. The reusable workflow's route input selects the update - # mechanism; `hermes update` and installer-rerun arms are declared TODOs - # there. - windows-desktop: - if: github.event_name != 'workflow_dispatch' || contains(fromJSON('["all", "windows-desktop"]'), inputs.route) + matrix: ${{ fromJSON(needs.generate-matrix.outputs.windows) }} uses: ./.github/workflows/install-e2e-windows-run.yml with: - route: desktop + route: ${{ matrix.route }} + install-ref: ${{ matrix.install_ref }} + + # Declared-but-unimplemented combinations. Each shows up as its own skipped + # check so coverage gaps are enumerable from the Checks tab; implementing + # one moves it from here into IMPLEMENTED in the generator. + skipped: + name: "skipped: ${{ matrix.os }}: ${{ matrix.install }} -> ${{ matrix.update }}" + if: fromJSON(needs.generate-matrix.outputs.skipped).include[0] != null + needs: generate-matrix + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.generate-matrix.outputs.skipped) }} + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - run: >- + echo "SKIPPED (${{ matrix.reason }}) -- ${{ matrix.os }} install via + '${{ matrix.install }}', update via '${{ matrix.update }}'" diff --git a/scripts/sandbox/generate-e2e-matrix.mjs b/scripts/sandbox/generate-e2e-matrix.mjs new file mode 100644 index 0000000000..21b257c5e3 --- /dev/null +++ b/scripts/sandbox/generate-e2e-matrix.mjs @@ -0,0 +1,238 @@ +#!/usr/bin/env node +/** + * Expand the install/update support matrix into concrete E2E combinations. + * + * One source of truth for every {os, install-method, update-method} pair a + * user could be on. Implemented combos fan out into real CI jobs -- one job + * per combination -- and everything else lands in a "skipped" matrix so the + * TODO surface stays visible in every run instead of buried in comments. + * + * Used by .github/workflows/install-e2e.yml: + * + * node scripts/sandbox/generate-e2e-matrix.mjs \ + * --tags '["v2026.8.3","v2026.3.12"]' --route all + * + * Prints JSON: { linux: {include:[...]}, windows: {include:[...]}, + * skipped: {include:[...]} }. The linux include entries carry + * {name, route, install_ref} for install-e2e-run.yml; windows entries carry + * {name, route, install_ref} for install-e2e-windows-run.yml. + */ + +import path from 'node:path'; +import { parseArgs } from 'node:util'; +import { fileURLToPath } from 'node:url'; + +/** + * Method ids are strict machine strings -- workflows and the IMPLEMENTED + * table key off them, so an unknown id must fail loudly (see KNOWN_METHODS). + * + * `versions` on an entry expands it into one combination per version. Only + * "latest" is allowed today: it means "the artifact published on the website + * right now" (Hermes-Setup.exe has no versioned archive yet). When archived + * installer versions exist, widen ALLOWED_VERSIONS. + */ +export const SPEC = { + windows: { + install: [ + // irm https://hermes.nousresearch.com/install.ps1 | iex + { method: 'irm-iex' }, + // Website Hermes-Setup.exe, clicked through the GUI. + { method: 'desktop-installer', versions: ['latest'] }, + ], + update: [ + { method: 'irm-iex' }, + // Run the bootstrap exe again over an existing install (--update flow). + { method: 'desktop-installer-rerun', versions: ['latest'] }, + { method: 'hermes-update' }, + // Settings -> About -> "Update now" inside the running desktop app. + { method: 'desktop-app' }, + ], + }, + macos: { + install: [ + { method: 'curl-bash' }, + { method: 'packaged-app' }, + ], + update: [ + { method: 'curl-bash' }, + { method: 'hermes-update' }, + { method: 'app-update' }, + ], + }, + linux: { + install: [ + { method: 'curl-bash' }, + ], + update: [ + { method: 'curl-bash' }, + { method: 'hermes-update' }, + ], + }, +}; + +const KNOWN_METHODS = new Set([ + 'irm-iex', + 'desktop-installer', + 'desktop-installer-rerun', + 'hermes-update', + 'desktop-app', + 'curl-bash', + 'packaged-app', + 'app-update', +]); + +const ALLOWED_VERSIONS = new Set(['latest']); + +/** + * The combinations CI actually runs today, and which reusable workflow + + * route input drives each. Everything in SPEC but not here is emitted as a + * skipped combo. Install ids include the expanded @version suffix. + */ +export const IMPLEMENTED = [ + // scripts/dev-sandbox.sh legs (install-e2e-run.yml, bubblewrap sandbox). + { os: 'linux', install: 'curl-bash', update: 'hermes-update', workflow: 'linux', route: 'update' }, + { os: 'linux', install: 'curl-bash', update: 'curl-bash', workflow: 'linux', route: 'installer' }, + // Real Windows GUI flow (install-e2e-windows-run.yml): website exe clicked + // by AutoHotkey, Update now clicked by Playwright. + { os: 'windows', install: 'desktop-installer@latest', update: 'desktop-app', workflow: 'windows', route: 'desktop' }, +]; + +function validateEntry(os, kind, entry) { + if (typeof entry.method !== 'string' || !KNOWN_METHODS.has(entry.method)) { + throw new Error(`${os}.${kind}: unknown method id ${JSON.stringify(entry.method)} -- add it to KNOWN_METHODS if intentional`); + } + if ('versions' in entry) { + if (!Array.isArray(entry.versions) || entry.versions.length === 0) { + throw new Error(`${os}.${kind}.${entry.method}: versions must be a non-empty array`); + } + for (const v of entry.versions) { + if (!ALLOWED_VERSIONS.has(v)) { + throw new Error(`${os}.${kind}.${entry.method}: version ${JSON.stringify(v)} not allowed -- only ${[...ALLOWED_VERSIONS].join(', ')} until versioned installer archives exist`); + } + } + } + const unknown = Object.keys(entry).filter((k) => k !== 'method' && k !== 'versions'); + if (unknown.length) { + throw new Error(`${os}.${kind}.${entry.method}: unknown keys ${unknown.join(', ')}`); + } +} + +/** Expand one method entry into concrete ids ("desktop-installer@latest"). */ +export function expandMethod(os, kind, entry) { + validateEntry(os, kind, entry); + if (!entry.versions) return [entry.method]; + return entry.versions.map((v) => `${entry.method}@${v}`); +} + +/** Every {os, install, update, secondUpdate} combination in SPEC. */ +export function generateEnvironments(spec) { + const envs = []; + for (const [os, osSpec] of Object.entries(spec)) { + const { install, update, secondUpdate = [], ...unknown } = osSpec; + if (Object.keys(unknown).length) { + throw new Error(`${os}: unknown spec keys ${Object.keys(unknown).join(', ')}`); + } + // Chained second updates (install -> update -> update again) are a real + // axis -- the updater that RESULTS from an update must itself update -- + // but nothing implements them yet. Refuse a spec that declares them so + // the first implementation is forced to come through here. + if (!Array.isArray(secondUpdate) || secondUpdate.length !== 0) { + throw new Error(`${os}: secondUpdate must be empty until a second-update leg is implemented`); + } + const installs = install.flatMap((e) => expandMethod(os, 'install', e)); + const updates = update.flatMap((e) => expandMethod(os, 'update', e)); + for (const i of installs) { + for (const u of updates) { + envs.push({ os, install: i, update: u, secondUpdate: '' }); + } + } + } + return envs; +} + +function findImplementation(env) { + return IMPLEMENTED.find( + (m) => m.os === env.os && m.install === env.install && m.update === env.update, + ); +} + +/** Mirror of install-e2e.yml's dispatch `route` choice. */ +function routeWants(route, env, impl) { + switch (route) { + case 'all': + return true; + case 'both': + return env.os === 'linux' && Boolean(impl); + case 'update': + return impl?.route === 'update'; + case 'installer': + return impl?.route === 'installer'; + case 'windows-desktop': + return env.os === 'windows' && Boolean(impl); + default: + throw new Error(`unknown route filter: ${JSON.stringify(route)}`); + } +} + +/** + * Split the combinations into per-workflow matrices. + * Linux combos fan out further over `tags` (which released version the leg + * installs first); Windows install versions come from the method id instead + * (desktop-installer@latest = whatever the website serves). + */ +export function buildMatrices(envs, { tags = [], route = 'all' } = {}) { + const linux = []; + const windows = []; + const skipped = []; + for (const env of envs) { + const impl = findImplementation(env); + if (!routeWants(route, env, impl)) continue; + if (!impl) { + skipped.push({ ...env, reason: 'not implemented yet' }); + } else if (impl.workflow === 'linux') { + if (tags.length === 0) { + throw new Error(`linux combo ${env.install} -> ${env.update} selected but no --tags given`); + } + for (const tag of tags) { + linux.push({ + name: `${env.install} @ ${tag} -> ${env.update}`, + route: impl.route, + install_ref: tag, + }); + } + } else if (impl.workflow === 'windows') { + windows.push({ + name: `${env.install} -> ${env.update}`, + route: impl.route, + install_ref: 'auto', + }); + } else { + throw new Error(`unknown workflow ${JSON.stringify(impl.workflow)} for ${env.os}`); + } + } + return { + linux: { include: linux }, + windows: { include: windows }, + skipped: { include: skipped }, + }; +} + +function main() { + const { values } = parseArgs({ + options: { + tags: { type: 'string', default: '[]' }, + route: { type: 'string', default: 'all' }, + }, + }); + const tags = JSON.parse(values.tags); + if (!Array.isArray(tags) || !tags.every((t) => typeof t === 'string')) { + throw new Error('--tags must be a JSON array of strings'); + } + const envs = generateEnvironments(SPEC); + const matrices = buildMatrices(envs, { tags, route: values.route }); + process.stdout.write(`${JSON.stringify(matrices, null, 2)}\n`); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) { + main(); +}