ci(install-e2e): generate every install/update combination - one job per combo

Replace the hand-enumerated update/installer/windows-desktop jobs with
a support-matrix generator (scripts/sandbox/generate-e2e-matrix.mjs).
The spec declares every {os, install-method, update-method} combination
a user could be on; generate-matrix expands it and fans out ONE JOB PER
COMBINATION:

* linux combos (curl-bash install x hermes-update/curl-bash rerun)
  drive install-e2e-run.yml, still multiplied by the sampled release
  tags from pick-releases;
* the windows combo (desktop-installer@latest -> desktop-app) drives
  install-e2e-windows-run.yml - the real GUI flow;
* every declared-but-unimplemented combo (all of macOS, the remaining
  windows methods) becomes its own visible skipped job, so the
  coverage gap is enumerable from the Checks tab and implementing one
  is a one-line move into IMPLEMENTED.

Strictness carried into the generator: method ids validate against a
closed set, installer 'versions' arrays only allow 'latest' until a
versioned archive exists, secondUpdate must stay empty until a chained
second-update leg is implemented, and unknown spec keys/routes throw.
Expansion, route filtering, empty-matrix gating, and all seven error
paths verified locally; the dispatch route choice keeps its exact
previous semantics (all/both/update/installer/windows-desktop).
This commit is contained in:
ethernet
2026-08-11 15:21:24 -04:00
parent 77c3a398f2
commit 1bf0a1c95b
2 changed files with 326 additions and 45 deletions
+88 -45
View File
@@ -2,22 +2,28 @@ name: Install & Update E2E
# Can a user on a released version get to this commit?
#
# For each release we sample, a leg installs that release through the real
# `curl | install.sh` one-liner (uv, a managed Python, Node, the venv) inside
# scripts/dev-sandbox.sh, then applies one update route and requires the
# checkout to land on this commit with a working `hermes`.
# The support matrix -- every {os, install-method, update-method} combination
# a user could be on -- lives in scripts/sandbox/generate-e2e-matrix.mjs.
# The generate-matrix job expands it into ONE JOB PER COMBINATION:
#
# The starting versions are chosen at runtime from the repo's release tags
# (scripts/sandbox/pick-release-tags.sh): newest, oldest, and a spread between.
# A hardcoded list would stop covering the newest release the day after it
# ships, and would pin an "oldest" that nobody still runs.
# * linux legs install a sampled release through the real
# `curl | install.sh` one-liner (uv, a managed Python, Node, the venv)
# inside scripts/dev-sandbox.sh, then apply the combo's update route and
# require the checkout to land on this commit with a working `hermes`;
# * windows legs run the real desktop user flow (install-e2e-windows-run
# .yml): the website's Hermes-Setup.exe clicked by AutoHotkey, then the
# update driven through the app itself, Playwright clicking "Update now";
# * combinations the spec declares but CI can't drive yet (macOS, the
# remaining windows install/update methods) surface as skipped jobs so
# the coverage gap is enumerable instead of buried in comments.
#
# A separate Windows axis (install-e2e-windows-run.yml) covers the desktop
# user flow: install OLD with the website's published bootstrap installer
# (headed, AutoHotkey-clicked), then update to HEAD through a real update
# surface — the app's own Update button clicked by Playwright. No tag
# matrix there: the published exe's own build pin is the starting point
# users actually have.
# The linux starting versions are chosen at runtime from the repo's release
# tags (scripts/sandbox/pick-release-tags.sh): newest, oldest, and a spread
# between. A hardcoded list would stop covering the newest release the day
# after it ships, and would pin an "oldest" that nobody still runs. The
# windows starting version is the newest release tag (the published
# installer has no commit pin -- it installs what `main` serves, which the
# harness stages to that tag).
#
# Triggers:
# * every 12 hours, so upstream drift (a new uv, a Node bump, a PyPI change)
@@ -62,7 +68,7 @@ concurrency:
jobs:
# Which released versions do we test updating FROM? Resolved once and shared
# by both route matrices, so the two routes cover the same set.
# by every linux leg, so all routes cover the same set.
pick-releases:
name: Pick release tags
runs-on: ubuntu-latest
@@ -86,44 +92,81 @@ jobs:
echo "Testing updates from: $tags"
echo "tags=$tags" >> "$GITHUB_OUTPUT"
# `hermes update` -- the route most users take.
update:
if: github.event_name != 'workflow_dispatch' || contains(fromJSON('["all", "both", "update"]'), inputs.route)
# Expand the {os, install-method, update-method} support matrix
# (scripts/sandbox/generate-e2e-matrix.mjs) into one job per combination.
# Implemented combos land in the linux/windows matrices below; everything
# the spec declares but CI can't drive yet lands in `skipped`, so the TODO
# surface is visible in every run instead of buried in comments.
generate-matrix:
name: Generate combination matrix
needs: pick-releases
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
linux: ${{ steps.gen.outputs.linux }}
windows: ${{ steps.gen.outputs.windows }}
skipped: ${{ steps.gen.outputs.skipped }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
sparse-checkout: scripts/sandbox/generate-e2e-matrix.mjs
sparse-checkout-cone-mode: false
- id: gen
run: |
set -euo pipefail
matrices="$(node scripts/sandbox/generate-e2e-matrix.mjs \
--tags '${{ needs.pick-releases.outputs.tags }}' \
--route '${{ inputs.route || 'all' }}')"
echo "$matrices"
for key in linux windows skipped; do
echo "$key=$(echo "$matrices" | node -e 'let d="";process.stdin.on("data",c=>d+=c).on("end",()=>console.log(JSON.stringify(JSON.parse(d)[process.argv[1]])))' "$key")" >> "$GITHUB_OUTPUT"
done
# Linux sandbox legs: one job per {install-method -> update-method, release
# tag} combination, each a real curl|bash install in bubblewrap followed by
# the combo's update route.
linux:
name: "linux: ${{ matrix.name }}"
if: fromJSON(needs.generate-matrix.outputs.linux).include[0] != null
needs: generate-matrix
strategy:
# One release breaking is worth knowing about even if another already
# One combo breaking is worth knowing about even if another already
# failed, so let every leg report.
fail-fast: false
matrix:
install-ref: ${{ fromJSON(needs.pick-releases.outputs.tags) }}
max-parallel: 4
matrix: ${{ fromJSON(needs.generate-matrix.outputs.linux) }}
uses: ./.github/workflows/install-e2e-run.yml
with:
route: update
install-ref: ${{ matrix.install-ref }}
route: ${{ matrix.route }}
install-ref: ${{ matrix.install_ref }}
# Re-running the curl one-liner over an existing checkout: autostash + pull
# rather than the updater's own git handling.
installer:
if: github.event_name != 'workflow_dispatch' || contains(fromJSON('["all", "both", "installer"]'), inputs.route)
needs: pick-releases
# Windows GUI legs: one job per combination of the real desktop user flow
# (website installer clicked by AutoHotkey, update driven through the app).
windows:
name: "windows: ${{ matrix.name }}"
if: fromJSON(needs.generate-matrix.outputs.windows).include[0] != null
needs: generate-matrix
strategy:
fail-fast: false
max-parallel: 3
matrix:
install-ref: ${{ fromJSON(needs.pick-releases.outputs.tags) }}
uses: ./.github/workflows/install-e2e-run.yml
with:
route: installer
install-ref: ${{ matrix.install-ref }}
# Windows desktop: install OLD with the website's published bootstrap
# installer (the exact bits a user downloads today, AutoHotkey-clicked),
# then update to HEAD by clicking the app's own Update button under
# Playwright. The reusable workflow's route input selects the update
# mechanism; `hermes update` and installer-rerun arms are declared TODOs
# there.
windows-desktop:
if: github.event_name != 'workflow_dispatch' || contains(fromJSON('["all", "windows-desktop"]'), inputs.route)
matrix: ${{ fromJSON(needs.generate-matrix.outputs.windows) }}
uses: ./.github/workflows/install-e2e-windows-run.yml
with:
route: desktop
route: ${{ matrix.route }}
install-ref: ${{ matrix.install_ref }}
# Declared-but-unimplemented combinations. Each shows up as its own skipped
# check so coverage gaps are enumerable from the Checks tab; implementing
# one moves it from here into IMPLEMENTED in the generator.
skipped:
name: "skipped: ${{ matrix.os }}: ${{ matrix.install }} -> ${{ matrix.update }}"
if: fromJSON(needs.generate-matrix.outputs.skipped).include[0] != null
needs: generate-matrix
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.generate-matrix.outputs.skipped) }}
runs-on: ubuntu-latest
timeout-minutes: 2
steps:
- run: >-
echo "SKIPPED (${{ matrix.reason }}) -- ${{ matrix.os }} install via
'${{ matrix.install }}', update via '${{ matrix.update }}'"
+238
View File
@@ -0,0 +1,238 @@
#!/usr/bin/env node
/**
* Expand the install/update support matrix into concrete E2E combinations.
*
* One source of truth for every {os, install-method, update-method} pair a
* user could be on. Implemented combos fan out into real CI jobs -- one job
* per combination -- and everything else lands in a "skipped" matrix so the
* TODO surface stays visible in every run instead of buried in comments.
*
* Used by .github/workflows/install-e2e.yml:
*
* node scripts/sandbox/generate-e2e-matrix.mjs \
* --tags '["v2026.8.3","v2026.3.12"]' --route all
*
* Prints JSON: { linux: {include:[...]}, windows: {include:[...]},
* skipped: {include:[...]} }. The linux include entries carry
* {name, route, install_ref} for install-e2e-run.yml; windows entries carry
* {name, route, install_ref} for install-e2e-windows-run.yml.
*/
import path from 'node:path';
import { parseArgs } from 'node:util';
import { fileURLToPath } from 'node:url';
/**
* Method ids are strict machine strings -- workflows and the IMPLEMENTED
* table key off them, so an unknown id must fail loudly (see KNOWN_METHODS).
*
* `versions` on an entry expands it into one combination per version. Only
* "latest" is allowed today: it means "the artifact published on the website
* right now" (Hermes-Setup.exe has no versioned archive yet). When archived
* installer versions exist, widen ALLOWED_VERSIONS.
*/
export const SPEC = {
windows: {
install: [
// irm https://hermes.nousresearch.com/install.ps1 | iex
{ method: 'irm-iex' },
// Website Hermes-Setup.exe, clicked through the GUI.
{ method: 'desktop-installer', versions: ['latest'] },
],
update: [
{ method: 'irm-iex' },
// Run the bootstrap exe again over an existing install (--update flow).
{ method: 'desktop-installer-rerun', versions: ['latest'] },
{ method: 'hermes-update' },
// Settings -> About -> "Update now" inside the running desktop app.
{ method: 'desktop-app' },
],
},
macos: {
install: [
{ method: 'curl-bash' },
{ method: 'packaged-app' },
],
update: [
{ method: 'curl-bash' },
{ method: 'hermes-update' },
{ method: 'app-update' },
],
},
linux: {
install: [
{ method: 'curl-bash' },
],
update: [
{ method: 'curl-bash' },
{ method: 'hermes-update' },
],
},
};
const KNOWN_METHODS = new Set([
'irm-iex',
'desktop-installer',
'desktop-installer-rerun',
'hermes-update',
'desktop-app',
'curl-bash',
'packaged-app',
'app-update',
]);
const ALLOWED_VERSIONS = new Set(['latest']);
/**
* The combinations CI actually runs today, and which reusable workflow +
* route input drives each. Everything in SPEC but not here is emitted as a
* skipped combo. Install ids include the expanded @version suffix.
*/
export const IMPLEMENTED = [
// scripts/dev-sandbox.sh legs (install-e2e-run.yml, bubblewrap sandbox).
{ os: 'linux', install: 'curl-bash', update: 'hermes-update', workflow: 'linux', route: 'update' },
{ os: 'linux', install: 'curl-bash', update: 'curl-bash', workflow: 'linux', route: 'installer' },
// Real Windows GUI flow (install-e2e-windows-run.yml): website exe clicked
// by AutoHotkey, Update now clicked by Playwright.
{ os: 'windows', install: 'desktop-installer@latest', update: 'desktop-app', workflow: 'windows', route: 'desktop' },
];
function validateEntry(os, kind, entry) {
if (typeof entry.method !== 'string' || !KNOWN_METHODS.has(entry.method)) {
throw new Error(`${os}.${kind}: unknown method id ${JSON.stringify(entry.method)} -- add it to KNOWN_METHODS if intentional`);
}
if ('versions' in entry) {
if (!Array.isArray(entry.versions) || entry.versions.length === 0) {
throw new Error(`${os}.${kind}.${entry.method}: versions must be a non-empty array`);
}
for (const v of entry.versions) {
if (!ALLOWED_VERSIONS.has(v)) {
throw new Error(`${os}.${kind}.${entry.method}: version ${JSON.stringify(v)} not allowed -- only ${[...ALLOWED_VERSIONS].join(', ')} until versioned installer archives exist`);
}
}
}
const unknown = Object.keys(entry).filter((k) => k !== 'method' && k !== 'versions');
if (unknown.length) {
throw new Error(`${os}.${kind}.${entry.method}: unknown keys ${unknown.join(', ')}`);
}
}
/** Expand one method entry into concrete ids ("desktop-installer@latest"). */
export function expandMethod(os, kind, entry) {
validateEntry(os, kind, entry);
if (!entry.versions) return [entry.method];
return entry.versions.map((v) => `${entry.method}@${v}`);
}
/** Every {os, install, update, secondUpdate} combination in SPEC. */
export function generateEnvironments(spec) {
const envs = [];
for (const [os, osSpec] of Object.entries(spec)) {
const { install, update, secondUpdate = [], ...unknown } = osSpec;
if (Object.keys(unknown).length) {
throw new Error(`${os}: unknown spec keys ${Object.keys(unknown).join(', ')}`);
}
// Chained second updates (install -> update -> update again) are a real
// axis -- the updater that RESULTS from an update must itself update --
// but nothing implements them yet. Refuse a spec that declares them so
// the first implementation is forced to come through here.
if (!Array.isArray(secondUpdate) || secondUpdate.length !== 0) {
throw new Error(`${os}: secondUpdate must be empty until a second-update leg is implemented`);
}
const installs = install.flatMap((e) => expandMethod(os, 'install', e));
const updates = update.flatMap((e) => expandMethod(os, 'update', e));
for (const i of installs) {
for (const u of updates) {
envs.push({ os, install: i, update: u, secondUpdate: '' });
}
}
}
return envs;
}
function findImplementation(env) {
return IMPLEMENTED.find(
(m) => m.os === env.os && m.install === env.install && m.update === env.update,
);
}
/** Mirror of install-e2e.yml's dispatch `route` choice. */
function routeWants(route, env, impl) {
switch (route) {
case 'all':
return true;
case 'both':
return env.os === 'linux' && Boolean(impl);
case 'update':
return impl?.route === 'update';
case 'installer':
return impl?.route === 'installer';
case 'windows-desktop':
return env.os === 'windows' && Boolean(impl);
default:
throw new Error(`unknown route filter: ${JSON.stringify(route)}`);
}
}
/**
* Split the combinations into per-workflow matrices.
* Linux combos fan out further over `tags` (which released version the leg
* installs first); Windows install versions come from the method id instead
* (desktop-installer@latest = whatever the website serves).
*/
export function buildMatrices(envs, { tags = [], route = 'all' } = {}) {
const linux = [];
const windows = [];
const skipped = [];
for (const env of envs) {
const impl = findImplementation(env);
if (!routeWants(route, env, impl)) continue;
if (!impl) {
skipped.push({ ...env, reason: 'not implemented yet' });
} else if (impl.workflow === 'linux') {
if (tags.length === 0) {
throw new Error(`linux combo ${env.install} -> ${env.update} selected but no --tags given`);
}
for (const tag of tags) {
linux.push({
name: `${env.install} @ ${tag} -> ${env.update}`,
route: impl.route,
install_ref: tag,
});
}
} else if (impl.workflow === 'windows') {
windows.push({
name: `${env.install} -> ${env.update}`,
route: impl.route,
install_ref: 'auto',
});
} else {
throw new Error(`unknown workflow ${JSON.stringify(impl.workflow)} for ${env.os}`);
}
}
return {
linux: { include: linux },
windows: { include: windows },
skipped: { include: skipped },
};
}
function main() {
const { values } = parseArgs({
options: {
tags: { type: 'string', default: '[]' },
route: { type: 'string', default: 'all' },
},
});
const tags = JSON.parse(values.tags);
if (!Array.isArray(tags) || !tags.every((t) => typeof t === 'string')) {
throw new Error('--tags must be a JSON array of strings');
}
const envs = generateEnvironments(SPEC);
const matrices = buildMatrices(envs, { tags, route: values.route });
process.stdout.write(`${JSON.stringify(matrices, null, 2)}\n`);
}
if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
main();
}