ci(install-e2e): generate every install/update combination - one job per combo
Replace the hand-enumerated update/installer/windows-desktop jobs with
a support-matrix generator (scripts/sandbox/generate-e2e-matrix.mjs).
The spec declares every {os, install-method, update-method} combination
a user could be on; generate-matrix expands it and fans out ONE JOB PER
COMBINATION:
* linux combos (curl-bash install x hermes-update/curl-bash rerun)
drive install-e2e-run.yml, still multiplied by the sampled release
tags from pick-releases;
* the windows combo (desktop-installer@latest -> desktop-app) drives
install-e2e-windows-run.yml - the real GUI flow;
* every declared-but-unimplemented combo (all of macOS, the remaining
windows methods) becomes its own visible skipped job, so the
coverage gap is enumerable from the Checks tab and implementing one
is a one-line move into IMPLEMENTED.
Strictness carried into the generator: method ids validate against a
closed set, installer 'versions' arrays only allow 'latest' until a
versioned archive exists, secondUpdate must stay empty until a chained
second-update leg is implemented, and unknown spec keys/routes throw.
Expansion, route filtering, empty-matrix gating, and all seven error
paths verified locally; the dispatch route choice keeps its exact
previous semantics (all/both/update/installer/windows-desktop).
This commit is contained in:
@@ -2,22 +2,28 @@ name: Install & Update E2E
|
||||
|
||||
# Can a user on a released version get to this commit?
|
||||
#
|
||||
# For each release we sample, a leg installs that release through the real
|
||||
# `curl | install.sh` one-liner (uv, a managed Python, Node, the venv) inside
|
||||
# scripts/dev-sandbox.sh, then applies one update route and requires the
|
||||
# checkout to land on this commit with a working `hermes`.
|
||||
# The support matrix -- every {os, install-method, update-method} combination
|
||||
# a user could be on -- lives in scripts/sandbox/generate-e2e-matrix.mjs.
|
||||
# The generate-matrix job expands it into ONE JOB PER COMBINATION:
|
||||
#
|
||||
# The starting versions are chosen at runtime from the repo's release tags
|
||||
# (scripts/sandbox/pick-release-tags.sh): newest, oldest, and a spread between.
|
||||
# A hardcoded list would stop covering the newest release the day after it
|
||||
# ships, and would pin an "oldest" that nobody still runs.
|
||||
# * linux legs install a sampled release through the real
|
||||
# `curl | install.sh` one-liner (uv, a managed Python, Node, the venv)
|
||||
# inside scripts/dev-sandbox.sh, then apply the combo's update route and
|
||||
# require the checkout to land on this commit with a working `hermes`;
|
||||
# * windows legs run the real desktop user flow (install-e2e-windows-run
|
||||
# .yml): the website's Hermes-Setup.exe clicked by AutoHotkey, then the
|
||||
# update driven through the app itself, Playwright clicking "Update now";
|
||||
# * combinations the spec declares but CI can't drive yet (macOS, the
|
||||
# remaining windows install/update methods) surface as skipped jobs so
|
||||
# the coverage gap is enumerable instead of buried in comments.
|
||||
#
|
||||
# A separate Windows axis (install-e2e-windows-run.yml) covers the desktop
|
||||
# user flow: install OLD with the website's published bootstrap installer
|
||||
# (headed, AutoHotkey-clicked), then update to HEAD through a real update
|
||||
# surface — the app's own Update button clicked by Playwright. No tag
|
||||
# matrix there: the published exe's own build pin is the starting point
|
||||
# users actually have.
|
||||
# The linux starting versions are chosen at runtime from the repo's release
|
||||
# tags (scripts/sandbox/pick-release-tags.sh): newest, oldest, and a spread
|
||||
# between. A hardcoded list would stop covering the newest release the day
|
||||
# after it ships, and would pin an "oldest" that nobody still runs. The
|
||||
# windows starting version is the newest release tag (the published
|
||||
# installer has no commit pin -- it installs what `main` serves, which the
|
||||
# harness stages to that tag).
|
||||
#
|
||||
# Triggers:
|
||||
# * every 12 hours, so upstream drift (a new uv, a Node bump, a PyPI change)
|
||||
@@ -62,7 +68,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
# Which released versions do we test updating FROM? Resolved once and shared
|
||||
# by both route matrices, so the two routes cover the same set.
|
||||
# by every linux leg, so all routes cover the same set.
|
||||
pick-releases:
|
||||
name: Pick release tags
|
||||
runs-on: ubuntu-latest
|
||||
@@ -86,44 +92,81 @@ jobs:
|
||||
echo "Testing updates from: $tags"
|
||||
echo "tags=$tags" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# `hermes update` -- the route most users take.
|
||||
update:
|
||||
if: github.event_name != 'workflow_dispatch' || contains(fromJSON('["all", "both", "update"]'), inputs.route)
|
||||
# Expand the {os, install-method, update-method} support matrix
|
||||
# (scripts/sandbox/generate-e2e-matrix.mjs) into one job per combination.
|
||||
# Implemented combos land in the linux/windows matrices below; everything
|
||||
# the spec declares but CI can't drive yet lands in `skipped`, so the TODO
|
||||
# surface is visible in every run instead of buried in comments.
|
||||
generate-matrix:
|
||||
name: Generate combination matrix
|
||||
needs: pick-releases
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
linux: ${{ steps.gen.outputs.linux }}
|
||||
windows: ${{ steps.gen.outputs.windows }}
|
||||
skipped: ${{ steps.gen.outputs.skipped }}
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
sparse-checkout: scripts/sandbox/generate-e2e-matrix.mjs
|
||||
sparse-checkout-cone-mode: false
|
||||
- id: gen
|
||||
run: |
|
||||
set -euo pipefail
|
||||
matrices="$(node scripts/sandbox/generate-e2e-matrix.mjs \
|
||||
--tags '${{ needs.pick-releases.outputs.tags }}' \
|
||||
--route '${{ inputs.route || 'all' }}')"
|
||||
echo "$matrices"
|
||||
for key in linux windows skipped; do
|
||||
echo "$key=$(echo "$matrices" | node -e 'let d="";process.stdin.on("data",c=>d+=c).on("end",()=>console.log(JSON.stringify(JSON.parse(d)[process.argv[1]])))' "$key")" >> "$GITHUB_OUTPUT"
|
||||
done
|
||||
|
||||
# Linux sandbox legs: one job per {install-method -> update-method, release
|
||||
# tag} combination, each a real curl|bash install in bubblewrap followed by
|
||||
# the combo's update route.
|
||||
linux:
|
||||
name: "linux: ${{ matrix.name }}"
|
||||
if: fromJSON(needs.generate-matrix.outputs.linux).include[0] != null
|
||||
needs: generate-matrix
|
||||
strategy:
|
||||
# One release breaking is worth knowing about even if another already
|
||||
# One combo breaking is worth knowing about even if another already
|
||||
# failed, so let every leg report.
|
||||
fail-fast: false
|
||||
matrix:
|
||||
install-ref: ${{ fromJSON(needs.pick-releases.outputs.tags) }}
|
||||
max-parallel: 4
|
||||
matrix: ${{ fromJSON(needs.generate-matrix.outputs.linux) }}
|
||||
uses: ./.github/workflows/install-e2e-run.yml
|
||||
with:
|
||||
route: update
|
||||
install-ref: ${{ matrix.install-ref }}
|
||||
route: ${{ matrix.route }}
|
||||
install-ref: ${{ matrix.install_ref }}
|
||||
|
||||
# Re-running the curl one-liner over an existing checkout: autostash + pull
|
||||
# rather than the updater's own git handling.
|
||||
installer:
|
||||
if: github.event_name != 'workflow_dispatch' || contains(fromJSON('["all", "both", "installer"]'), inputs.route)
|
||||
needs: pick-releases
|
||||
# Windows GUI legs: one job per combination of the real desktop user flow
|
||||
# (website installer clicked by AutoHotkey, update driven through the app).
|
||||
windows:
|
||||
name: "windows: ${{ matrix.name }}"
|
||||
if: fromJSON(needs.generate-matrix.outputs.windows).include[0] != null
|
||||
needs: generate-matrix
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 3
|
||||
matrix:
|
||||
install-ref: ${{ fromJSON(needs.pick-releases.outputs.tags) }}
|
||||
uses: ./.github/workflows/install-e2e-run.yml
|
||||
with:
|
||||
route: installer
|
||||
install-ref: ${{ matrix.install-ref }}
|
||||
|
||||
# Windows desktop: install OLD with the website's published bootstrap
|
||||
# installer (the exact bits a user downloads today, AutoHotkey-clicked),
|
||||
# then update to HEAD by clicking the app's own Update button under
|
||||
# Playwright. The reusable workflow's route input selects the update
|
||||
# mechanism; `hermes update` and installer-rerun arms are declared TODOs
|
||||
# there.
|
||||
windows-desktop:
|
||||
if: github.event_name != 'workflow_dispatch' || contains(fromJSON('["all", "windows-desktop"]'), inputs.route)
|
||||
matrix: ${{ fromJSON(needs.generate-matrix.outputs.windows) }}
|
||||
uses: ./.github/workflows/install-e2e-windows-run.yml
|
||||
with:
|
||||
route: desktop
|
||||
route: ${{ matrix.route }}
|
||||
install-ref: ${{ matrix.install_ref }}
|
||||
|
||||
# Declared-but-unimplemented combinations. Each shows up as its own skipped
|
||||
# check so coverage gaps are enumerable from the Checks tab; implementing
|
||||
# one moves it from here into IMPLEMENTED in the generator.
|
||||
skipped:
|
||||
name: "skipped: ${{ matrix.os }}: ${{ matrix.install }} -> ${{ matrix.update }}"
|
||||
if: fromJSON(needs.generate-matrix.outputs.skipped).include[0] != null
|
||||
needs: generate-matrix
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJSON(needs.generate-matrix.outputs.skipped) }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 2
|
||||
steps:
|
||||
- run: >-
|
||||
echo "SKIPPED (${{ matrix.reason }}) -- ${{ matrix.os }} install via
|
||||
'${{ matrix.install }}', update via '${{ matrix.update }}'"
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Expand the install/update support matrix into concrete E2E combinations.
|
||||
*
|
||||
* One source of truth for every {os, install-method, update-method} pair a
|
||||
* user could be on. Implemented combos fan out into real CI jobs -- one job
|
||||
* per combination -- and everything else lands in a "skipped" matrix so the
|
||||
* TODO surface stays visible in every run instead of buried in comments.
|
||||
*
|
||||
* Used by .github/workflows/install-e2e.yml:
|
||||
*
|
||||
* node scripts/sandbox/generate-e2e-matrix.mjs \
|
||||
* --tags '["v2026.8.3","v2026.3.12"]' --route all
|
||||
*
|
||||
* Prints JSON: { linux: {include:[...]}, windows: {include:[...]},
|
||||
* skipped: {include:[...]} }. The linux include entries carry
|
||||
* {name, route, install_ref} for install-e2e-run.yml; windows entries carry
|
||||
* {name, route, install_ref} for install-e2e-windows-run.yml.
|
||||
*/
|
||||
|
||||
import path from 'node:path';
|
||||
import { parseArgs } from 'node:util';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
/**
|
||||
* Method ids are strict machine strings -- workflows and the IMPLEMENTED
|
||||
* table key off them, so an unknown id must fail loudly (see KNOWN_METHODS).
|
||||
*
|
||||
* `versions` on an entry expands it into one combination per version. Only
|
||||
* "latest" is allowed today: it means "the artifact published on the website
|
||||
* right now" (Hermes-Setup.exe has no versioned archive yet). When archived
|
||||
* installer versions exist, widen ALLOWED_VERSIONS.
|
||||
*/
|
||||
export const SPEC = {
|
||||
windows: {
|
||||
install: [
|
||||
// irm https://hermes.nousresearch.com/install.ps1 | iex
|
||||
{ method: 'irm-iex' },
|
||||
// Website Hermes-Setup.exe, clicked through the GUI.
|
||||
{ method: 'desktop-installer', versions: ['latest'] },
|
||||
],
|
||||
update: [
|
||||
{ method: 'irm-iex' },
|
||||
// Run the bootstrap exe again over an existing install (--update flow).
|
||||
{ method: 'desktop-installer-rerun', versions: ['latest'] },
|
||||
{ method: 'hermes-update' },
|
||||
// Settings -> About -> "Update now" inside the running desktop app.
|
||||
{ method: 'desktop-app' },
|
||||
],
|
||||
},
|
||||
macos: {
|
||||
install: [
|
||||
{ method: 'curl-bash' },
|
||||
{ method: 'packaged-app' },
|
||||
],
|
||||
update: [
|
||||
{ method: 'curl-bash' },
|
||||
{ method: 'hermes-update' },
|
||||
{ method: 'app-update' },
|
||||
],
|
||||
},
|
||||
linux: {
|
||||
install: [
|
||||
{ method: 'curl-bash' },
|
||||
],
|
||||
update: [
|
||||
{ method: 'curl-bash' },
|
||||
{ method: 'hermes-update' },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
const KNOWN_METHODS = new Set([
|
||||
'irm-iex',
|
||||
'desktop-installer',
|
||||
'desktop-installer-rerun',
|
||||
'hermes-update',
|
||||
'desktop-app',
|
||||
'curl-bash',
|
||||
'packaged-app',
|
||||
'app-update',
|
||||
]);
|
||||
|
||||
const ALLOWED_VERSIONS = new Set(['latest']);
|
||||
|
||||
/**
|
||||
* The combinations CI actually runs today, and which reusable workflow +
|
||||
* route input drives each. Everything in SPEC but not here is emitted as a
|
||||
* skipped combo. Install ids include the expanded @version suffix.
|
||||
*/
|
||||
export const IMPLEMENTED = [
|
||||
// scripts/dev-sandbox.sh legs (install-e2e-run.yml, bubblewrap sandbox).
|
||||
{ os: 'linux', install: 'curl-bash', update: 'hermes-update', workflow: 'linux', route: 'update' },
|
||||
{ os: 'linux', install: 'curl-bash', update: 'curl-bash', workflow: 'linux', route: 'installer' },
|
||||
// Real Windows GUI flow (install-e2e-windows-run.yml): website exe clicked
|
||||
// by AutoHotkey, Update now clicked by Playwright.
|
||||
{ os: 'windows', install: 'desktop-installer@latest', update: 'desktop-app', workflow: 'windows', route: 'desktop' },
|
||||
];
|
||||
|
||||
function validateEntry(os, kind, entry) {
|
||||
if (typeof entry.method !== 'string' || !KNOWN_METHODS.has(entry.method)) {
|
||||
throw new Error(`${os}.${kind}: unknown method id ${JSON.stringify(entry.method)} -- add it to KNOWN_METHODS if intentional`);
|
||||
}
|
||||
if ('versions' in entry) {
|
||||
if (!Array.isArray(entry.versions) || entry.versions.length === 0) {
|
||||
throw new Error(`${os}.${kind}.${entry.method}: versions must be a non-empty array`);
|
||||
}
|
||||
for (const v of entry.versions) {
|
||||
if (!ALLOWED_VERSIONS.has(v)) {
|
||||
throw new Error(`${os}.${kind}.${entry.method}: version ${JSON.stringify(v)} not allowed -- only ${[...ALLOWED_VERSIONS].join(', ')} until versioned installer archives exist`);
|
||||
}
|
||||
}
|
||||
}
|
||||
const unknown = Object.keys(entry).filter((k) => k !== 'method' && k !== 'versions');
|
||||
if (unknown.length) {
|
||||
throw new Error(`${os}.${kind}.${entry.method}: unknown keys ${unknown.join(', ')}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Expand one method entry into concrete ids ("desktop-installer@latest"). */
|
||||
export function expandMethod(os, kind, entry) {
|
||||
validateEntry(os, kind, entry);
|
||||
if (!entry.versions) return [entry.method];
|
||||
return entry.versions.map((v) => `${entry.method}@${v}`);
|
||||
}
|
||||
|
||||
/** Every {os, install, update, secondUpdate} combination in SPEC. */
|
||||
export function generateEnvironments(spec) {
|
||||
const envs = [];
|
||||
for (const [os, osSpec] of Object.entries(spec)) {
|
||||
const { install, update, secondUpdate = [], ...unknown } = osSpec;
|
||||
if (Object.keys(unknown).length) {
|
||||
throw new Error(`${os}: unknown spec keys ${Object.keys(unknown).join(', ')}`);
|
||||
}
|
||||
// Chained second updates (install -> update -> update again) are a real
|
||||
// axis -- the updater that RESULTS from an update must itself update --
|
||||
// but nothing implements them yet. Refuse a spec that declares them so
|
||||
// the first implementation is forced to come through here.
|
||||
if (!Array.isArray(secondUpdate) || secondUpdate.length !== 0) {
|
||||
throw new Error(`${os}: secondUpdate must be empty until a second-update leg is implemented`);
|
||||
}
|
||||
const installs = install.flatMap((e) => expandMethod(os, 'install', e));
|
||||
const updates = update.flatMap((e) => expandMethod(os, 'update', e));
|
||||
for (const i of installs) {
|
||||
for (const u of updates) {
|
||||
envs.push({ os, install: i, update: u, secondUpdate: '' });
|
||||
}
|
||||
}
|
||||
}
|
||||
return envs;
|
||||
}
|
||||
|
||||
function findImplementation(env) {
|
||||
return IMPLEMENTED.find(
|
||||
(m) => m.os === env.os && m.install === env.install && m.update === env.update,
|
||||
);
|
||||
}
|
||||
|
||||
/** Mirror of install-e2e.yml's dispatch `route` choice. */
|
||||
function routeWants(route, env, impl) {
|
||||
switch (route) {
|
||||
case 'all':
|
||||
return true;
|
||||
case 'both':
|
||||
return env.os === 'linux' && Boolean(impl);
|
||||
case 'update':
|
||||
return impl?.route === 'update';
|
||||
case 'installer':
|
||||
return impl?.route === 'installer';
|
||||
case 'windows-desktop':
|
||||
return env.os === 'windows' && Boolean(impl);
|
||||
default:
|
||||
throw new Error(`unknown route filter: ${JSON.stringify(route)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Split the combinations into per-workflow matrices.
|
||||
* Linux combos fan out further over `tags` (which released version the leg
|
||||
* installs first); Windows install versions come from the method id instead
|
||||
* (desktop-installer@latest = whatever the website serves).
|
||||
*/
|
||||
export function buildMatrices(envs, { tags = [], route = 'all' } = {}) {
|
||||
const linux = [];
|
||||
const windows = [];
|
||||
const skipped = [];
|
||||
for (const env of envs) {
|
||||
const impl = findImplementation(env);
|
||||
if (!routeWants(route, env, impl)) continue;
|
||||
if (!impl) {
|
||||
skipped.push({ ...env, reason: 'not implemented yet' });
|
||||
} else if (impl.workflow === 'linux') {
|
||||
if (tags.length === 0) {
|
||||
throw new Error(`linux combo ${env.install} -> ${env.update} selected but no --tags given`);
|
||||
}
|
||||
for (const tag of tags) {
|
||||
linux.push({
|
||||
name: `${env.install} @ ${tag} -> ${env.update}`,
|
||||
route: impl.route,
|
||||
install_ref: tag,
|
||||
});
|
||||
}
|
||||
} else if (impl.workflow === 'windows') {
|
||||
windows.push({
|
||||
name: `${env.install} -> ${env.update}`,
|
||||
route: impl.route,
|
||||
install_ref: 'auto',
|
||||
});
|
||||
} else {
|
||||
throw new Error(`unknown workflow ${JSON.stringify(impl.workflow)} for ${env.os}`);
|
||||
}
|
||||
}
|
||||
return {
|
||||
linux: { include: linux },
|
||||
windows: { include: windows },
|
||||
skipped: { include: skipped },
|
||||
};
|
||||
}
|
||||
|
||||
function main() {
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
tags: { type: 'string', default: '[]' },
|
||||
route: { type: 'string', default: 'all' },
|
||||
},
|
||||
});
|
||||
const tags = JSON.parse(values.tags);
|
||||
if (!Array.isArray(tags) || !tags.every((t) => typeof t === 'string')) {
|
||||
throw new Error('--tags must be a JSON array of strings');
|
||||
}
|
||||
const envs = generateEnvironments(SPEC);
|
||||
const matrices = buildMatrices(envs, { tags, route: values.route });
|
||||
process.stdout.write(`${JSON.stringify(matrices, null, 2)}\n`);
|
||||
}
|
||||
|
||||
if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
|
||||
main();
|
||||
}
|
||||
Reference in New Issue
Block a user