test(install-e2e): macos desktop-installer arm - the published dmg, driven for real

macos gains the desktop-installer@latest install method: the website's
Hermes-Setup.dmg (verified live), mounted with hdiutil and its app
binary run DIRECTLY - an open-launched app inherits none of the git
redirect env, so direct exec is what keeps the isolation honest while
staying the same binary and first-launch flow.

install-e2e-macos-run.yml takes the windows shape: one workflow, one
inner job per driver arm, native skips. Arm 1 delegates script installs
to the shared OS-agnostic run workflow; arm 2 stages, installs from the
dmg, and drives both app-update methods through launch-from-spec.mjs -
open-app-update launches the installed .app (the double-click surface,
env via Playwright), hermes-desktop-app-update captures the product's
own hermes desktop spawn. Both end on sha asserts, never version
strings.
This commit is contained in:
ethernet
2026-08-12 04:36:03 -04:00
parent adf7d55f4b
commit 2b39b885d6
5 changed files with 434 additions and 6 deletions
+149
View File
@@ -0,0 +1,149 @@
# Reusable runner for ONE macOS install/update combination.
#
# Two driver arms, one runs per dispatch (the other natively skips):
#
# e2e (script arms) tests/install/installer-script-e2e.sh - the
# OS-agnostic git-redirect driver shared with
# linux. installer-script(+desktop) installs,
# script/updater/hermes-desktop-app-update
# updates.
# gui-e2e (desktop arm) tests/install/macos-desktop-e2e.sh - the
# published Hermes-Setup.dmg, mounted and run,
# then the app driven by Playwright for the
# app-update methods.
#
# Method pairs without a driver arm yet NATIVELY SKIP (grey check, no
# runner): the capability knowledge lives here, next to the drivers.
name: install-e2e macos leg
on:
workflow_call:
inputs:
install-method:
description: 'How OLD gets installed. Supported: installer-script, installer-script+desktop (curl | bash one-liner, optionally with --include-desktop) and desktop-installer@latest (the published Hermes-Setup.dmg).'
required: true
type: string
update-method:
description: 'How the install updates to HEAD. Script installs support hermes-update / installer-script / installer-script+desktop / hermes-desktop-app-update; dmg installs support open-app-update / hermes-desktop-app-update.'
required: true
type: string
install-ref:
description: 'What to install before updating: a branch, a tag, or a SHA reachable from main.'
required: false
type: string
default: refs/heads/main
tag-has-desktop:
description: "Whether install-ref ships the desktop app (apps/desktop). Desktop-method legs from pre-desktop releases natively skip."
required: false
type: boolean
default: true
dmg-url:
description: 'Bootstrap dmg to install OLD with. Default: the latest published one — what a user downloads today.'
required: false
type: string
default: https://hermes-assets.nousresearch.com/Hermes-Setup.dmg
timeout-minutes:
description: 'Job timeout. App-update legs do a full Electron build.'
required: false
type: number
default: 120
permissions:
contents: read
jobs:
# ---- arm 1: script installs (the shared OS-agnostic driver) --------------
e2e:
name: install & update
if: >-
(inputs.install-method == 'installer-script'
|| (inputs.install-method == 'installer-script+desktop' && inputs.tag-has-desktop))
&& (contains(fromJSON('["hermes-update", "installer-script"]'), inputs.update-method)
|| (contains(fromJSON('["installer-script+desktop", "hermes-desktop-app-update"]'), inputs.update-method) && inputs.tag-has-desktop))
uses: ./.github/workflows/install-e2e-run.yml
with:
install-method: ${{ inputs.install-method }}
update-method: ${{ inputs.update-method }}
install-ref: ${{ inputs.install-ref }}
tag-has-desktop: ${{ inputs.tag-has-desktop }}
runner: macos-latest
timeout-minutes: ${{ inputs.timeout-minutes }}
# ---- arm 2: the published dmg, then Playwright drives the app ------------
gui-e2e:
# Short static name on purpose: name expressions render UNEXPANDED on
# skipped jobs.
name: Hermes-Setup.dmg
if: >-
inputs.install-method == 'desktop-installer@latest' && inputs.tag-has-desktop
&& contains(fromJSON('["open-app-update", "hermes-desktop-app-update"]'), inputs.update-method)
runs-on: macos-latest
timeout-minutes: ${{ inputs.timeout-minutes }}
steps:
# Full history: the driver bare-clones this checkout as the repo the
# installer/updater talk to.
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
fetch-depth: 0
- name: Start screen recording
uses: ./.github/actions/e2e-screen-record
with:
mode: start
output: ${{ runner.temp }}/e2e-logs/recording.mkv
- name: Stage serve repo (main -> ${{ inputs.install-ref }})
run: |
set -euo pipefail
tests/install/macos-desktop-e2e.sh --phase stage \
--update-method '${{ inputs.update-method }}' \
--install-ref '${{ inputs.install-ref }}' \
--dmg-url '${{ inputs.dmg-url }}'
env:
HERMES_E2E_LOG_DIR: ${{ runner.temp }}/e2e-logs
- name: Install ${{ inputs.install-ref }} via Hermes-Setup.dmg
run: |
set -euo pipefail
tests/install/macos-desktop-e2e.sh --phase install \
--update-method '${{ inputs.update-method }}' \
--install-ref '${{ inputs.install-ref }}' \
--dmg-url '${{ inputs.dmg-url }}'
env:
HERMES_E2E_LOG_DIR: ${{ runner.temp }}/e2e-logs
- name: Update ${{ inputs.install-ref }} -> HEAD (${{ inputs.update-method }})
run: |
set -euo pipefail
tests/install/macos-desktop-e2e.sh --phase update \
--update-method '${{ inputs.update-method }}' \
--install-ref '${{ inputs.install-ref }}' \
--dmg-url '${{ inputs.dmg-url }}'
env:
HERMES_E2E_LOG_DIR: ${{ runner.temp }}/e2e-logs
- name: Stop screen recording
if: always()
uses: ./.github/actions/e2e-screen-record
with:
mode: stop
output: ${{ runner.temp }}/e2e-logs/recording.mkv
# Artifact names cannot contain '/'; install-ref may be a full ref.
- name: Build artifact name
id: artifact
if: always()
run: |
safe_ref="$(printf '%s' '${{ inputs.install-ref }}' | tr '/:' '--')"
echo "name=install-e2e-macos-dmg-${{ inputs.update-method }}-${safe_ref}-${{ github.sha }}" >> "$GITHUB_OUTPUT"
- name: Upload logs
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ${{ steps.artifact.outputs.name }}
path: ${{ runner.temp }}/e2e-logs
retention-days: 14
if-no-files-found: ignore
+7 -5
View File
@@ -13,8 +13,10 @@ name: Install & Update E2E
# Matrix: windows the real desktop user flow: website Hermes-Setup.exe
# clicked by AutoHotkey, update via the app, Playwright
# clicking "Update now" (install-e2e-windows-run.yml)
# Matrix: macos the same OS-agnostic driver as linux, on macos-latest
# (install-e2e-run.yml; app-update variants are TODO)
# Matrix: macos script installs on the shared OS-agnostic driver,
# plus the real desktop user flow: website
# Hermes-Setup.dmg mounted and run, updates via the
# app under Playwright (install-e2e-macos-run.yml)
#
# Every combination is dispatched to its OS's run workflow; the run
# workflow natively skips (grey) what its driver cannot run yet -- an
@@ -186,14 +188,14 @@ jobs:
# 10x-cost runners: keep concurrency low.
max-parallel: 2
matrix: ${{ fromJSON(needs.generate-matrix.outputs.macos) }}
# The same OS-agnostic driver as linux -- only the runner differs.
uses: ./.github/workflows/install-e2e-run.yml
# Two driver arms: the OS-agnostic script driver (shared with linux)
# and the published-dmg GUI driver; the run workflow routes.
uses: ./.github/workflows/install-e2e-macos-run.yml
with:
install-method: ${{ matrix.install_method }}
update-method: ${{ matrix.update_method }}
install-ref: ${{ matrix.install_ref }}
tag-has-desktop: ${{ matrix.tag_has_desktop }}
runner: macos-latest
# The outcome, human-readable: the plan chart again, with each cell
# replaced by how that leg actually concluded. Per-leg conclusions are
+2
View File
@@ -106,6 +106,8 @@ export const SPEC = {
install: [
{ method: 'installer-script' },
{ method: 'installer-script+desktop' },
// The published Hermes-Setup.dmg from the website, mounted and run.
{ method: 'desktop-installer', versions: ['latest'] },
],
update: [
{ method: 'installer-script' },
+1 -1
View File
@@ -47,7 +47,7 @@ A leg can install a release from months back. The driver must not assume that th
- `installer-script`: the platform's one-liner (`curl | bash` on linux and macos, `irm | iex` on windows).
- `installer-script+desktop`: the same one-liner with its desktop stage opted in (`--include-desktop` / `-IncludeDesktop`). The stage builds the desktop app during the install. On windows it also registers Start Menu and Desktop shortcuts. On linux and macos it builds the app inside the checkout and registers no OS entry point.
- `desktop-installer@latest`: the published GUI installer (`Hermes-Setup.exe` on windows), clicked through the real window.
- `desktop-installer@latest`: the published GUI installer (`Hermes-Setup.exe` on windows, `Hermes-Setup.dmg` on macos), driven through the real user flow.
## The two app-update variants
+275
View File
@@ -0,0 +1,275 @@
#!/usr/bin/env bash
# Prove a macOS user who installed OLD via the published desktop installer
# (Hermes-Setup.dmg from the website) can reach HEAD.
#
# The macOS sibling of tests/install/windows-e2e.ps1's desktop-installer
# arm, sharing the staging trick: every git process is pointed at a local
# bare clone via url.<file://serve.git>.insteadOf in a driver-owned
# GIT_CONFIG_GLOBAL. The published dmg carries no commit pin - it installs
# whatever `main` serves - so parking serve.git's main at OLD stages the
# "user on the current release" start, and advancing it to HEAD makes an
# update available exactly the way it does for a real user.
#
# Phases (state shared via the workroot, mirroring the windows driver):
# stage bare-clone this checkout to serve.git, park main at OLD
# install download the dmg, hdiutil attach, run the installer app's
# binary DIRECTLY (env inheritance: an `open`-launched app sees
# none of our redirect env), wait for the install to land
# update advance served main to HEAD, apply ONE update method:
# open-app-update launch the installed app binary
# under Playwright, click Update now
# hermes-desktop-app-update capture `hermes desktop`'s spawn,
# launch the spec under Playwright,
# click Update now
#
# Usage:
# tests/install/macos-desktop-e2e.sh --phase stage|install|update|all
# --update-method open-app-update|hermes-desktop-app-update
# [--install-ref REF] [--dmg-url URL]
#
# Requires a clean full-history checkout with release tags fetched, on a
# macOS host with a window server (the GitHub macos runners qualify).
set -euo pipefail
PHASE="all"
UPDATE_METHOD=""
INSTALL_REF=""
DMG_URL="https://hermes-assets.nousresearch.com/Hermes-Setup.dmg"
PLAYWRIGHT_VERSION="1.58.2"
while [ "$#" -gt 0 ]; do
case "$1" in
--phase)
[ "$#" -ge 2 ] || { echo 'error: --phase needs a value' >&2; exit 1; }
PHASE="$2"; shift 2 ;;
--update-method)
[ "$#" -ge 2 ] || { echo 'error: --update-method needs a value' >&2; exit 1; }
UPDATE_METHOD="$2"; shift 2 ;;
--install-ref)
[ "$#" -ge 2 ] || { echo 'error: --install-ref needs a value' >&2; exit 1; }
INSTALL_REF="$2"; shift 2 ;;
--dmg-url)
[ "$#" -ge 2 ] || { echo 'error: --dmg-url needs a value' >&2; exit 1; }
DMG_URL="$2"; shift 2 ;;
-h|--help) sed -n '2,32p' "$0"; exit 0 ;;
*) echo "error: unknown argument: $1" >&2; exit 1 ;;
esac
done
case "$UPDATE_METHOD" in
open-app-update|hermes-desktop-app-update) ;;
*) echo "error: --update-method must be open-app-update or hermes-desktop-app-update, got '$UPDATE_METHOD'" >&2; exit 1 ;;
esac
[ "$(uname -s)" = "Darwin" ] || { echo "error: this driver runs on macOS only" >&2; exit 1; }
REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
REPO_URL_SSH="git@github.com:NousResearch/hermes-agent.git"
REPO_URL_HTTPS="https://github.com/NousResearch/hermes-agent.git"
ASSETS="$REPO_ROOT/tests/install/e2e-assets"
WORK_ROOT="${HERMES_E2E_WORKROOT:-${RUNNER_TEMP:-${TMPDIR:-/tmp}}/hermes-macos-desktop-e2e}"
LOG_DIR="${HERMES_E2E_LOG_DIR:-$WORK_ROOT/logs}"
SERVE_REPO="$WORK_ROOT/serve.git"
STATE="$WORK_ROOT/shas.env"
export HOME_SANDBOX="$WORK_ROOT/home"
step() { printf '\n=== %s ===\n' "$*"; }
ok() { printf ' OK %s\n' "$*"; }
fail() { printf 'E2E ASSERTION FAILED: %s\n' "$*" >&2; exit 1; }
log_group() {
printf '::group::%s\n' "$1"
cat "$2"
printf '::endgroup::\n'
}
# Every phase runs in its own process (separate CI steps), so the redirect
# env is re-established here, not inherited.
arm_redirect() {
GIT_CFG="$WORK_ROOT/gitconfig"
export GIT_CONFIG_GLOBAL="$GIT_CFG"
export HOME="$HOME_SANDBOX"
export PATH="$HOME/.local/bin:$PATH"
export HERMES_HOME="$HOME/.hermes"
export INSTALL_DIR="$HERMES_HOME/hermes-agent"
}
phase_stage() {
step "staging serve.git (main -> OLD)"
[ -z "$(git -C "$REPO_ROOT" status --porcelain -uno)" ] \
|| fail "checkout has uncommitted tracked changes; the staged clone must be a reviewable commit"
rm -rf "$WORK_ROOT"
mkdir -p "$WORK_ROOT" "$LOG_DIR" "$HOME_SANDBOX/.local/bin"
local old_ref="$INSTALL_REF"
if [ -z "$old_ref" ] || [ "$old_ref" = "auto" ]; then
old_ref="$(git -C "$REPO_ROOT" tag --list 'v[0-9]*' --sort=-creatordate | head -1)"
[ -n "$old_ref" ] || fail "no release tags in the checkout to use as OLD"
fi
local old_sha head_sha
old_sha="$(git -C "$REPO_ROOT" rev-parse "${old_ref}^{commit}")"
head_sha="$(git -C "$REPO_ROOT" rev-parse HEAD)"
[ "$old_sha" != "$head_sha" ] || fail "OLD ($old_ref) IS HEAD; no update would be available"
git clone --bare --quiet "$REPO_ROOT" "$SERVE_REPO"
git -C "$SERVE_REPO" update-ref refs/heads/main "$old_sha"
git -C "$SERVE_REPO" symbolic-ref HEAD refs/heads/main
git -C "$SERVE_REPO" config uploadpack.allowAnySHA1InWant true
cat > "$WORK_ROOT/gitconfig" <<EOF
[url "file://$SERVE_REPO"]
insteadOf = $REPO_URL_HTTPS
insteadOf = $REPO_URL_SSH
EOF
arm_redirect
mkdir -p "$HERMES_HOME"
touch "$HERMES_HOME/.skip_upstream_prompt"
printf 'OLD_SHA=%s\nOLD_REF=%s\nHEAD_SHA=%s\n' "$old_sha" "$old_ref" "$head_sha" > "$STATE"
ok "serve.git main = $old_sha ($old_ref), update target $head_sha"
}
find_installed_app() {
# The bootstrap installs the packaged app; look where the product puts it
# (the checkout's release dir), plus /Applications for a copied bundle.
local cand
for cand in \
"$INSTALL_DIR/apps/desktop/release/mac-arm64/Hermes.app" \
"$INSTALL_DIR/apps/desktop/release/mac/Hermes.app" \
"/Applications/Hermes.app"; do
[ -d "$cand" ] && { printf '%s' "$cand"; return 0; }
done
return 1
}
phase_install() {
# shellcheck disable=SC1090
. "$STATE"
arm_redirect
step "installing OLD ($OLD_REF) via the published Hermes-Setup.dmg"
local dmg="$WORK_ROOT/Hermes-Setup.dmg"
[ -f "$dmg" ] || curl -fsSL -o "$dmg" "$DMG_URL"
[ "$(stat -f%z "$dmg")" -gt 1000000 ] || fail "dmg download too small: $(stat -f%z "$dmg") bytes"
# curl'd files carry no quarantine attr, but belt and braces on a runner.
xattr -dr com.apple.quarantine "$dmg" 2>/dev/null || true
local mount
mount="$(hdiutil attach -nobrowse -readonly "$dmg" | awk -F'\t' '/\/Volumes\//{print $NF; exit}')"
[ -n "$mount" ] || fail "hdiutil attach produced no mount point"
ok "dmg mounted at $mount"
local app_bin=""
local app
app="$(find "$mount" -maxdepth 1 -name '*.app' | head -1)"
[ -n "$app" ] || { hdiutil detach "$mount" >/dev/null 2>&1 || true; fail "no .app inside the dmg"; }
app_bin="$(find "$app/Contents/MacOS" -type f -perm +111 | head -1)"
[ -n "$app_bin" ] || fail "no executable inside $app/Contents/MacOS"
# Run the installer binary DIRECTLY: `open` launches via launchd, which
# inherits NONE of the redirect env (GIT_CONFIG_GLOBAL, HOME) - the whole
# isolation would silently evaporate. Direct exec is the same binary and
# the same first-launch flow.
local rc=0
"$app_bin" > "$LOG_DIR/bootstrap-install.log" 2>&1 || rc=$?
log_group "Hermes-Setup (dmg bootstrap) transcript" "$LOG_DIR/bootstrap-install.log"
hdiutil detach "$mount" >/dev/null 2>&1 || true
[ "$rc" -eq 0 ] || fail "dmg bootstrap exited $rc; transcript above"
[ -d "$INSTALL_DIR/.git" ] || fail "no checkout landed at $INSTALL_DIR"
local got
got="$(git -C "$INSTALL_DIR" rev-parse HEAD)"
[ "$got" = "$OLD_SHA" ] || fail "installed checkout is $got, expected OLD ($OLD_SHA)"
ok "checkout is OLD ($OLD_SHA)"
local hermes="$INSTALL_DIR/venv/bin/hermes"
[ -x "$hermes" ] || fail "no hermes console script at $hermes"
"$hermes" --version > "$LOG_DIR/version-old.log" 2>&1 || fail "hermes --version failed after install"
ok "hermes --version works: $(head -c 120 "$LOG_DIR/version-old.log" | tr -d '\n')"
find_installed_app >/dev/null || fail "no installed Hermes.app after the dmg bootstrap"
ok "installed app: $(find_installed_app)"
}
run_playwright_update() {
# $1: spec file to launch from. Installs the driver's OWN pinned
# @playwright/test into a scratch dir (never the installed tree's copy).
local spec="$1"
local pw_dir="$WORK_ROOT/playwright"
mkdir -p "$pw_dir"
(cd "$pw_dir" && npm install --no-save --no-audit --no-fund \
"@playwright/test@$PLAYWRIGHT_VERSION" > "$LOG_DIR/playwright-install.log" 2>&1) \
|| { log_group "playwright install transcript" "$LOG_DIR/playwright-install.log"; fail "playwright install failed"; }
cp "$ASSETS/launch-from-spec.mjs" "$pw_dir/"
local rc=0
(cd "$pw_dir" && node launch-from-spec.mjs \
--spec "$spec" \
--result "$HERMES_HOME/.hermes-update-result.json" \
--expect-sha "$HEAD_SHA" \
--repo-dir "$INSTALL_DIR" \
> "$LOG_DIR/app-update.log" 2>&1) || rc=$?
log_group "app update (Playwright) transcript" "$LOG_DIR/app-update.log"
[ "$rc" -eq 0 ] || fail "app-driven update exited $rc; transcript above"
}
phase_update() {
# shellcheck disable=SC1090
. "$STATE"
arm_redirect
step "advancing served main to HEAD"
git -C "$SERVE_REPO" update-ref refs/heads/main "$HEAD_SHA"
ok "serve.git main = $HEAD_SHA"
step "updating via $UPDATE_METHOD"
case "$UPDATE_METHOD" in
open-app-update)
# The installed app IS the user surface here (double-click the .app);
# hand-build the spec Playwright launches from. Env: the redirect set,
# which is exactly what the app's children (git, hermes update) need.
local app app_bin
app="$(find_installed_app)" || fail "no installed app to launch"
app_bin="$(find "$app/Contents/MacOS" -type f -perm +111 | head -1)"
python3 - "$app_bin" "$WORK_ROOT/launch-spec.json" <<'PYEOF'
import json, os, sys
spec = {
"argv": [sys.argv[1]],
"cwd": os.path.dirname(sys.argv[1]),
"env": dict(os.environ),
"matchedShape": "packaged",
}
with open(sys.argv[2], "w") as fh:
json.dump(spec, fh, indent=2)
PYEOF
run_playwright_update "$WORK_ROOT/launch-spec.json"
;;
hermes-desktop-app-update)
# The product's own launch, captured at its spawn site.
local hermes="$INSTALL_DIR/venv/bin/hermes"
local spec="$WORK_ROOT/launch-spec.json"
local rc=0
(cd "$INSTALL_DIR" && \
PYTHONPATH="$ASSETS/launch-capture${PYTHONPATH:+:$PYTHONPATH}" \
HERMES_E2E_CAPTURE_LAUNCH="$spec" \
"$hermes" desktop < /dev/null > "$LOG_DIR/desktop-launch-capture.log" 2>&1) || rc=$?
log_group "hermes desktop (launch capture) transcript" "$LOG_DIR/desktop-launch-capture.log"
[ "$rc" -eq 0 ] || fail "hermes desktop exited $rc during launch capture"
[ -f "$spec.captured" ] || fail "hermes desktop exited 0 but no launch was captured"
ok "captured $(cat "$spec.captured") launch spec"
run_playwright_update "$spec"
;;
esac
local got
got="$(git -C "$INSTALL_DIR" rev-parse HEAD)"
[ "$got" = "$HEAD_SHA" ] || fail "checkout is $got, expected HEAD ($HEAD_SHA)"
ok "checkout landed on HEAD ($HEAD_SHA)"
"$INSTALL_DIR/venv/bin/hermes" --version > "$LOG_DIR/version-head.log" 2>&1 \
|| fail "hermes --version failed after update"
ok "hermes --version works post-update"
step "PASS: $OLD_REF -> HEAD via $UPDATE_METHOD"
}
case "$PHASE" in
stage) phase_stage ;;
install) phase_install ;;
update) phase_update ;;
all) phase_stage; phase_install; phase_update ;;
*) echo "error: --phase must be stage, install, update or all" >&2; exit 1 ;;
esac