feat(install-e2e): classify exact known failures with report footnotes
Keep unknown failures red, rotate evidence per attempt, and emit receipts for signature-confirmed historical cases. Add CI-only diagnostics and an exact-tag input for the unresolved July hand-off.
This commit is contained in:
ethernet
@@ -137,9 +137,19 @@ jobs:
|
||||
run: powershell -NoProfile -ExecutionPolicy Bypass -File tests\install\windows-e2e.ps1 -Phase install -InstallMethod "${{ inputs.install-method }}" -Route "${{ inputs.update-method }}" -InstallRef "${{ inputs.install-ref }}" -SetupExeUrl ${{ inputs.setup-exe-url }}
|
||||
|
||||
- name: Update ${{ inputs.install-ref }} -> HEAD (${{ inputs.update-method }})
|
||||
id: update
|
||||
shell: powershell
|
||||
run: powershell -NoProfile -ExecutionPolicy Bypass -File tests\install\windows-e2e.ps1 -Phase update -InstallMethod "${{ inputs.install-method }}" -Route "${{ inputs.update-method }}" -InstallRef "${{ inputs.install-ref }}" -SetupExeUrl ${{ inputs.setup-exe-url }}
|
||||
|
||||
- name: Upload known-failure receipt
|
||||
if: steps.update.outputs.known_failure != ''
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: install-e2e-known-${{ steps.update.outputs.known_failure }}--${{ inputs.leg-id }}
|
||||
path: ${{ env.HERMES_E2E_WORKROOT }}\known-failure.json
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
- name: Stop screen recording
|
||||
if: always()
|
||||
uses: ./.github/actions/e2e-screen-record
|
||||
|
||||
@@ -57,6 +57,11 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
default: '3'
|
||||
install-ref:
|
||||
description: 'Optional exact release tag for a focused reproduction; overrides tag-count.'
|
||||
required: false
|
||||
type: string
|
||||
default: ''
|
||||
schedule:
|
||||
# Every 12 hours, off the hour to avoid the top-of-hour runner crunch.
|
||||
- cron: '20 7,19 * * *'
|
||||
@@ -102,10 +107,17 @@ jobs:
|
||||
# applies to each per-OS matrix separately; at 10 tags the largest
|
||||
# is windows at 180 (first over the cap at 15 tags = 270).
|
||||
TAG_COUNT: ${{ inputs.tag-count || 2 }}
|
||||
INSTALL_REF: ${{ inputs.install-ref }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "$TAG_COUNT" =~ ^(10|[1-9])$ ]] || { echo "tag-count must be 1-10, got: $TAG_COUNT" >&2; exit 1; }
|
||||
tags="$(scripts/sandbox/pick-release-tags.sh --count "$TAG_COUNT")"
|
||||
if [ -n "$INSTALL_REF" ]; then
|
||||
[[ "$INSTALL_REF" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?$ ]] || { echo 'install-ref must be an exact release tag' >&2; exit 1; }
|
||||
git rev-parse --verify "refs/tags/$INSTALL_REF^{commit}" >/dev/null
|
||||
tags="$(jq -cn --arg ref "$INSTALL_REF" '[$ref]')"
|
||||
else
|
||||
tags="$(scripts/sandbox/pick-release-tags.sh --count "$TAG_COUNT")"
|
||||
fi
|
||||
echo "Testing updates from: $tags"
|
||||
# Annotate each tag with what its own tree supports, so run
|
||||
# workflows can natively skip surfaces the starting version does
|
||||
@@ -241,7 +253,9 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
sparse-checkout: scripts/sandbox/generate-e2e-matrix.mjs
|
||||
sparse-checkout: |
|
||||
scripts/sandbox/generate-e2e-matrix.mjs
|
||||
tests/install/e2e-assets/known-failures.json
|
||||
sparse-checkout-cone-mode: false
|
||||
- env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
@@ -294,6 +294,9 @@ export function renderMarkdownPlan(envs, tags) {
|
||||
* @returns {string}
|
||||
*/
|
||||
export function renderMarkdownResults(jobs, tagAnnotations = [], artifactById = new Map()) {
|
||||
const knownRules = JSON.parse(fs.readFileSync(new URL('../../tests/install/e2e-assets/known-failures.json', import.meta.url), 'utf8'));
|
||||
/** @type {Map<string, {number: number, rule: any}>} */
|
||||
const footnotes = new Map();
|
||||
const LEG = /^(linux|windows|macos): (\S+) -> (\S+) \(([^)]+) -> HEAD\) \//;
|
||||
/** @type {Map<string, boolean>} */
|
||||
const desktopByTag = new Map(tagAnnotations.map((t) => [t.ref, t.desktop]));
|
||||
@@ -315,7 +318,7 @@ export function renderMarkdownResults(jobs, tagAnnotations = [], artifactById =
|
||||
// run workflow may have one inner job per driver arm; exactly one runs
|
||||
// and the others natively skip), so cells merge by significance: a real
|
||||
// outcome always beats a skip, and a bad outcome beats a good one.
|
||||
const RANK = ['skip', 'TODO', 'pre-desktop', '✅', 'running', 'cancelled', '❌'];
|
||||
const RANK = ['skip', 'TODO', 'pre-desktop', '✅', 'known', 'running', 'cancelled', '❌'];
|
||||
const SKIPS = ['skip', 'TODO', 'pre-desktop'];
|
||||
// Rendered success/failure cells carry artifact links after the glyph;
|
||||
// rank by the leading token or every such cell would rank as unknown (-1)
|
||||
@@ -351,7 +354,14 @@ export function renderMarkdownResults(jobs, tagAnnotations = [], artifactById =
|
||||
? ` [📼](${runBase}/artifacts/${playerId}#zip=${encodeURIComponent(`${runBase}/artifacts/${logsId}`)}) [⬇️](${runBase}/artifacts/${logsId})`
|
||||
: '';
|
||||
switch (job.conclusion) {
|
||||
case 'success': return `✅${reel}`;
|
||||
case 'success': {
|
||||
// Only the classifier's uploaded receipt turns a successful job into
|
||||
// a known-failure cell. Tag membership alone never suppresses a red.
|
||||
const rule = knownRules.find((/** @type {any} */ r) => artifactById.has(`install-e2e-known-${r.id}--${legId2}`));
|
||||
if (!rule) return `✅${reel}`;
|
||||
if (!footnotes.has(rule.id)) footnotes.set(rule.id, { number: footnotes.size + 1, rule });
|
||||
return `known [^${footnotes.get(rule.id)?.number}]${reel}`;
|
||||
}
|
||||
case 'failure': return `❌${reel}`;
|
||||
case 'skipped': return skipLabel(m[2], m[3], tag);
|
||||
case 'cancelled': return 'cancelled';
|
||||
@@ -369,10 +379,11 @@ export function renderMarkdownResults(jobs, tagAnnotations = [], artifactById =
|
||||
const passed = cells.filter((c) => c.startsWith('✅')).length;
|
||||
const failed = cells.filter((c) => c.startsWith('❌')).length;
|
||||
const skipped = cells.filter((c) => SKIPS.includes(c)).length;
|
||||
const known = cells.filter((c) => c.startsWith('known ')).length;
|
||||
const lines = [
|
||||
'### Install & Update E2E results',
|
||||
'',
|
||||
`${passed} passed, ${failed} failed, ${skipped} skipped (TODO = declared, no driver arm yet; pre-desktop = the starting release predates apps/desktop), ${cells.length} legs total`,
|
||||
`${passed} passed, ${failed} failed, ${known} known failures, ${skipped} skipped (TODO = declared, no driver arm yet; pre-desktop = the starting release predates apps/desktop), ${cells.length} legs total`,
|
||||
'',
|
||||
`| combination | ${tags.join(' | ')} |`,
|
||||
`|---|${tags.map(() => '---').join('|')}|`,
|
||||
@@ -381,6 +392,9 @@ export function renderMarkdownResults(jobs, tagAnnotations = [], artifactById =
|
||||
lines.push(`| \`${combo}\` | ${tags.map((t) => byTag.get(t) || '-').join(' | ')} |`);
|
||||
}
|
||||
lines.push('');
|
||||
for (const { number, rule } of footnotes.values()) {
|
||||
lines.push(`[^${number}]: **${rule.title}.** ${rule.explanation} [Evidence](${rule.evidence}).`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { createRequire } from 'node:module'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
const { matchKnownFailure, rules } = createRequire(import.meta.url)('../tests/install/e2e-assets/known-failures.cjs')
|
||||
const classifier = path.resolve(import.meta.dirname, '../tests/install/e2e-assets/known-failures.cjs')
|
||||
const lockedLog = [
|
||||
'error: failed to remove file `C:/install/venv/Lib/site-packages/../../Scripts/hermes.exe`: Access is denied. (os error 5)',
|
||||
'File "C:/install/venv/Scripts/hermes.exe/__main__.py", line 10, in <module>',
|
||||
"subprocess.CalledProcessError: Command '['uv', 'pip', 'install', '-e', '.', '--quiet']' returned non-zero exit status 2.",
|
||||
].join('\n')
|
||||
const base = {
|
||||
platform: 'windows', phase: 'update', commit: 'a370ab8391ca5f8de7ebbc449f05cb0df36ade7c',
|
||||
installMethod: 'installer-script', updateMethod: 'hermes-update',
|
||||
error: 'E2E ASSERTION FAILED: hermes update exited 1 (expected 0)', logs: { update: lockedLog },
|
||||
}
|
||||
|
||||
describe('known install failures', () => {
|
||||
it('recognizes the released launcher self-lock, not generic access denied', () => {
|
||||
expect(matchKnownFailure(base)?.id).toBe('windows-launcher-self-lock')
|
||||
expect(matchKnownFailure({ ...base, logs: { update: lockedLog.replaceAll('hermes.exe', 'other.exe') } })).toBeNull()
|
||||
expect(matchKnownFailure({ ...base, logs: { update: lockedLog.replace('(os error 5)', '(os error 32)') } })).toBeNull()
|
||||
})
|
||||
|
||||
it.each([
|
||||
{ platform: 'linux' }, { phase: 'install' }, { commit: 'v2026.3.12' },
|
||||
{ commit: 'f'.repeat(40) }, { installMethod: 'desktop-installer@latest' },
|
||||
{ updateMethod: 'installer-script' }, { error: 'E2E ASSERTION FAILED: update marker cleaned up' },
|
||||
{ logs: {} },
|
||||
])('rejects a different case or missing evidence: %j', change => {
|
||||
expect(matchKnownFailure({ ...base, ...change })).toBeNull()
|
||||
})
|
||||
|
||||
it('matches manual-only app updates only for the three proven July script cases', () => {
|
||||
const sample = {
|
||||
...base, commit: '7c1a029553d87c43ecff8a3821336bc95872213b',
|
||||
updateMethod: 'hermes-desktop-app-update',
|
||||
error: 'E2E ASSERTION FAILED: app driven via captured hermes desktop spec; update completed',
|
||||
logs: { desktop: '[hermes] [updates] no staged updater; surfacing manual `hermes update` for CLI install at C:/install\n[hermes] [updates] manual: hermes update\n' },
|
||||
}
|
||||
expect(matchKnownFailure(sample)?.id).toBe('windows-july-manual-app-update')
|
||||
expect(matchKnownFailure({ ...sample, installMethod: 'desktop-installer@latest' })).toBeNull()
|
||||
expect(matchKnownFailure({ ...sample, error: 'onboarding timed out' })).toBeNull()
|
||||
expect(matchKnownFailure({ ...sample, logs: { desktop: '[updates] manual: hermes update' } })).toBeNull()
|
||||
})
|
||||
|
||||
it('CLI writes a receipt and exits zero only on a confirmed match', () => {
|
||||
const root = mkdtempSync(path.join(os.tmpdir(), 'known-install-'))
|
||||
try {
|
||||
mkdirSync(path.join(root, 'logs'))
|
||||
writeFileSync(path.join(root, 'shas.json'), '\uFEFF' + JSON.stringify({ old: base.commit, current: 'f'.repeat(40), old_ref: 'v2026.3.12' }))
|
||||
writeFileSync(path.join(root, 'logs/update.log'), lockedLog)
|
||||
const args = [classifier, root, base.installMethod, base.updateMethod, base.error]
|
||||
expect(spawnSync(process.execPath, args).status).toBe(0)
|
||||
expect(JSON.parse(readFileSync(path.join(root, 'known-failure.json'), 'utf8')).id).toBe('windows-launcher-self-lock')
|
||||
writeFileSync(path.join(root, 'logs/update.log'), 'an unrelated failure')
|
||||
expect(spawnSync(process.execPath, args).status).toBe(1)
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
it('renders known receipts as footnotes, without suppressing a red job', async () => {
|
||||
const modulePath = '../scripts/sandbox/generate-e2e-matrix.mjs'
|
||||
const { renderMarkdownResults, legId } = await import(modulePath)
|
||||
const name = 'windows: installer-script -> hermes-update (v2026.3.12 -> HEAD)'
|
||||
const artifacts = new Map([[`install-e2e-known-${rules[0].id}--${legId(name)}`, 42]])
|
||||
const known = renderMarkdownResults([{ name: name + ' / e2e', conclusion: 'success' }], [], artifacts)
|
||||
expect(known).toContain('0 passed, 0 failed, 1 known failures')
|
||||
expect(known).toContain('known [^1]')
|
||||
expect(known).toContain(`[^1]: **${rules[0].title}.**`)
|
||||
const failed = renderMarkdownResults([{ name: name + ' / e2e', conclusion: 'failure' }], [], artifacts)
|
||||
expect(failed).toContain('0 passed, 1 failed, 0 known failures')
|
||||
expect(failed).not.toContain('known [^1]')
|
||||
const passed = renderMarkdownResults([{ name: name + ' / e2e', conclusion: 'success' }])
|
||||
expect(passed).toContain('1 passed, 0 failed, 0 known failures')
|
||||
})
|
||||
@@ -1,6 +1,8 @@
|
||||
# Confirmed historical upgrade limitations
|
||||
|
||||
These failures cannot be repaired by changing the update target: the failing code is already loaded from the starting release. This is an evidence register, not a skip list. The workflow still runs these legs and preserves their failing exit codes. A later failure must match the recorded cause before it receives this classification; the tag alone is not enough.
|
||||
These failures cannot be repaired by changing the update target: the failing code is already loaded from the starting release. The workflow still executes each original update path. Only a match on the exact starting commit, method pair, failed assertion, and fresh log signatures produces a non-red known-failure receipt. Other errors still fail. The results table shows matched cases as `known [n]`, with the explanation and evidence in a footnote at the bottom. These cases are counted separately from passed upgrades.
|
||||
|
||||
The machine-readable rules in `e2e-assets/known-failures.json` own the matcher and report footnote text. This document explains their historical evidence. Logs are rotated before each attempt so an earlier failure cannot classify a later one.
|
||||
|
||||
## Windows launcher self-lock
|
||||
|
||||
|
||||
@@ -63,7 +63,7 @@ A grey leg is normal. There are two causes:
|
||||
- The method pair is declared but cannot run: either no OS entry point exists for it (open-app-update after a plain script install registers nothing to open), or no driver arm exists yet. The gate in the run workflow lists the pairs that run.
|
||||
- The starting release predates the surface under test. Example: a release without `apps/desktop` has no window to launch. The tag annotation `tag_has_desktop` from the primary workflow marks these releases.
|
||||
|
||||
The result chart on the run summary shows each leg as passed, failed, or skipped. [Confirmed historical upgrade limitations](KNOWN_FAILURES.md) records failures that cannot be fixed in the update target, with exact release commits and CI evidence. These are not blanket skips: the original paths still run and failures remain visible.
|
||||
The result chart on the run summary shows each leg as passed, failed, or skipped. [Confirmed historical upgrade limitations](KNOWN_FAILURES.md) records failures that cannot be fixed in the update target, with exact release commits and CI evidence. These are not blanket skips: the original paths still run. Exact signature matches are non-red, counted separately as known failures, and linked to footnotes at the bottom of the result chart. An unrelated error on the same tag still fails.
|
||||
|
||||
## Triggers and cost
|
||||
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
"""CI-only Python stack snapshots for the opaque staged-updater hand-off.
|
||||
|
||||
No command arguments, environment, or frame locals are recorded. The real
|
||||
updater runs unchanged; stacks identify where its child is blocked.
|
||||
"""
|
||||
import os
|
||||
|
||||
if os.environ.get("GITHUB_ACTIONS") == "true" and os.environ.get("HERMES_E2E_HANDOFF_TRACE"):
|
||||
import faulthandler
|
||||
from pathlib import Path
|
||||
|
||||
_directory = Path(os.environ["HERMES_E2E_HANDOFF_TRACE"])
|
||||
_directory.mkdir(parents=True, exist_ok=True)
|
||||
_stream = (_directory / f"python-stacks-{os.getpid()}.log").open("a", encoding="utf-8")
|
||||
faulthandler.dump_traceback_later(90, repeat=True, file=_stream)
|
||||
@@ -0,0 +1,156 @@
|
||||
# CI-only diagnostic sampler for the July staged-updater handoff stall.
|
||||
#
|
||||
# Runs on the GitHub Actions Windows runner only; never on a user workstation.
|
||||
# Captures a bounded, secret-free snapshot of the update handoff state:
|
||||
# - processes whose executable or command line references the staged
|
||||
# hermes-setup.exe (or anything under the e2e hermes-home), plus their
|
||||
# full descendant tree: pid, ppid, exe path, sanitized command line,
|
||||
# CPU seconds, working set
|
||||
# - the update-in-progress marker file (pid + timestamp, non-secret)
|
||||
# - git HEAD + `git status --porcelain` file NAMES only (no diffs)
|
||||
# - update log filenames/sizes (no contents)
|
||||
#
|
||||
# Everything prints to stdout so the parent job log carries the snapshot.
|
||||
# Usage: powershell -File july-handoff-diagnostics.ps1 -WorkRoot <dir> [-Label handoff|timeout|finally]
|
||||
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$WorkRoot,
|
||||
[string]$Label = "sample"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "SilentlyContinue"
|
||||
if ($env:GITHUB_ACTIONS -ne "true") { throw "handoff diagnostics are restricted to disposable CI runners" }
|
||||
|
||||
function Write-Section([string]$Name) {
|
||||
Write-Output ""
|
||||
Write-Output "=== july-handoff-diagnostics [$Label] $Name ==="
|
||||
}
|
||||
|
||||
if (-not (Test-Path $WorkRoot)) {
|
||||
Write-Output "=== july-handoff-diagnostics [$Label] WorkRoot not found: $WorkRoot ==="
|
||||
exit 0
|
||||
}
|
||||
$WorkRoot = (Resolve-Path $WorkRoot).Path
|
||||
|
||||
# Flags whose VALUE is redacted from command lines. Names only are kept.
|
||||
$SensitiveFlags = @("--token", "--key", "--api-key", "--password", "--secret", "-t", "--auth")
|
||||
|
||||
function Format-Cmdline([string]$ExePath, [string]$Cmdline) {
|
||||
# Tokenize on whitespace, redact the value that follows a sensitive flag,
|
||||
# and redact anything that looks like an embedded secret assignment.
|
||||
if ([string]::IsNullOrWhiteSpace($Cmdline)) { return "<no cmdline>" }
|
||||
$parts = @($Cmdline -split '\s+')
|
||||
$out = New-Object System.Collections.Generic.List[string]
|
||||
for ($i = 0; $i -lt $parts.Count; $i++) {
|
||||
$p = $parts[$i]
|
||||
if ($SensitiveFlags -contains $p.ToLower()) {
|
||||
$out.Add($p)
|
||||
if ($i + 1 -lt $parts.Count) { $out.Add("<redacted>"); $i++ }
|
||||
}
|
||||
elseif ($p -match '(?i)(token|secret|password|api[_-]?key)\s*=') {
|
||||
$out.Add(($p -replace '=.*$', '=<redacted>'))
|
||||
}
|
||||
else { $out.Add($p) }
|
||||
}
|
||||
return ($out -join " ")
|
||||
}
|
||||
|
||||
Write-Section "meta"
|
||||
Write-Output ("utc={0} workroot={1}" -f (Get-Date).ToUniversalTime().ToString("o"), $WorkRoot)
|
||||
|
||||
Write-Section "processes"
|
||||
$procs = @(Get-CimInstance Win32_Process -ErrorAction SilentlyContinue)
|
||||
if ($procs.Count -eq 0) {
|
||||
Write-Output "Get-CimInstance returned nothing"
|
||||
}
|
||||
$rootPids = @{}
|
||||
foreach ($p in $procs) {
|
||||
$exe = [string]$p.ExecutablePath
|
||||
$cmd = [string]$p.CommandLine
|
||||
$ref = ($exe -like "$WorkRoot\*") -or ($cmd -like "*$WorkRoot\*")
|
||||
if ($ref) { $rootPids[[uint32]$p.ProcessId] = $true }
|
||||
}
|
||||
# Expand descendants transitively (both directions of interest: children of
|
||||
# the staged updater and children of its hermes update child).
|
||||
$changed = $true
|
||||
while ($changed) {
|
||||
$changed = $false
|
||||
foreach ($p in $procs) {
|
||||
$pp = [uint32]$p.ParentProcessId
|
||||
$cp = [uint32]$p.ProcessId
|
||||
if (-not $rootPids.ContainsKey($cp) -and $rootPids.ContainsKey($pp)) {
|
||||
$rootPids[$cp] = $true
|
||||
$changed = $true
|
||||
}
|
||||
}
|
||||
}
|
||||
if ($rootPids.Count -eq 0) {
|
||||
Write-Output "no hermes/staged-updater processes alive"
|
||||
}
|
||||
foreach ($p in $procs | Sort-Object ProcessId) {
|
||||
$cp = [uint32]$p.ProcessId
|
||||
if (-not $rootPids.ContainsKey($cp)) { continue }
|
||||
$cpu = "-"
|
||||
$ws = "-"
|
||||
try {
|
||||
$raw = Get-Process -Id $cp -ErrorAction SilentlyContinue
|
||||
if ($raw) {
|
||||
$cpu = [math]::Round($raw.TotalProcessorTime.TotalSeconds, 1)
|
||||
$ws = [math]::Round($raw.WorkingSet64 / 1MB, 1)
|
||||
}
|
||||
} catch {}
|
||||
$marker = ""
|
||||
if ($rootPids.ContainsKey([uint32]$p.ParentProcessId)) { $marker = "child-of=$($p.ParentProcessId)" }
|
||||
elseif ([uint32]$p.ParentProcessId -ne 0) { $marker = "root(parent=$($p.ParentProcessId))" }
|
||||
Write-Output ("pid={0} {1} cpu_s={2} ws_mb={3} exe={4}" -f $cp, $marker, $cpu, $ws, $p.ExecutablePath)
|
||||
Write-Output (" cmd: {0}" -f (Format-Cmdline $p.ExecutablePath $p.CommandLine))
|
||||
}
|
||||
|
||||
Write-Section "update-in-progress-marker"
|
||||
$markerPath = Join-Path $WorkRoot "hermes-home\hermes-agent\.hermes-update-in-progress"
|
||||
if (-not (Test-Path $markerPath)) {
|
||||
# Common alternate layout: marker lives directly under hermes-home.
|
||||
$alt = Join-Path $WorkRoot "hermes-home\.hermes-update-in-progress"
|
||||
if (Test-Path $alt) { $markerPath = $alt } else { $markerPath = $null }
|
||||
}
|
||||
if ($markerPath -and (Test-Path $markerPath)) {
|
||||
$fi = Get-Item $markerPath
|
||||
Write-Output ("marker={0} size={1} mtime={2}" -f $fi.FullName, $fi.Length, $fi.LastWriteTimeUtc.ToString("o"))
|
||||
# Contents are "pid\nstarted_at" — non-secret by contract.
|
||||
Write-Output ("marker-contents: {0}" -f ((Get-Content $markerPath -Raw) -replace "`r?`n", " / ").Trim())
|
||||
} else {
|
||||
Write-Output "no update-in-progress marker found"
|
||||
}
|
||||
|
||||
Write-Section "git"
|
||||
$repo = Join-Path $WorkRoot "hermes-home\hermes-agent"
|
||||
if (Test-Path (Join-Path $repo ".git")) {
|
||||
$head = & git -C $repo rev-parse HEAD 2>$null
|
||||
$branch = & git -C $repo rev-parse --abbrev-ref HEAD 2>$null
|
||||
Write-Output ("head={0} branch={1}" -f $head, $branch)
|
||||
# Names only: no diff content, no remote URLs, no stash payloads.
|
||||
$st = & git -C $repo status --porcelain 2>$null
|
||||
if ($st) { $st | ForEach-Object { Write-Output ("status: {0}" -f $_) } }
|
||||
else { Write-Output "status: clean" }
|
||||
$last = & git -C $repo log -1 --format="%h %ad %s" --date=short 2>$null
|
||||
Write-Output ("last-commit: {0}" -f $last)
|
||||
} else {
|
||||
Write-Output "no .git under $repo"
|
||||
}
|
||||
|
||||
Write-Section "logs"
|
||||
foreach ($dir in @(
|
||||
(Join-Path $WorkRoot "hermes-home\hermes-agent\logs"),
|
||||
(Join-Path $WorkRoot "hermes-home\logs"))) {
|
||||
if (Test-Path $dir) {
|
||||
Get-ChildItem $dir -File -ErrorAction SilentlyContinue |
|
||||
Sort-Object LastWriteTimeUtc -Descending |
|
||||
Select-Object -First 15 |
|
||||
ForEach-Object {
|
||||
Write-Output ("{0} size={1} mtime={2}" -f $_.FullName, $_.Length, $_.LastWriteTimeUtc.ToString("o"))
|
||||
}
|
||||
}
|
||||
}
|
||||
Write-Output ""
|
||||
Write-Output "=== july-handoff-diagnostics [$Label] done ==="
|
||||
exit 0
|
||||
@@ -0,0 +1,54 @@
|
||||
const fs = require('node:fs')
|
||||
const path = require('node:path')
|
||||
const rules = require('./known-failures.json')
|
||||
|
||||
function matchKnownFailure({ platform, phase, commit, installMethod, updateMethod, error, logs }) {
|
||||
if (platform !== 'windows' || phase !== 'update' || !/^[0-9a-f]{40}$/.test(commit || '')) return null
|
||||
return rules.find(rule =>
|
||||
rule.commits.includes(commit) &&
|
||||
rule.cases.some(([install, update]) => install === installMethod && update === updateMethod) &&
|
||||
rule.errors.some(pattern => new RegExp(pattern).test(error || '')) &&
|
||||
rule.signatures.every(pattern => new RegExp(pattern, 'i').test(logs[rule.log] || '')),
|
||||
) || null
|
||||
}
|
||||
|
||||
function readOptional(file) {
|
||||
try { return fs.readFileSync(file, 'utf8').replace(/^\uFEFF/, '') } catch (error) {
|
||||
if (error.code === 'ENOENT') return ''
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function classifyWorkRoot(root, installMethod, updateMethod, error) {
|
||||
const state = JSON.parse(fs.readFileSync(path.join(root, 'shas.json'), 'utf8').replace(/^\uFEFF/, ''))
|
||||
const rule = matchKnownFailure({
|
||||
platform: 'windows', phase: 'update', commit: state.old, installMethod, updateMethod, error,
|
||||
logs: {
|
||||
update: readOptional(path.join(root, 'logs', 'update.log')),
|
||||
desktop: readOptional(path.join(root, 'hermes-home', 'logs', 'desktop.log')),
|
||||
},
|
||||
})
|
||||
if (!rule) return null
|
||||
return {
|
||||
id: rule.id, title: rule.title, explanation: rule.explanation, evidence: rule.evidence,
|
||||
commit: state.old, target: state.current, installRef: state.old_ref,
|
||||
installMethod, updateMethod, error,
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { matchKnownFailure, classifyWorkRoot, rules }
|
||||
|
||||
if (require.main === module) {
|
||||
const [root, install, update, error] = process.argv.slice(2)
|
||||
try {
|
||||
const receipt = classifyWorkRoot(root, install, update, error)
|
||||
if (!receipt) process.exitCode = 1
|
||||
else {
|
||||
fs.writeFileSync(path.join(root, 'known-failure.json'), JSON.stringify(receipt, null, 2) + '\n')
|
||||
console.log(JSON.stringify(receipt))
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`known-failure classification failed: ${error.message}`)
|
||||
process.exitCode = 2
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
[
|
||||
{
|
||||
"id": "windows-launcher-self-lock",
|
||||
"title": "Released Windows updater locks its own console launcher",
|
||||
"commits": ["a370ab8391ca5f8de7ebbc449f05cb0df36ade7c", "86960cdbb0148145890e2ee90b4e157fa899f6e1"],
|
||||
"cases": [["installer-script", "hermes-update"]],
|
||||
"errors": ["^E2E ASSERTION FAILED: hermes update exited [1-9][0-9]* \\(expected 0\\)$"],
|
||||
"log": "update",
|
||||
"signatures": ["failed to remove file[^\\r\\n]*Scripts[/\\\\]hermes\\.exe[^\\r\\n]*Access is denied\\. \\(os error 5\\)", "hermes\\.exe[/\\\\]__main__\\.py", "CalledProcessError[^\\r\\n]*pip[^\\r\\n]*install[^\\r\\n]*returned non-zero exit status 2"],
|
||||
"explanation": "The March/April Windows updater is already running from hermes.exe when uv tries to replace it. Windows refuses the locked launcher. The update target cannot change that loaded code; re-running the installer is a separate recovery route.",
|
||||
"evidence": "https://github.com/ethernet8023/hermes-agent/actions/runs/34055462305/job/101546487700"
|
||||
},
|
||||
{
|
||||
"id": "windows-july-manual-app-update",
|
||||
"title": "July script installs offer a manual update instead of an app hand-off",
|
||||
"commits": ["7c1a029553d87c43ecff8a3821336bc95872213b"],
|
||||
"cases": [["installer-script", "hermes-desktop-app-update"], ["installer-script+desktop", "hermes-desktop-app-update"], ["installer-script+desktop", "open-app-update"]],
|
||||
"errors": ["^E2E ASSERTION FAILED: app driven via captured hermes desktop spec; update completed$", "^E2E ASSERTION FAILED: GUI driver clicked Update now and the app quit for hand-off$"],
|
||||
"log": "desktop",
|
||||
"signatures": ["\\[updates\\] no staged updater; surfacing manual `hermes update` for CLI install at", "\\[updates\\] manual: hermes update(?:\\r?\\n|$)"],
|
||||
"explanation": "The July Windows app has no staged updater after a script install. Its Update button explicitly returns manual: hermes update without starting a hand-off. Desktop-installer installs are not covered by this exception.",
|
||||
"evidence": "https://github.com/ethernet8023/hermes-agent/actions/runs/34055462305/job/101546487667"
|
||||
}
|
||||
]
|
||||
@@ -797,7 +797,12 @@ function Invoke-GuiUpdateDesktopRoute([string]$TargetSha) {
|
||||
$updateLog = Join-Path $HermesHome "logs\update.log"
|
||||
$updateLogPos = 0
|
||||
$deadline = (Get-Date).AddMinutes(35)
|
||||
$nextDiagnostic = Get-Date
|
||||
while ((Get-Date) -lt $deadline) {
|
||||
if ($env:GITHUB_ACTIONS -eq "true" -and (Get-Date) -ge $nextDiagnostic) {
|
||||
& powershell -NoProfile -ExecutionPolicy Bypass -File (Join-Path $AssetsDir "july-handoff-diagnostics.ps1") -WorkRoot $WorkRoot -Label "waiting"
|
||||
$nextDiagnostic = (Get-Date).AddMinutes(2)
|
||||
}
|
||||
if (Test-Path -LiteralPath $resultPath) { break }
|
||||
$head = ""
|
||||
try { $head = Get-InstalledHead } catch {}
|
||||
@@ -871,6 +876,9 @@ function Invoke-GuiUpdateDesktopRoute([string]$TargetSha) {
|
||||
Write-Host "::endgroup::"
|
||||
Copy-Item $handoffLog (Join-Path $proof "desktop-update-handoff.log") -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
if ($env:GITHUB_ACTIONS -eq "true") {
|
||||
& powershell -NoProfile -ExecutionPolicy Bypass -File (Join-Path $AssetsDir "july-handoff-diagnostics.ps1") -WorkRoot $WorkRoot -Label "before-teardown"
|
||||
}
|
||||
# Quit the relaunched app so job teardown is clean.
|
||||
Stop-HermesAppProcesses "post-update"
|
||||
}
|
||||
@@ -965,6 +973,40 @@ function Invoke-PhaseUpdate {
|
||||
Test-HermesRuns "post-update"
|
||||
}
|
||||
|
||||
function Invoke-CheckedPhaseUpdate {
|
||||
# Trace old Python stacks on CI without replacing updater behavior.
|
||||
$state = Read-State
|
||||
if ($env:GITHUB_ACTIONS -eq "true" -and $state.old -eq "7c1a029553d87c43ecff8a3821336bc95872213b" -and $InstallMethod -eq "desktop-installer@latest" -and $Route -eq "open-app-update") {
|
||||
$traceDir = Join-Path $AssetsDir "handoff-trace"
|
||||
$env:PYTHONPATH = if ($env:PYTHONPATH) { "$traceDir;$env:PYTHONPATH" } else { $traceDir }
|
||||
$env:HERMES_E2E_HANDOFF_TRACE = Join-Path $WorkRoot "proof\handoff-stacks"
|
||||
$env:PYTHONUNBUFFERED = "1"
|
||||
}
|
||||
Remove-Item -LiteralPath (Join-Path $WorkRoot "known-failure.json") -Force -ErrorAction SilentlyContinue
|
||||
# Only evidence produced by this update attempt can match an exception.
|
||||
foreach ($oldLog in @((Join-Path $WorkRoot "logs\update.log"), (Join-Path $HermesHome "logs\desktop.log"))) {
|
||||
if (Test-Path -LiteralPath $oldLog) { Move-Item -LiteralPath $oldLog -Destination "$oldLog.before-update" -Force }
|
||||
}
|
||||
try {
|
||||
Invoke-PhaseUpdate
|
||||
} catch {
|
||||
$failure = $_
|
||||
$node = Get-ManagedNode
|
||||
$classification = & $node (Join-Path $AssetsDir "known-failures.cjs") $WorkRoot $InstallMethod $Route $failure.Exception.Message
|
||||
$classificationExit = $LASTEXITCODE
|
||||
if ($classificationExit -ne 0) { throw $failure }
|
||||
$receipt = ($classification | Out-String) | ConvertFrom-Json
|
||||
Write-Host "KNOWN FAILURE [$($receipt.id)]: $($receipt.title)"
|
||||
Write-Host " $($receipt.explanation)"
|
||||
if ($env:GITHUB_OUTPUT) {
|
||||
Add-Content -LiteralPath $env:GITHUB_OUTPUT -Value "known_failure=$($receipt.id)" -Encoding UTF8
|
||||
}
|
||||
if ($env:GITHUB_STEP_SUMMARY) {
|
||||
Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Encoding UTF8 -Value "Known historical failure: $($receipt.title). See the result chart footnote and uploaded known-failure.json."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Dispatch
|
||||
# ----------------------------------------------------------------------------
|
||||
@@ -982,11 +1024,11 @@ Set-GitRedirect
|
||||
switch ($Phase) {
|
||||
"stage" { Invoke-PhaseStage }
|
||||
"install" { Invoke-PhaseInstall }
|
||||
"update" { Invoke-PhaseUpdate }
|
||||
"update" { Invoke-CheckedPhaseUpdate }
|
||||
"all" {
|
||||
Invoke-PhaseStage
|
||||
Invoke-PhaseInstall
|
||||
Invoke-PhaseUpdate
|
||||
Invoke-CheckedPhaseUpdate
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user