fix(update): conservative outcomes + serve-ledger coverage for fresh restart recovery

Salvage adjustments to PR #94392 per review:

- Narrow the supervisor claim to the systemd-VERIFIED path only. The fresh
  recovery child now probes 'systemctl --user is-active' after each relaunch;
  only an observed-active systemd unit is reported 'verified'. A relaunch that
  merely exited 0 is labelled 'relaunch_attempted', never counts as supervisor
  coverage, and never clears gateway_fleet_restart_incomplete.
- Serve-owned runtimes (serve/dashboard entries from the spawn ledger, per the
  update_inventory serve collector) are no longer silently skipped: the
  recovery pass records them (and manual gateways) as skipped-with-reason in
  the recovery result and the persisted update receipt.
- Receipt fresh_recovery persists the conservative vocabulary
  (requested/verified/relaunch_attempted/failed/skipped); 'succeeded' is gone.
- Added an end-to-end test that drives the real recovery module in a genuinely
  fresh interpreter (sitecustomize shim intercepts the grandchild
  'gateway restart' and systemctl probes).
This commit is contained in:
Teknium
2026-08-26 16:33:46 -07:00
parent f9135c189c
commit b2c011364e
7 changed files with 545 additions and 107 deletions
+1
View File
@@ -5083,6 +5083,7 @@ _LAZY_COMMAND_EXPORTS = {
"_format_concurrent_instances_message",
"_format_time_ago",
"_gateway_service_matches_profile",
"_gateway_recovery_partition",
"_gateway_restart_recovery_profiles",
"_handoff_reapable_backend_pids",
"_ledger_reapable_backend_pids",
+143 -58
View File
@@ -5615,36 +5615,89 @@ def _gateway_service_matches_profile(profile: str, service: object) -> bool:
}
def _gateway_restart_recovery_profiles(
def _gateway_recovery_partition(
plan, *, skip_profiles: set[str] | None = None
) -> list[str]:
"""Return supervised gateway profiles that a fresh process may restart.
) -> tuple[dict[str, str], list[dict]]:
"""Partition pre-update runtimes into fresh-restart candidates and skips.
The update inventory is captured before the checkout changes. It is the
only safe source here: re-importing ``hermes_cli.gateway`` in the failing
interpreter is exactly what can raise the original ``ImportError``. Manual
gateways are intentionally excluded because no supervisor is available to
bring them back after a stop.
interpreter is exactly what can raise the original ``ImportError``.
Returns ``(candidates, skipped)`` where ``candidates`` maps profile →
supervisor for supervised gateway runtimes the fresh process may restart,
and ``skipped`` lists every other inventoried runtime the recovery pass
deliberately does NOT touch, each with an explicit reason. Nothing from
the spawn ledger may vanish from the recovery pass silently: manual
gateways have no relaunch authority, and serve/dashboard runtimes (the
``update_inventory`` serve collector) are owned by the Desktop app or a
human terminal, not by this recovery boundary.
"""
profiles: set[str] = set()
skip_profiles = skip_profiles or set()
candidates: dict[str, str] = {}
skipped: list[dict] = []
try:
for runtime in getattr(plan, "runtimes", ()) or ():
if (
getattr(runtime, "kind", None) == "gateway"
and getattr(runtime, "supervisor", None) in _FRESH_RESTART_SUPERVISORS
):
profile = getattr(runtime, "profile", None)
if isinstance(profile, str) and profile and profile not in skip_profiles:
profiles.add(profile)
kind = getattr(runtime, "kind", None)
profile = getattr(runtime, "profile", None)
supervisor = getattr(runtime, "supervisor", None)
if not isinstance(profile, str) or not profile:
continue
if kind == "gateway":
if profile in skip_profiles:
continue
if supervisor in _FRESH_RESTART_SUPERVISORS:
candidates.setdefault(profile, str(supervisor))
else:
skipped.append(
{
"profile": profile,
"kind": "gateway",
"supervisor": str(supervisor),
"reason": (
"manual gateway has no supervisor relaunch"
" authority; left running for explicit operator"
" restart"
),
}
)
elif kind in ("serve", "dashboard"):
if supervisor == "desktop":
reason = (
"desktop app owns and respawns this serve backend;"
" the recovery pass must not restart it out from under"
" its supervisor"
)
else:
reason = (
"manually launched serve/dashboard has no relaunch"
" authority; left running for explicit operator"
" restart"
)
skipped.append(
{
"profile": profile,
"kind": str(kind),
"supervisor": str(supervisor),
"reason": reason,
}
)
except Exception as exc:
logger.debug("Could not prepare fresh gateway restart profiles: %s", exc)
return sorted(profiles)
return candidates, skipped
def _gateway_restart_recovery_profiles(
plan, *, skip_profiles: set[str] | None = None
) -> list[str]:
"""Return supervised gateway profiles that a fresh process may restart."""
candidates, _ = _gateway_recovery_partition(plan, skip_profiles=skip_profiles)
return sorted(candidates)
def _recover_gateway_restart_after_abort(
plan, *, gateway_mode: bool, skip_profiles: set[str] | None = None
) -> dict[str, list[str]]:
) -> dict[str, list]:
"""Retry supervised gateway restarts from a clean Python process.
``hermes update`` normally performs the fleet restart in the interpreter
@@ -5658,12 +5711,38 @@ def _recover_gateway_restart_after_abort(
Only profiles classified as supervisor-owned by the pre-update inventory
are handed off. A manual gateway must remain running and be reported for
explicit operator action rather than being killed without a relaunch
authority. The returned protocol is persisted in the update receipt so
operators can distinguish a spawn failure from a per-profile failure.
authority; serve/dashboard runtimes from the spawn ledger are likewise
recorded as skipped with a reason instead of vanishing from the pass.
The returned protocol is persisted in the update receipt so operators can
distinguish a spawn failure from a per-profile failure.
Outcome honesty: ``verified`` means the fresh child independently observed
the profile's systemd unit active after the relaunch. A zero exit from
``gateway restart`` alone is NOT observed proof that the new code
generation is serving, so those outcomes are reported as
``relaunch_attempted`` and never claim supervisor coverage.
"""
profiles = _gateway_restart_recovery_profiles(plan, skip_profiles=skip_profiles)
candidates, skipped = _gateway_recovery_partition(
plan, skip_profiles=skip_profiles
)
profiles = sorted(candidates)
if not profiles:
return {"requested": [], "succeeded": [], "failed": []}
return {
"requested": [],
"verified": [],
"relaunch_attempted": [],
"failed": [],
"skipped": skipped,
}
def _all_failed() -> dict[str, list]:
return {
"requested": profiles,
"verified": [],
"relaunch_attempted": [],
"failed": profiles,
"skipped": skipped,
}
command = [
sys.executable,
@@ -5685,7 +5764,7 @@ def _recover_gateway_restart_after_abort(
systemd_run = shutil.which("systemd-run")
if not systemd_run:
logger.warning("Cannot isolate fresh gateway recovery from the gateway cgroup")
return {"requested": profiles, "succeeded": [], "failed": profiles}
return _all_failed()
command = [
systemd_run,
"--user",
@@ -5697,7 +5776,7 @@ def _recover_gateway_restart_after_abort(
]
kwargs = {
"input": json.dumps({"profiles": profiles}),
"input": json.dumps({"profiles": profiles, "supervisors": candidates}),
"capture_output": True,
"text": True,
"encoding": "utf-8",
@@ -5718,39 +5797,52 @@ def _recover_gateway_restart_after_abort(
result = subprocess.run(command, **kwargs)
except (OSError, subprocess.TimeoutExpired) as exc:
logger.warning("Fresh gateway restart recovery failed: %s", exc)
return {"requested": profiles, "succeeded": [], "failed": profiles}
return _all_failed()
if result.returncode != 0:
logger.warning("Fresh gateway restart recovery exited %s", result.returncode)
return {"requested": profiles, "succeeded": [], "failed": profiles}
return _all_failed()
try:
recovery_result = json.loads(result.stdout or "")
succeeded = recovery_result.get("succeeded")
verified = recovery_result.get("verified")
relaunch_attempted = recovery_result.get("relaunch_attempted")
failed = recovery_result.get("failed")
except (AttributeError, TypeError, ValueError):
logger.warning("Fresh gateway restart recovery returned invalid JSON")
return {"requested": profiles, "succeeded": [], "failed": profiles}
return _all_failed()
buckets = (verified, relaunch_attempted, failed)
reported: list[str] = []
if all(isinstance(bucket, list) for bucket in buckets):
reported = [*verified, *relaunch_attempted, *failed]
if (
not isinstance(succeeded, list)
or not isinstance(failed, list)
or any(not isinstance(profile, str) for profile in [*succeeded, *failed])
or set(succeeded) != set(profiles)
or failed
not all(isinstance(bucket, list) for bucket in buckets)
or any(not isinstance(profile, str) for profile in reported)
or set(reported) != set(profiles)
or len(reported) != len(set(reported))
):
logger.warning("Fresh gateway restart recovery returned incomplete profiles")
return {
"requested": profiles,
"succeeded": succeeded if isinstance(succeeded, list) else [],
"failed": failed if isinstance(failed, list) else profiles,
}
return _all_failed()
print(
" ✓ Retried supervised gateway restart(s) in a fresh process: "
+ ", ".join(profiles)
)
return {"requested": profiles, "succeeded": succeeded, "failed": []}
if verified:
print(
" ✓ Restarted supervised gateway(s) in a fresh process"
" (systemd-verified active): " + ", ".join(sorted(verified))
)
if relaunch_attempted:
print(
" ⚠ Relaunch attempted in a fresh process but not"
" supervisor-verified (check these gateways manually): "
+ ", ".join(sorted(relaunch_attempted))
)
return {
"requested": profiles,
"verified": sorted(verified),
"relaunch_attempted": sorted(relaunch_attempted),
"failed": sorted(failed),
"skipped": skipped,
}
def _warn_gateway_restart_phase_aborted(exc: BaseException, pids) -> None:
@@ -8802,16 +8894,14 @@ def _cmd_update_impl(args, gateway_mode: bool):
gateway_mode=gateway_mode,
skip_profiles=_already_restarted_profiles,
)
_recovery_profiles = list(_recovery_result.get("requested") or [])
_recovery_succeeded = set(_recovery_result.get("succeeded") or [])
_recovered = not _recovery_profiles or (
_recovery_succeeded == set(_recovery_profiles)
and not _recovery_result.get("failed")
)
if _recovery_succeeded:
# Only systemd-VERIFIED outcomes may claim supervisor coverage.
# A relaunch that merely exited 0 ("relaunch_attempted") was never
# observed by the code and must not clear the incomplete flag.
_recovery_verified = set(_recovery_result.get("verified") or [])
if _recovery_verified:
relaunched_profiles.extend(
profile
for profile in sorted(_recovery_succeeded)
for profile in sorted(_recovery_verified)
if profile not in relaunched_profiles
)
_planned_gateway_runtimes = [
@@ -8823,18 +8913,13 @@ def _cmd_update_impl(args, gateway_mode: bool):
_planned_gateway_profiles = {
runtime.profile for runtime in _planned_gateway_runtimes
}
_covered_gateway_profiles = _already_restarted_profiles | set(
_recovery_succeeded
)
_all_gateway_profiles_are_safe_to_recover = all(
runtime.profile in _already_restarted_profiles
or getattr(runtime, "supervisor", None) in _FRESH_RESTART_SUPERVISORS
for runtime in _planned_gateway_runtimes
_covered_gateway_profiles = (
_already_restarted_profiles | _recovery_verified
)
_recovery_complete = bool(_planned_gateway_profiles) and (
_planned_gateway_profiles <= _covered_gateway_profiles
and _all_gateway_profiles_are_safe_to_recover
and (not _recovery_profiles or _recovered)
and not _recovery_result.get("failed")
and not _recovery_result.get("relaunch_attempted")
)
if _recovery_complete:
# The fresh child is the recovery terminal result. Leave the
+19 -10
View File
@@ -113,17 +113,26 @@ class UpdateReceipt:
"phase_error": phase_error,
}
if fresh_recovery is not None:
result["fresh_recovery"] = {
"requested": [
str(profile) for profile in fresh_recovery.get("requested", [])
],
"succeeded": [
str(profile) for profile in fresh_recovery.get("succeeded", [])
],
"failed": [
str(profile) for profile in fresh_recovery.get("failed", [])
],
# Conservative outcome vocabulary: "verified" is the only bucket
# allowed to claim supervisor coverage; "relaunch_attempted" means
# the relaunch exited 0 without independent supervisor
# observation. "skipped" preserves runtimes (manual gateways,
# serve/dashboard entries) the pass deliberately did not touch.
persisted: dict[str, Any] = {
key: [str(profile) for profile in fresh_recovery.get(key, [])]
for key in ("requested", "verified", "relaunch_attempted", "failed")
}
persisted["skipped"] = [
{
"profile": str(entry.get("profile", "")),
"kind": str(entry.get("kind", "")),
"supervisor": str(entry.get("supervisor", "")),
"reason": str(entry.get("reason", "")),
}
for entry in fresh_recovery.get("skipped", [])
if isinstance(entry, dict)
]
result["fresh_recovery"] = persisted
self.data["gateway_restart"] = result
def finalize(self, outcome: str) -> None:
+115 -12
View File
@@ -6,6 +6,19 @@ itself and launches the regular per-profile gateway command in a new
interpreter. It is used only after the in-process restart phase has raised, so
that the recovery path cannot inherit the stale ``sys.modules`` graph that
caused the failure.
Outcome vocabulary (deliberately conservative):
- ``verified`` — the relaunch command exited 0 AND the profile's
systemd unit was independently observed ``active`` afterwards. This is the
only outcome that may claim supervisor coverage.
- ``relaunch_attempted`` — the relaunch command exited 0 but no independent
supervisor observation was possible (non-systemd supervisor, ``systemctl``
missing, or the unit probe was inconclusive). ``rc == 0`` from
``gateway restart`` is not proof that the new code generation is running,
so this outcome must never be treated as verified coverage.
- ``failed`` — the relaunch command errored, timed out, or exited
non-zero.
"""
from __future__ import annotations
@@ -14,15 +27,18 @@ import argparse
import json
import os
import re
import shutil
import subprocess
import sys
from collections.abc import Callable, Iterable
from collections.abc import Callable, Iterable, Mapping
from typing import Any
_RECOVERY_ENV = "HERMES_UPDATE_RESTART_RECOVERY"
_GATEWAY_MARKERS = ("_HERMES_GATEWAY", "HERMES_GATEWAY", "HERMES_GATEWAY_MODE")
_PROFILE_RESTART_TIMEOUT = 90
_VERIFY_TIMEOUT = 15
_PROFILE_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$")
_SUPERVISOR_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,31}$")
def _profile_command(profile: str) -> list[str]:
@@ -78,8 +94,57 @@ def _run_profile_restart(
return getattr(result, "returncode", 1) == 0
def _systemd_unit_candidates(profile: str) -> tuple[str, ...]:
"""Unit names the existing systemd gateway lifecycle produces per profile."""
if profile == "default":
return (
"hermes-gateway.service",
"gateway.service",
"gateway-default.service",
)
return (
f"hermes-gateway-{profile}.service",
f"gateway-{profile}.service",
)
def _systemd_verified_active(profile: str, *, run: Callable[..., Any]) -> bool:
"""Return True only when systemd itself reports the profile's unit active.
This is the observation that separates ``verified`` from
``relaunch_attempted``. Any failure here (no ``systemctl``, probe error,
unit not ``active``) means we could NOT verify — never that the restart
failed.
"""
systemctl = shutil.which("systemctl")
if not systemctl:
return False
for unit in _systemd_unit_candidates(profile):
try:
result = run(
[systemctl, "--user", "is-active", unit],
capture_output=True,
text=True,
encoding="utf-8",
errors="replace",
check=False,
timeout=_VERIFY_TIMEOUT,
)
except (OSError, subprocess.TimeoutExpired):
continue
if (
getattr(result, "returncode", 1) == 0
and (getattr(result, "stdout", "") or "").strip() == "active"
):
return True
return False
def restart_profiles(
profiles: Iterable[str], *, run: Callable[..., Any] = subprocess.run
profiles: Iterable[str],
*,
supervisors: Mapping[str, str] | None = None,
run: Callable[..., Any] = subprocess.run,
) -> dict[str, list[str]]:
"""Restart the supplied profiles and return per-profile terminal results.
@@ -87,21 +152,38 @@ def restart_profiles(
supervisor. Manual gateways are intentionally excluded before this module
is called: killing one without a relaunch authority would turn stale code
into an outage.
A profile only lands in ``verified`` when its supervisor is systemd and
``systemctl --user is-active`` independently confirms the unit after the
relaunch command succeeded. Every other zero-exit relaunch is reported as
``relaunch_attempted`` — the code cannot observe supervisor coverage for
those paths and must not claim it.
"""
supervisors = supervisors or {}
normalized = sorted(
{profile for profile in profiles if isinstance(profile, str) and profile}
)
succeeded: list[str] = []
verified: list[str] = []
relaunch_attempted: list[str] = []
failed: list[str] = []
for profile in normalized:
if _run_profile_restart(profile, run=run):
succeeded.append(profile)
else:
if not _run_profile_restart(profile, run=run):
failed.append(profile)
return {"succeeded": succeeded, "failed": failed}
continue
if supervisors.get(profile) == "systemd" and _systemd_verified_active(
profile, run=run
):
verified.append(profile)
else:
relaunch_attempted.append(profile)
return {
"verified": verified,
"relaunch_attempted": relaunch_attempted,
"failed": failed,
}
def _parse_payload(stream) -> list[str]:
def _parse_payload(stream) -> tuple[list[str], dict[str, str]]:
payload = json.load(stream)
profiles = payload.get("profiles") if isinstance(payload, dict) else None
if not isinstance(profiles, list):
@@ -111,7 +193,19 @@ def _parse_payload(stream) -> list[str]:
for profile in profiles
):
raise ValueError("recovery profiles contain an invalid profile id")
return profiles
raw_supervisors = payload.get("supervisors") if isinstance(payload, dict) else None
supervisors: dict[str, str] = {}
if raw_supervisors is not None:
if not isinstance(raw_supervisors, dict) or any(
not isinstance(profile, str)
or not isinstance(supervisor, str)
or not _PROFILE_ID_RE.fullmatch(profile)
or not _SUPERVISOR_RE.fullmatch(supervisor)
for profile, supervisor in raw_supervisors.items()
):
raise ValueError("recovery supervisors map is invalid")
supervisors = dict(raw_supervisors)
return profiles, supervisors
def main(argv: list[str] | None = None) -> int:
@@ -126,10 +220,19 @@ def main(argv: list[str] | None = None) -> int:
parser.error("this command is an internal update-recovery entry point")
try:
profiles = _parse_payload(sys.stdin)
result = restart_profiles(profiles)
profiles, supervisors = _parse_payload(sys.stdin)
result = restart_profiles(profiles, supervisors=supervisors)
except (ValueError, json.JSONDecodeError) as exc:
print(json.dumps({"error": str(exc), "succeeded": [], "failed": []}))
print(
json.dumps(
{
"error": str(exc),
"verified": [],
"relaunch_attempted": [],
"failed": [],
}
)
)
return 2
print(json.dumps(result, sort_keys=True))