"""Shutdown forensics — capture context when the gateway receives SIGTERM/SIGINT. ``shutdown_signal_handler`` runs synchronously inside the asyncio loop, so :func:`snapshot_shutdown_context` is a fast (<10ms) non-blocking probe it can log immediately, and :func:`spawn_async_diagnostic` is a fire-and-forget ``ps`` walk in a detached subprocess so it can't block teardown even if /proc is wedged. Anything that waits belongs in the async helper, never in the synchronous probe. """ from __future__ import annotations import json import os import signal import subprocess import sys import time from pathlib import Path from typing import Any, Dict, List, Optional from gateway.restart import DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, resolve_systemd_timeout_stop_sec import contextlib _SIGNAL_NAME_BY_NUM: Dict[int, str] = { int(getattr(signal, _name)): _name for _name in ("SIGTERM", "SIGINT", "SIGHUP", "SIGQUIT", "SIGUSR1", "SIGUSR2") if getattr(signal, _name, None) is not None } def _signal_name(sig: Any) -> str: """Return a human-readable signal name (or ``str(sig)`` as fallback).""" if sig is None: return "UNKNOWN" try: sig_int = int(sig) except (TypeError, ValueError): return str(sig) return _SIGNAL_NAME_BY_NUM.get(sig_int, f"signal#{sig_int}") def _read_proc_field(pid: int, key: str) -> Optional[str]: """Read a single field from /proc//status. Linux only; None elsewhere.""" try: with open(f"/proc/{pid}/status", encoding="utf-8") as fh: for line in fh: if line.startswith(key + ":"): return line.split(":", 1)[1].strip() except (FileNotFoundError, PermissionError, OSError): pass return None def _read_proc_cmdline(pid: int) -> Optional[str]: """Read /proc//cmdline (NUL-separated) as a printable string. Linux only.""" try: with open(f"/proc/{pid}/cmdline", "rb") as fh: data = fh.read() except (FileNotFoundError, PermissionError, OSError): return None if not data: return None return data.replace(b"\x00", b" ").decode("utf-8", errors="replace").strip() def _proc_summary(pid: int) -> Dict[str, Any]: """Compact /proc/ snapshot (pid, ppid, state, uid, cmdline); missing fields omitted.""" summary: Dict[str, Any] = {"pid": pid} if pid <= 0: return summary for out_key, proc_key in (("name", "Name"), ("state", "State")): value = _read_proc_field(pid, proc_key) if value is not None: summary[out_key] = value ppid = _read_proc_field(pid, "PPid") if ppid is not None: with contextlib.suppress(ValueError): summary["ppid"] = int(ppid) uid = _read_proc_field(pid, "Uid") if uid is not None: # "real effective saved fs" summary["uid"] = uid.split()[0] if uid else uid cmdline = _read_proc_cmdline(pid) if cmdline: # Truncate aggressively — these can be 4KB summary["cmdline"] = cmdline[:300] return summary def _read_marker(path: Path) -> Optional[str]: """Return the marker file's text, or None if absent/unreadable.""" if not path.exists(): return None try: return path.read_text(encoding="utf-8") except OSError: return None def snapshot_shutdown_context(received_signal: Any = None) -> Dict[str, Any]: """Fast (<10ms) snapshot of who/what is asking us to shut down. Signal name/number, own + parent /proc summaries, systemd parentage, takeover/planned-stop markers, TracerPid, 1-min load, wall/monotonic timestamps. Pure stdlib, never raises, never blocks on subprocesses. """ pid = os.getpid() ppid = os.getppid() ctx: Dict[str, Any] = { "ts": time.time(), "ts_monotonic": time.monotonic(), "signal": _signal_name(received_signal), "signal_num": int(received_signal) if received_signal is not None else None, "pid": pid, "ppid": ppid, "parent": _proc_summary(ppid), "self": _proc_summary(pid), } # INVOCATION_ID is set by systemd units; ppid==1 also strongly suggests # systemd reaped+forwarded the SIGTERM. invocation_id = os.environ.get("INVOCATION_ID") if invocation_id: ctx["systemd_invocation_id"] = invocation_id journal_stream = os.environ.get("JOURNAL_STREAM") if journal_stream: ctx["systemd_journal_stream"] = journal_stream ctx["under_systemd"] = bool(invocation_id) or ppid == 1 # High load points at "something crushing the box" rather than an external killer. with contextlib.suppress(OSError, AttributeError): ctx["loadavg_1m"] = os.getloadavg()[0] # Nonzero TracerPid means a debugger/strace is attached. try: tracer = _read_proc_field(pid, "TracerPid") if tracer is not None and tracer != "0": ctx["tracer_pid"] = int(tracer) if tracer.isdigit() else tracer ctx["tracer"] = _proc_summary(int(tracer)) if tracer.isdigit() else None except (TypeError, ValueError): pass # Race hint: a takeover marker on disk that does NOT name us is a smoking # gun for "another --replace instance is killing us". Filenames mirror # gateway.status; literals keep the signal-handler path import-light. try: hermes_home_str = os.environ.get("HERMES_HOME") if hermes_home_str: raw = _read_marker(Path(hermes_home_str) / ".gateway-takeover.json") if raw is not None: ctx["takeover_marker"] = raw[:300] ctx["takeover_marker_for_self"] = ( f'"target_pid": {pid}' in raw or f"'target_pid': {pid}" in raw ) raw = _read_marker(Path(hermes_home_str) / ".gateway-planned-stop.json") if raw is not None: ctx["planned_stop_marker"] = raw[:300] except Exception: # noqa: BLE001 — never raise from a signal handler pass return ctx def spawn_async_diagnostic( log_path: Path, signal_name: str, *, timeout_seconds: float = 5.0, ) -> Optional[int]: """Fire-and-forget ``ps``-style snapshot appended to ``log_path``. A detached subprocess (own ``timeout`` so a wedged ``ps`` self-cleans) rather than a blocking ``ps aux`` in the signal handler, which can freeze the loop >2s on a busy host. Returns the subprocess PID, or ``None`` on failure / Windows. """ try: log_path.parent.mkdir(parents=True, exist_ok=True) except OSError: return None # bash -c is available on every POSIX target; Windows has no ps anyway. if sys.platform == "win32": return None script = ( f"echo '=== shutdown diagnostic @ {signal_name} ==='; " "echo '--- date ---'; date -u +%Y-%m-%dT%H:%M:%SZ; " "echo '--- ps auxf (top 60 by cpu) ---'; " "ps auxf --sort=-pcpu 2>/dev/null | head -60; " "echo '--- pstree of self ---'; " f"pstree -plau {os.getpid()} 2>/dev/null | head -40 || true; " "echo '--- /proc/loadavg ---'; " "cat /proc/loadavg 2>/dev/null || true; " "echo '--- recent dmesg (oom/killed) ---'; " "dmesg -T 2>/dev/null | tail -20 || journalctl --user -n 20 --no-pager 2>/dev/null | tail -20 || true; " "echo '=== end ==='" ) try: # O_APPEND so concurrent diagnostics from rapid signals don't trample each other. fd = os.open(str(log_path), os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o644) except OSError: return None try: # start_new_session: survive systemd killing our cgroup (KillMode=control-group) # long enough to flush. proc = subprocess.Popen( ["timeout", f"{timeout_seconds:.0f}", "bash", "-c", script], stdout=fd, stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, start_new_session=True, close_fds=True, ) except (FileNotFoundError, OSError): with contextlib.suppress(OSError): os.close(fd) return None finally: # Subprocess inherited the fd; we can drop our handle. with contextlib.suppress(OSError): os.close(fd) return proc.pid def format_context_for_log(ctx: Dict[str, Any]) -> str: """Render a shutdown context dict as a single, scannable log line.""" sig = ctx.get("signal", "?") parent = ctx.get("parent") or {} parent_cmd = parent.get("cmdline", "(unknown)") parent_name = parent.get("name") or "?" parent_pid = parent.get("pid") or "?" under_systemd = "yes" if ctx.get("under_systemd") else "no" load = ctx.get("loadavg_1m") load_str = f"{load:.2f}" if isinstance(load, (int, float)) else "?" extras: List[str] = [] if ctx.get("takeover_marker") is not None: for_self = ctx.get("takeover_marker_for_self") extras.append(f"takeover_marker_present={'self' if for_self else 'other'}") if ctx.get("planned_stop_marker") is not None: extras.append("planned_stop_marker_present=yes") if ctx.get("tracer_pid"): extras.append(f"tracer_pid={ctx['tracer_pid']}") extras_str = (" " + " ".join(extras)) if extras else "" # Parent cmdline is the most useful single signal — log it prominently. return ( f"signal={sig} under_systemd={under_systemd} parent_pid={parent_pid} " f"parent_name={parent_name} loadavg_1m={load_str}{extras_str} " f"parent_cmdline={parent_cmd!r}" ) def context_as_json(ctx: Dict[str, Any]) -> str: """JSON-serialise a context dict for structured ingestion. Never raises.""" try: return json.dumps(ctx, default=str, sort_keys=True) except (TypeError, ValueError): return "{}" def check_systemd_timing_alignment( drain_timeout: float, cron_drain_timeout: float = DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, ) -> Optional[Dict[str, Any]]: """At startup, sanity-check that systemd's TimeoutStopSec covers stop. A stale unit file (upgraded without re-running ``hermes setup``) can have ``TimeoutStopSec`` below the stop budget, so systemd SIGKILLs the cgroup mid-drain — a phantom ``code=killed status=9`` in the journal. Returns ``None`` when aligned OR undeterminable (not under systemd, no ``systemctl``); otherwise a dict with ``timeout_stop_sec``/``drain_timeout``/``expected_min``/``mismatch``. """ invocation_id = os.environ.get("INVOCATION_ID") if not invocation_id: return None # Not running under systemd (or at least not directly) # /proc/self/cgroup: "0::/user.slice/.../hermes-gateway.service" unit_name: Optional[str] = None try: with open("/proc/self/cgroup", encoding="utf-8") as fh: for line in fh: unit_name = next( (p for p in reversed(line.strip().split("/")) if p.endswith(".service")), None ) if unit_name: break except (OSError, FileNotFoundError): pass if not unit_name: return None # Try --user first (the common case for hermes), then the system manager. timeout_us: Optional[int] = None for flag in (["--user"], []): try: result = subprocess.run( ["systemctl", *flag, "show", unit_name, "--property=TimeoutStopUSec"], capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=2.0, ) except (FileNotFoundError, subprocess.TimeoutExpired, OSError): continue if result.returncode != 0: continue # Output: "TimeoutStopUSec=1min 30s" or "TimeoutStopUSec=90000000" for line in result.stdout.splitlines(): if line.startswith("TimeoutStopUSec="): value = line.split("=", 1)[1].strip() timeout_us = ( int(value) if value.isdigit() else parse_systemd_duration_to_us(value) ) if timeout_us is not None: break if timeout_us is not None: break if timeout_us is None: return None timeout_stop_sec = timeout_us / 1_000_000.0 expected = float(resolve_systemd_timeout_stop_sec(drain_timeout, cron_drain_timeout)) return { "unit": unit_name, "timeout_stop_sec": timeout_stop_sec, "drain_timeout": drain_timeout, "cron_drain_timeout": cron_drain_timeout, "expected_min": expected, "mismatch": timeout_stop_sec < expected, } def parse_systemd_duration_to_us(raw: str) -> Optional[int]: """Parse 'TimeoutStopUSec=1min 30s' / '90s' style values to microseconds. Covers the common units (us, ms, s, min, h); a bare number is seconds. Returns None on anything unexpected. Never raises. Public: also consumed by hermes_cli.gateway's restart-wait sizing. """ if not raw: return None units = { "us": 1, "ms": 1_000, "s": 1_000_000, "sec": 1_000_000, "min": 60_000_000, "h": 3_600_000_000, "hr": 3_600_000_000, } total_us = 0 token = "" digits = "" def _flush() -> bool: """Fold the pending ``digits``/``token`` pair into ``total_us``.""" nonlocal total_us, token, digits multiplier = units.get(token.lower()) if token else 1_000_000 if multiplier is None or not digits: return False try: total_us += int(float(digits) * multiplier) except ValueError: return False digits = "" token = "" return True for ch in raw + " ": if ch.isdigit() or ch == ".": # A digit after a unit ends the previous number if token and not _flush(): return None digits += ch elif ch.isalpha(): token += ch elif digits and not _flush(): return None return total_us if total_us > 0 else None # Backward-compat private alias (pre-promotion name). _parse_systemd_duration_to_us = parse_systemd_duration_to_us