diff --git a/gateway/lifecycle_ledger.py b/gateway/lifecycle_ledger.py index 9c0732a7aa..a92968cbd5 100644 --- a/gateway/lifecycle_ledger.py +++ b/gateway/lifecycle_ledger.py @@ -104,16 +104,17 @@ def _append_exit_diag(record: Dict[str, Any], home: Optional[Path]) -> None: logger.debug("Failed to append unclean-exit record", exc_info=True) -def _pid_is_sentinel_owner(pid: Any, create_time: Any) -> bool: +def _pid_is_sentinel_owner(pid: Any, start_time: Any, create_time: Any) -> bool: """True when ``pid`` is a live process that is the sentinel's incarnation — guards the ``--replace`` race: a live matching owner mid-teardown is a handover, not a death. Identity is the psutil ``create_time`` the sentinel stamps at claim (epoch seconds, same - producer on both sides). The sentinel's ``start_time`` is the ledger claim time and is NOT comparable with - ``gateway.status.get_process_start_time`` (proc ticks on Linux, centiseconds elsewhere) — the - old comparison never matched, so every ``--replace`` handover read as an unclean death. A - sentinel without ``create_time`` (pre-stamp gateway) cannot disambiguate PID reuse; a live PID - is taken as the owner, matching the psutil-silent case below.""" + producer on both sides). The sentinel's ``start_time`` is the ledger claim time and is NOT + comparable with ``gateway.status.get_process_start_time`` (proc ticks on Linux, centiseconds + elsewhere) — the old comparison never matched, so every ``--replace`` handover read as an + unclean death. A pre-stamp sentinel (no ``create_time``) still has ``start_time`` in epoch + seconds: the owner was born BEFORE it claimed, a PID reuser AFTER the owner died, so a birth + later than the claim is a reuser.""" try: pid_int = int(pid) # NOT os.kill(pid, 0): on Windows that sends CTRL_C_EVENT to the target's console group. @@ -123,12 +124,16 @@ def _pid_is_sentinel_owner(pid: Any, create_time: Any) -> bool: return False except Exception: return False - if type(create_time) not in (int, float): - return True from hermes_cli.process_identity import _process_create_time actual = _process_create_time(pid_int) - return actual is None or abs(actual - float(create_time)) <= 2.0 # None: can't disambiguate PID reuse + if actual is None: + return True # can't disambiguate PID reuse + if type(create_time) in (int, float): + return abs(actual - float(create_time)) <= 2.0 + if type(start_time) in (int, float): + return actual <= float(start_time) + 2.0 + return True def _suspected_oom(mem: Dict[str, Any]) -> bool: @@ -149,7 +154,7 @@ def detect_unclean_exit(home: Optional[Path] = None) -> Optional[Dict[str, Any]] sentinel = _read_json(get_lifecycle_sentinel_path(home)) if not sentinel or sentinel.get("phase") != "running": return None - if _pid_is_sentinel_owner(sentinel.get("pid"), sentinel.get("create_time")): + if _pid_is_sentinel_owner(sentinel.get("pid"), sentinel.get("start_time"), sentinel.get("create_time")): return None # live owner — planned takeover in flight, not a death evidence: Dict[str, Any] = { "prior_pid": sentinel.get("pid"), "prior_started_at": sentinel.get("started_at"), diff --git a/tests/gateway/test_lifecycle_ledger.py b/tests/gateway/test_lifecycle_ledger.py index 4796d77be6..54127a6928 100644 --- a/tests/gateway/test_lifecycle_ledger.py +++ b/tests/gateway/test_lifecycle_ledger.py @@ -235,9 +235,6 @@ def test_replace_handover_is_not_a_death_and_pid_reuse_is(tmp_path: Path, monkey (same producer, epoch seconds). The ledger's ``start_time`` (claim time) is never compared with ``get_process_start_time`` (proc ticks / centiseconds): that comparison could not match, so a ``--replace`` handover was reported as an unclean death.""" - from gateway import lifecycle_ledger - - monkeypatch.setattr(lifecycle_ledger, "_pid_exists", lambda pid: True, raising=False) monkeypatch.setattr("gateway.status._pid_exists", lambda pid: True) monkeypatch.setattr("hermes_cli.process_identity._process_create_time", lambda pid=None: 5000.0) live = {"phase": "running", "pid": 4242, "start_time": 5003.7, "started_at": "x"} @@ -248,5 +245,8 @@ def test_replace_handover_is_not_a_death_and_pid_reuse_is(tmp_path: Path, monkey _write_sentinel(tmp_path, {**live, "create_time": 4000.0}) assert detect_unclean_exit(home=tmp_path) is not None # PID reused by another process: death - _write_sentinel(tmp_path, live) # pre-stamp sentinel: a live PID is taken as the owner + # Pre-stamp sentinel (no create_time): the owner was born before it claimed; a reuser after. + _write_sentinel(tmp_path, live) # birth 5000.0 <= claim 5003.7 → owner assert detect_unclean_exit(home=tmp_path) is None + _write_sentinel(tmp_path, {**live, "start_time": 4000.0}) # born after the claim → reuser → death + assert detect_unclean_exit(home=tmp_path) is not None