fix(gateway): preserve systemd handoff recovery
This commit is contained in:
+47
-7
@@ -1245,13 +1245,15 @@ def _wait_for_systemd_service_restart(
|
||||
*,
|
||||
system: bool = False,
|
||||
previous_pid: int | None = None,
|
||||
timeout: float = 60.0,
|
||||
timeout: float | None = None,
|
||||
) -> bool:
|
||||
"""Wait for the gateway service to become active after a restart handoff."""
|
||||
import time
|
||||
|
||||
svc = get_service_name()
|
||||
scope_label = _service_scope_label(system).capitalize()
|
||||
if timeout is None:
|
||||
timeout = _systemd_restart_wait_timeout(system=system)
|
||||
deadline = time.monotonic() + timeout
|
||||
printed_runtime_wait = False
|
||||
|
||||
@@ -1308,6 +1310,25 @@ def _wait_for_systemd_service_restart(
|
||||
return False
|
||||
|
||||
|
||||
def _systemd_restart_wait_timeout(system: bool = False) -> float:
|
||||
"""Cover systemd's relaunch delays before applying the runtime wait floor."""
|
||||
from gateway.shutdown_forensics import _parse_systemd_duration_to_us
|
||||
|
||||
props = _read_systemd_unit_properties(
|
||||
system=system,
|
||||
properties=("RestartUSec", "TimeoutStartUSec"),
|
||||
)
|
||||
supervisor_budget = 0.0
|
||||
for name in ("RestartUSec", "TimeoutStartUSec"):
|
||||
raw = props.get(name, "")
|
||||
duration_us = (
|
||||
int(raw) if raw.isdigit() else _parse_systemd_duration_to_us(raw)
|
||||
)
|
||||
if duration_us is not None:
|
||||
supervisor_budget += duration_us / 1_000_000
|
||||
return 60.0 + supervisor_budget
|
||||
|
||||
|
||||
def _systemd_unit_is_start_limited(props: dict[str, str]) -> bool:
|
||||
result = props.get("Result", "").lower()
|
||||
sub_state = props.get("SubState", "").lower()
|
||||
@@ -4124,13 +4145,32 @@ def systemd_restart(system: bool = False):
|
||||
# Exit 75 transfers restart ownership to systemd. Observe that
|
||||
# single replacement instead of issuing another restart that can
|
||||
# stop the process systemd has already brought up.
|
||||
_wait_for_systemd_service_restart(system=system, previous_pid=pid)
|
||||
return
|
||||
if _wait_for_systemd_service_restart(system=system, previous_pid=pid):
|
||||
return
|
||||
if _systemd_service_is_start_limited(system=system):
|
||||
return
|
||||
|
||||
# A replacement may have started but not reached gateway runtime
|
||||
# readiness before the wait expired. Never stop that generation.
|
||||
props = _read_systemd_unit_properties(system=system)
|
||||
replacement_pid = _systemd_main_pid_from_props(props)
|
||||
if (
|
||||
props.get("ActiveState") in {"active", "activating", "reloading"}
|
||||
or props.get("SubState") == "auto-restart"
|
||||
or (replacement_pid is not None and replacement_pid != pid)
|
||||
):
|
||||
return
|
||||
|
||||
print(
|
||||
"⚠ Systemd did not relaunch the gateway after its graceful exit; "
|
||||
"forcing a service restart..."
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f"⚠ Graceful restart did not complete within {int(wait_budget)}s; "
|
||||
"forcing a service restart..."
|
||||
)
|
||||
|
||||
print(
|
||||
f"⚠ Graceful restart did not complete within {int(wait_budget)}s; "
|
||||
"forcing a service restart..."
|
||||
)
|
||||
_run_systemctl(
|
||||
["reset-failed", svc],
|
||||
system=system,
|
||||
|
||||
Reference in New Issue
Block a user