fix(gateway): preserve systemd handoff recovery

This commit is contained in:
fangliquan
2026-08-22 07:51:54 +08:00
committed by kshitij
parent 5b024c7ccc
commit 91fb175188
4 changed files with 146 additions and 9 deletions
+47 -7
View File
@@ -1245,13 +1245,15 @@ def _wait_for_systemd_service_restart(
*,
system: bool = False,
previous_pid: int | None = None,
timeout: float = 60.0,
timeout: float | None = None,
) -> bool:
"""Wait for the gateway service to become active after a restart handoff."""
import time
svc = get_service_name()
scope_label = _service_scope_label(system).capitalize()
if timeout is None:
timeout = _systemd_restart_wait_timeout(system=system)
deadline = time.monotonic() + timeout
printed_runtime_wait = False
@@ -1308,6 +1310,25 @@ def _wait_for_systemd_service_restart(
return False
def _systemd_restart_wait_timeout(system: bool = False) -> float:
"""Cover systemd's relaunch delays before applying the runtime wait floor."""
from gateway.shutdown_forensics import _parse_systemd_duration_to_us
props = _read_systemd_unit_properties(
system=system,
properties=("RestartUSec", "TimeoutStartUSec"),
)
supervisor_budget = 0.0
for name in ("RestartUSec", "TimeoutStartUSec"):
raw = props.get(name, "")
duration_us = (
int(raw) if raw.isdigit() else _parse_systemd_duration_to_us(raw)
)
if duration_us is not None:
supervisor_budget += duration_us / 1_000_000
return 60.0 + supervisor_budget
def _systemd_unit_is_start_limited(props: dict[str, str]) -> bool:
result = props.get("Result", "").lower()
sub_state = props.get("SubState", "").lower()
@@ -4124,13 +4145,32 @@ def systemd_restart(system: bool = False):
# Exit 75 transfers restart ownership to systemd. Observe that
# single replacement instead of issuing another restart that can
# stop the process systemd has already brought up.
_wait_for_systemd_service_restart(system=system, previous_pid=pid)
return
if _wait_for_systemd_service_restart(system=system, previous_pid=pid):
return
if _systemd_service_is_start_limited(system=system):
return
# A replacement may have started but not reached gateway runtime
# readiness before the wait expired. Never stop that generation.
props = _read_systemd_unit_properties(system=system)
replacement_pid = _systemd_main_pid_from_props(props)
if (
props.get("ActiveState") in {"active", "activating", "reloading"}
or props.get("SubState") == "auto-restart"
or (replacement_pid is not None and replacement_pid != pid)
):
return
print(
"⚠ Systemd did not relaunch the gateway after its graceful exit; "
"forcing a service restart..."
)
else:
print(
f"⚠ Graceful restart did not complete within {int(wait_budget)}s; "
"forcing a service restart..."
)
print(
f"⚠ Graceful restart did not complete within {int(wait_budget)}s; "
"forcing a service restart..."
)
_run_systemctl(
["reset-failed", svc],
system=system,