fix(cron): degrade gracefully when systemd user scopes are unavailable

A systemd-supervised gateway (INVOCATION_ID set) with no user D-Bus
session (containers, minimal LXCs, supervisors without linger) fails
EVERY scheduled job at dispatch: restart_safe_gateway_child_argv()
raises, run_one_job() records a failure, and the only symptom is
silently skipped executions (a missed nightly backup, dead watchdogs,
no alert).

Cron now degrades to a direct external subprocess with a
once-per-process warning instead of raising, unless
cron.require_restart_safe_scope=true (config.yaml, default false)
restores fail-closed. Degraded jobs keep process separation and the
full #101940 ownership handoff - only cgroup isolation is lost, so a
mid-job gateway restart kills the worker and the execution ledger
records exactly that.

The dispatch is a GatewayChildDispatch NamedTuple (in_process /
scoped / degraded) so the degraded case can never collapse into the
"not managed, stay in-process" sentinel - the failure mode that would
recreate the restart-interruption edge #101940 closed.

Kanban stays fail-closed (require_restart_safe_scope=True at its call
sites): its workers are long-lived agentic runs, so the degrade policy
is limited to bounded cron jobs in this PR.

Addresses the #102431 review: the env-var flag became a config key per
AGENTS.md (no new HERMES_* non-secret vars), Kanban keeps fail-closed
instead of updating its tests to a degraded contract, main's
enable-linger remedy message is preserved, and the degrade warning
fires once per process.
This commit is contained in:
Paul Robertson
2026-09-06 18:49:40 +12:00
committed by kshitij
parent 9a60a7f316
commit 560b6d2e81
6 changed files with 379 additions and 46 deletions
@@ -0,0 +1,2 @@
kiwipaulrob
# PR #102431 rework (cron scope graceful degrade)
+28 -7
View File
@@ -3050,12 +3050,17 @@ def _wait_for_external_cron_worker(
def _launch_external_cron_worker(job: dict) -> bool:
"""Launch *job* outside a managed gateway cgroup when required.
"""Launch *job* outside the managed gateway process when required.
Returns ``False`` when the caller is not a managed systemd gateway and the
existing in-process path should be used. In managed topology, failure to
establish the transient scope raises: falling back would recreate the
restart interruption this handoff exists to prevent.
existing in-process path should be used. In managed topology the job is
always handed to an external worker with the #101940 ownership handoff:
either inside a transient user scope (isolated) or - when no user D-Bus
session exists and ``cron.require_restart_safe_scope`` is false (the
default) - as a direct subprocess (process separation without cgroup
isolation). Setting ``cron.require_restart_safe_scope: true`` restores
fail-closed. Falling back to in-process in managed topology would
recreate the restart interruption this handoff exists to prevent.
"""
execution_id = str(job["execution_id"])
job_id = str(job["id"])
@@ -3085,13 +3090,29 @@ def _launch_external_cron_worker(job: dict) -> bool:
systemd_user_bus_env,
)
cfg = load_config() or {}
require_restart_safe_scope = bool(
((cfg.get("cron") or {}) if isinstance(cfg, dict) else {}).get(
"require_restart_safe_scope", False
)
)
multiplex_active = is_multiplex_active()
scoped_command = restart_safe_gateway_child_argv(
dispatch = restart_safe_gateway_child_argv(
command,
unit_suffix=f"cron-{job_id}-exec-{execution_id}",
require_restart_safe_scope=require_restart_safe_scope,
)
if scoped_command == command:
if dispatch.mode == "in_process":
# Not a managed systemd gateway: keep the existing in-process path.
return False
# "scoped" AND "degraded" both launch an external worker with the same
# #101940 ownership handoff below. Degraded only differs in isolation:
# the direct command runs in the gateway cgroup (documented in the
# warning), so a mid-job gateway restart kills it — but the execution
# ledger still records exactly what happened (failed/unknown) instead of
# the job silently never running. Never fall back to in-process here:
# that would recreate the restart-interruption edge #101940 closed.
launch_command = dispatch.argv
if mark_execution_handoff_pending(execution_id) is None:
raise RuntimeError(
@@ -3134,7 +3155,7 @@ def _launch_external_cron_worker(job: dict) -> bool:
worker_env = systemd_user_bus_env(worker_env)
try:
process = subprocess.Popen(
scoped_command,
launch_command,
cwd=str(Path(__file__).resolve().parent.parent),
env=worker_env,
stdin=subprocess.DEVNULL,
+7
View File
@@ -1705,6 +1705,13 @@ DEFAULT_CONFIG = {
# (long TTS audio, big exports) need more than 30s. Env: HERMES_CRON_MEDIA_SEND_TIMEOUT.
# Keep in sync with cron.scheduler._DEFAULT_MEDIA_SEND_TIMEOUT.
"media_send_timeout_seconds": 300,
# Restart-safety policy when the managed gateway has no user systemd session
# (containers/LXCs without linger): false (default) degrades cron jobs to a direct
# external subprocess with a once-per-process warning - process separation kept,
# cgroup isolation lost, a mid-job gateway restart kills the worker. True fails
# closed with the enable-linger remedy. The Kanban dispatcher always requires a
# scope regardless (see #102431).
"require_restart_safe_scope": False,
},
# Kanban multi-agent coordination. The dispatcher ticks every N seconds, reclaims stale claims,
# promotes dependency-satisfied todos to ready, and fires `hermes -p <assignee> chat -q ...` per
+15 -6
View File
@@ -2146,17 +2146,25 @@ def _open_worker_log(task: Task, board: Optional[str]):
def _restart_safe_worker_argv(task: Task, command: list[str]) -> list[str]:
"""Wrap a managed-gateway worker in the shared restart-safe scope."""
"""Wrap a managed-gateway worker in the shared restart-safe scope.
Kanban workers are long-lived agentic runs, so unlike bounded cron jobs
they are never dispatched in the degraded (unsupervised) mode: a managed
gateway with no user systemd session fails closed through the shared
helper's raise (remedy text included). See #102431.
"""
from tools.process_registry import restart_safe_gateway_child_argv
if task.current_run_id is None:
# Outside managed systemd this is harmless, but a managed dispatch must
# never mint an untraceable scope. Check topology through the shared
# never mint an untraceable worker. Check topology through the shared
# helper first, using a placeholder suffix that cannot be launched.
scoped = restart_safe_gateway_child_argv(
command, unit_suffix=f"kanban-{task.id}-run-missing"
dispatch = restart_safe_gateway_child_argv(
command,
unit_suffix=f"kanban-{task.id}-run-missing",
require_restart_safe_scope=True,
)
if scoped is not command:
if dispatch.mode == "scoped":
raise RuntimeError(
"cannot create restart-safe systemd scope for Kanban worker: "
"the claimed task has no current run id"
@@ -2166,7 +2174,8 @@ def _restart_safe_worker_argv(task: Task, command: list[str]) -> list[str]:
return restart_safe_gateway_child_argv(
command,
unit_suffix=f"kanban-{task.id}-run-{task.current_run_id}",
)
require_restart_safe_scope=True,
).argv
def _default_spawn(task: Task, workspace: str, *, board: Optional[str] = None) -> Optional[int]:
+228 -13
View File
@@ -78,7 +78,7 @@ def test_genuine_external_worker_crash_is_recovered_unknown(
@pytest.mark.linux_only
def test_restart_safe_gateway_child_fails_closed_without_scope(monkeypatch):
def test_restart_safe_gateway_child_fails_closed_when_required(monkeypatch):
import tools.process_registry as process_registry
monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True)
@@ -87,19 +87,73 @@ def test_restart_safe_gateway_child_fails_closed_without_scope(monkeypatch):
with pytest.raises(RuntimeError, match="systemd-run --user --scope is unavailable"):
process_registry.restart_safe_gateway_child_argv(
["python", "worker.py"], unit_suffix="cron-job-1"
["python", "worker.py"],
unit_suffix="cron-job-1",
require_restart_safe_scope=True,
)
@pytest.mark.linux_only
def test_restart_safe_gateway_child_degrades_without_scope(monkeypatch):
"""Managed gateway + no user bus degrades to a distinguishable state.
The degraded dispatch must NOT be identical to the in-process passthrough:
the scheduler launches it as an external subprocess (with the #101940
ownership handoff), never in-process. See #102431 review.
"""
import tools.process_registry as process_registry
monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True)
monkeypatch.setenv("INVOCATION_ID", "managed-service")
monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", lambda: False)
command = ["python", "worker.py"]
dispatch = process_registry.restart_safe_gateway_child_argv(
command, unit_suffix="cron-job-1", require_restart_safe_scope=False
)
assert dispatch.mode == "degraded"
assert dispatch.argv == command
assert dispatch.reason == "no-user-bus"
@pytest.mark.linux_only
def test_restart_safe_gateway_child_scoped_and_in_process_modes(monkeypatch):
import tools.process_registry as process_registry
command = ["python", "worker.py"]
# Managed + working bus -> scoped wrapper.
monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True)
monkeypatch.setenv("INVOCATION_ID", "managed-service")
monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", lambda: True)
monkeypatch.setattr(
process_registry, "_build_systemd_scope_argv",
lambda cmd, unit_suffix: ["scope", "--", *cmd],
)
scoped = process_registry.restart_safe_gateway_child_argv(
command, unit_suffix="cron-job-1", require_restart_safe_scope=False
)
assert scoped.mode == "scoped"
assert scoped.argv == ["scope", "--", "python", "worker.py"]
# Outside managed topology -> in-process passthrough (identity preserved).
monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: False)
passthrough = process_registry.restart_safe_gateway_child_argv(
command, unit_suffix="cron-job-1", require_restart_safe_scope=False
)
assert passthrough.mode == "in_process"
assert passthrough.argv is command
def test_restart_safe_gateway_child_is_unchanged_outside_managed_gateway(monkeypatch):
import tools.process_registry as process_registry
command = ["python", "worker.py"]
monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: False)
assert process_registry.restart_safe_gateway_child_argv(
command, unit_suffix="cron-job-1"
) is command
dispatch = process_registry.restart_safe_gateway_child_argv(
command, unit_suffix="cron-job-1", require_restart_safe_scope=False
)
assert dispatch.mode == "in_process"
assert dispatch.argv is command
def test_restart_safe_gateway_child_never_probes_systemd_off_linux(monkeypatch):
@@ -112,9 +166,11 @@ def test_restart_safe_gateway_child_never_probes_systemd_off_linux(monkeypatch):
monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", probe)
monkeypatch.setenv("INVOCATION_ID", "managed-service")
assert process_registry.restart_safe_gateway_child_argv(
command, unit_suffix="cron-job-1"
) is command
dispatch = process_registry.restart_safe_gateway_child_argv(
command, unit_suffix="cron-job-1", require_restart_safe_scope=False
)
assert dispatch.mode == "in_process"
assert dispatch.argv is command
probe.assert_not_called()
@@ -197,10 +253,11 @@ def test_launch_external_worker_uses_restart_safe_scope_and_acknowledges(
)
register_env_passthrough(["SERVICE_TOKEN"])
wrapped_commands = []
from tools.process_registry import GatewayChildDispatch
def wrap(command, *, unit_suffix):
def wrap(command, *, unit_suffix, require_restart_safe_scope=False):
wrapped_commands.append((command, unit_suffix))
return ["scope", "--", *command]
return GatewayChildDispatch("scoped", ["scope", "--", *command])
monkeypatch.setattr(
"tools.process_registry.restart_safe_gateway_child_argv", wrap
@@ -316,15 +373,16 @@ def test_launch_external_worker_stays_in_process_outside_managed_gateway(
monkeypatch,
):
import cron.scheduler as scheduler
from tools.process_registry import GatewayChildDispatch
command_calls = []
def unchanged(command, *, unit_suffix):
def passthrough(command, *, unit_suffix, require_restart_safe_scope=False):
command_calls.append((command, unit_suffix))
return command
return GatewayChildDispatch("in_process", command)
monkeypatch.setattr(
"tools.process_registry.restart_safe_gateway_child_argv", unchanged
"tools.process_registry.restart_safe_gateway_child_argv", passthrough
)
popen = Mock()
monkeypatch.setattr(scheduler.subprocess, "Popen", popen)
@@ -336,6 +394,163 @@ def test_launch_external_worker_stays_in_process_outside_managed_gateway(
popen.assert_not_called()
def test_launch_external_worker_dispatches_degraded_case_as_external_subprocess(
tmp_path, monkeypatch,
):
"""Managed gateway + no user bus: the job MUST still Popen externally.
Regression for the #102431 review blocker: the degraded dispatch must
take the external-worker path (payload written, handoff fenced, Popen
called with the direct command) — never silently fall back in-process.
"""
import cron.scheduler as scheduler
from tools.process_registry import GatewayChildDispatch
job = {"id": "job-1", "execution_id": "exec-1", "prompt": "work"}
monkeypatch.setattr(scheduler, "_get_hermes_home", lambda: tmp_path)
def degraded(command, *, unit_suffix, require_restart_safe_scope=False):
assert unit_suffix == "cron-job-1-exec-exec-1"
return GatewayChildDispatch("degraded", command, "no-user-bus")
monkeypatch.setattr(
"tools.process_registry.restart_safe_gateway_child_argv", degraded
)
class FakeProcess:
returncode = None
def poll(self):
return self.returncode
def wait(self, timeout=None):
if self.returncode is None:
raise subprocess.TimeoutExpired(cmd="worker", timeout=timeout)
return self.returncode
spawned = []
def popen(command, **kwargs):
spawned.append((command, kwargs))
assert command[0:2] == [sys.executable, "-m"], command[:3]
assert "--external-worker-file" in command
payload_index = command.index("--external-worker-file") + 1
payload = json.loads(Path(command[payload_index]).read_text())
assert payload["job"]["id"] == "job-1"
ack_index = command.index("--ack-file") + 1
Path(command[ack_index]).write_text(
json.dumps({"pid": 4321, "execution_id": "exec-1"}),
encoding="utf-8",
)
return FakeProcess()
handoff = Mock(return_value={"id": "exec-1", "handoff_pending": 1})
monkeypatch.setattr(scheduler, "mark_execution_handoff_pending", handoff)
monkeypatch.setattr(scheduler.subprocess, "Popen", popen)
observed_statuses = iter(
[
{"id": "exec-1", "status": "running"},
{"id": "exec-1", "status": "completed"},
]
)
get = Mock(side_effect=lambda _execution_id: next(observed_statuses))
monkeypatch.setattr(scheduler, "get_execution", get)
assert scheduler._launch_external_cron_worker(job) is True
# Direct command, NOT a systemd-run wrapper — but still an external Popen.
assert "systemd-run" not in " ".join(spawned[0][0])
assert spawned[0][1]["start_new_session"] is True
handoff.assert_called_once_with("exec-1")
assert not (tmp_path / "cron/external-workers/exec-1.json").exists()
def test_launch_external_worker_fails_closed_when_config_requires_scope(monkeypatch):
"""cron.require_restart_safe_scope=true restores the fail-closed raise
through the real helper (config plumbing, not a stubbed dispatch)."""
import cron.scheduler as scheduler
import tools.process_registry as process_registry
monkeypatch.setattr(
scheduler, "load_config",
lambda: {"cron": {"require_restart_safe_scope": True}},
)
monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True)
monkeypatch.setenv("INVOCATION_ID", "managed-service")
monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", lambda: False)
popen = Mock()
monkeypatch.setattr(scheduler.subprocess, "Popen", popen)
with pytest.raises(RuntimeError, match="systemd-run --user --scope is unavailable"):
scheduler._launch_external_cron_worker(
{"id": "job-1", "execution_id": "exec-1", "prompt": "work"}
)
popen.assert_not_called()
def test_launch_external_worker_degrades_by_default_with_real_helper(
tmp_path, monkeypatch,
):
"""Managed gateway + no bus + default config (key absent): the real helper
degrades and the job still Popens externally with the #101940 handoff.
Exercises the config default end-to-end (cron.require_restart_safe_scope
defaults to false), not a stubbed dispatch.
"""
import cron.scheduler as scheduler
import tools.process_registry as process_registry
job = {"id": "job-1", "execution_id": "exec-1", "prompt": "work"}
monkeypatch.setattr(scheduler, "_get_hermes_home", lambda: tmp_path)
monkeypatch.setattr(scheduler, "load_config", lambda: {})
monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True)
monkeypatch.setenv("INVOCATION_ID", "managed-service")
monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", lambda: False)
class FakeProcess:
returncode = None
def poll(self):
return self.returncode
def wait(self, timeout=None):
if self.returncode is None:
raise subprocess.TimeoutExpired(cmd="worker", timeout=timeout)
return self.returncode
spawned = []
def popen(command, **kwargs):
spawned.append((command, kwargs))
payload_index = command.index("--external-worker-file") + 1
payload = json.loads(Path(command[payload_index]).read_text())
assert payload["job"]["id"] == "job-1"
ack_index = command.index("--ack-file") + 1
Path(command[ack_index]).write_text(
json.dumps({"pid": 4321, "execution_id": "exec-1"}),
encoding="utf-8",
)
return FakeProcess()
handoff = Mock(return_value={"id": "exec-1", "handoff_pending": 1})
monkeypatch.setattr(scheduler, "mark_execution_handoff_pending", handoff)
monkeypatch.setattr(scheduler.subprocess, "Popen", popen)
observed_statuses = iter(
[
{"id": "exec-1", "status": "running"},
{"id": "exec-1", "status": "completed"},
]
)
get = Mock(side_effect=lambda _execution_id: next(observed_statuses))
monkeypatch.setattr(scheduler, "get_execution", get)
assert scheduler._launch_external_cron_worker(job) is True
# Direct command, NOT a systemd-run wrapper — but still an external Popen.
assert "systemd-run" not in " ".join(spawned[0][0])
assert spawned[0][1]["start_new_session"] is True
handoff.assert_called_once_with("exec-1")
assert not (tmp_path / "cron/external-workers/exec-1.json").exists()
def test_shared_run_path_hands_gateway_fire_to_external_worker(monkeypatch):
import cron.scheduler as scheduler
+99 -20
View File
@@ -27,7 +27,7 @@ _IS_LINUX = platform.system() == "Linux"
from tools.environments.local import _find_shell, _resolve_safe_cwd, _sanitize_subprocess_env
from hermes_cli._subprocess_compat import windows_hide_flags
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Literal, NamedTuple, Optional
from hermes_cli.config import get_hermes_home
@@ -291,36 +291,115 @@ def _build_systemd_scope_argv(shell_argv: List[str], unit_suffix: str) -> List[s
return _systemd_scope_argv(binary, f"hermes-worker-{unit_suffix}", *shell_argv)
# --- restart-safe gateway child dispatch --------------------------------------
# A systemd-supervised gateway restart kills every process in the service cgroup, so
# children that must survive it are launched outside that cgroup via a transient user
# scope (systemd-run --user --scope) when one can be created. Hosts with no user
# systemd session at all (containers, LXCs without linger) cannot create scopes; the
# callers set policy explicitly through ``require_restart_safe_scope`` (cron reads
# ``cron.require_restart_safe_scope`` from config.yaml; kanban always requires a scope).
_scope_degraded_warned = False
def _warn_scope_degraded_once(unit_suffix: str) -> None:
"""Emit the degrade warning once per process.
The scope-availability verdict is cached, so without this guard the warning
would fire on every dispatch (every cron fire on a bus-less host).
"""
global _scope_degraded_warned
if _scope_degraded_warned:
return
_scope_degraded_warned = True
logger.warning(
"%s: systemd-run --user --scope is unavailable (no user D-Bus session at "
"/run/user/%d/bus); dispatching the gateway child as a direct external "
"subprocess without restart-safe cgroup isolation. The job still runs "
"outside the gateway process, but will be killed if the gateway restarts "
"mid-job. Remediate with `sudo loginctl enable-linger <gateway-user>` (plus "
"XDG_RUNTIME_DIR/DBUS_SESSION_BUS_ADDRESS in the service unit), or set "
"cron.require_restart_safe_scope=true in config.yaml to fail closed instead.",
unit_suffix, os.getuid(),
)
class GatewayChildDispatch(NamedTuple):
"""How a managed-gateway child should be launched.
Three mutually exclusive topologies - the whole point of this type is
that (1) and (3) must never collapse into the same value:
- ``"in_process"`` - not a managed systemd gateway (standalone process,
non-systemd supervisor, non-Linux host). The caller keeps its existing
in-process path. ``argv is command`` holds, preserving the historical
passthrough contract.
- ``"scoped"`` - managed gateway with a working user bus. ``argv`` is the
``systemd-run --user --scope`` wrapper; the caller launches it as an
external worker with the #101940 ownership handoff.
- ``"degraded"`` - managed gateway WITHOUT a user bus. ``argv`` is the
direct command, but the caller MUST still launch it as an external
subprocess (same ownership handoff as ``"scoped"``) - never fall back
to the in-process path, which would recreate the restart-interruption
edge #101940 closed. Isolation is lost but process separation is kept.
"""
mode: Literal["in_process", "scoped", "degraded"]
argv: List[str]
reason: str = ""
def restart_safe_gateway_child_argv(
command: List[str], *, unit_suffix: str
) -> List[str]:
command: List[str], *, unit_suffix: str, require_restart_safe_scope: bool,
) -> GatewayChildDispatch:
"""Place a managed-systemd gateway child outside the gateway cgroup.
Returns a :class:`GatewayChildDispatch` distinguishing three topologies -
never the bare command list, so callers cannot mistake a degraded dispatch
for "not managed, stay in-process".
Children that must survive an intentional gateway restart cannot rely on
``start_new_session`` alone: systemd still kills every process in the
service cgroup. In that topology, require a transient user scope and fail
closed if it cannot be established. Standalone processes, non-systemd
supervisors, and non-Linux hosts retain the direct command.
service cgroup. In that topology, prefer a transient user scope.
When a user systemd session is genuinely absent (containers, minimal LXCs,
macOS-style supervisors) the scope cannot be created, but hard failing takes
down every scheduled job on the host - a silent cron outage with no
operator-visible symptom beyond skipped executions (the #101940 durability
contract covers the restart case, not the never-had-a-bus case). Callers
therefore state their policy explicitly: ``require_restart_safe_scope=True``
raises when no scope can be established (fail-closed, the kanban contract);
``False`` degrades to a direct external subprocess with a once-per-process
warning (the cron default behind ``cron.require_restart_safe_scope``).
Standalone processes, non-systemd supervisors, and non-Linux hosts return
``mode == "in_process"`` - the caller keeps its existing in-process path.
"""
if not _IS_LINUX:
return command
return GatewayChildDispatch("in_process", command)
if not _is_supervised_gateway_process() or not os.environ.get("INVOCATION_ID"):
return command
return GatewayChildDispatch("in_process", command)
if not _systemd_run_user_scope_available():
# Stored as the cron execution's error and shown on the job row: name the remedy.
raise RuntimeError(
"cannot create restart-safe systemd scope for gateway child: "
"systemd-run --user --scope is unavailable (usually no reachable user D-Bus session at "
f"/run/user/{os.getuid()}/bus). On a system-level service install, run " # windows-footgun: ok — behind the _IS_LINUX return above
"`sudo loginctl enable-linger <gateway-user>` and restart the gateway."
)
if require_restart_safe_scope:
# Stored as the cron execution's error and shown on the job row: name the remedy.
raise RuntimeError(
"cannot create restart-safe systemd scope for gateway child: "
"systemd-run --user --scope is unavailable (usually no reachable user D-Bus session at "
f"/run/user/{os.getuid()}/bus). On a system-level service install, run " # windows-footgun: ok — behind the _IS_LINUX return above
"`sudo loginctl enable-linger <gateway-user>` and restart the gateway."
)
_warn_scope_degraded_once(unit_suffix)
return GatewayChildDispatch("degraded", command, "no-user-bus")
scoped = _build_systemd_scope_argv(command, unit_suffix=unit_suffix)
if scoped == command:
raise RuntimeError(
"cannot create restart-safe systemd scope for gateway child: "
"systemd-run disappeared after the availability probe"
)
return scoped
if require_restart_safe_scope:
raise RuntimeError(
"cannot create restart-safe systemd scope for gateway child: "
"systemd-run disappeared after the availability probe"
)
_warn_scope_degraded_once(unit_suffix)
return GatewayChildDispatch("degraded", command, "scope-binary-vanished")
return GatewayChildDispatch("scoped", scoped)
def _stop_systemd_unit(unit_name: str) -> bool: