fix(cron): degrade gracefully when systemd user scopes are unavailable
A systemd-supervised gateway (INVOCATION_ID set) with no user D-Bus session (containers, minimal LXCs, supervisors without linger) fails EVERY scheduled job at dispatch: restart_safe_gateway_child_argv() raises, run_one_job() records a failure, and the only symptom is silently skipped executions (a missed nightly backup, dead watchdogs, no alert). Cron now degrades to a direct external subprocess with a once-per-process warning instead of raising, unless cron.require_restart_safe_scope=true (config.yaml, default false) restores fail-closed. Degraded jobs keep process separation and the full #101940 ownership handoff - only cgroup isolation is lost, so a mid-job gateway restart kills the worker and the execution ledger records exactly that. The dispatch is a GatewayChildDispatch NamedTuple (in_process / scoped / degraded) so the degraded case can never collapse into the "not managed, stay in-process" sentinel - the failure mode that would recreate the restart-interruption edge #101940 closed. Kanban stays fail-closed (require_restart_safe_scope=True at its call sites): its workers are long-lived agentic runs, so the degrade policy is limited to bounded cron jobs in this PR. Addresses the #102431 review: the env-var flag became a config key per AGENTS.md (no new HERMES_* non-secret vars), Kanban keeps fail-closed instead of updating its tests to a degraded contract, main's enable-linger remedy message is preserved, and the degrade warning fires once per process.
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
kiwipaulrob
|
||||
# PR #102431 rework (cron scope graceful degrade)
|
||||
+28
-7
@@ -3050,12 +3050,17 @@ def _wait_for_external_cron_worker(
|
||||
|
||||
|
||||
def _launch_external_cron_worker(job: dict) -> bool:
|
||||
"""Launch *job* outside a managed gateway cgroup when required.
|
||||
"""Launch *job* outside the managed gateway process when required.
|
||||
|
||||
Returns ``False`` when the caller is not a managed systemd gateway and the
|
||||
existing in-process path should be used. In managed topology, failure to
|
||||
establish the transient scope raises: falling back would recreate the
|
||||
restart interruption this handoff exists to prevent.
|
||||
existing in-process path should be used. In managed topology the job is
|
||||
always handed to an external worker with the #101940 ownership handoff:
|
||||
either inside a transient user scope (isolated) or - when no user D-Bus
|
||||
session exists and ``cron.require_restart_safe_scope`` is false (the
|
||||
default) - as a direct subprocess (process separation without cgroup
|
||||
isolation). Setting ``cron.require_restart_safe_scope: true`` restores
|
||||
fail-closed. Falling back to in-process in managed topology would
|
||||
recreate the restart interruption this handoff exists to prevent.
|
||||
"""
|
||||
execution_id = str(job["execution_id"])
|
||||
job_id = str(job["id"])
|
||||
@@ -3085,13 +3090,29 @@ def _launch_external_cron_worker(job: dict) -> bool:
|
||||
systemd_user_bus_env,
|
||||
)
|
||||
|
||||
cfg = load_config() or {}
|
||||
require_restart_safe_scope = bool(
|
||||
((cfg.get("cron") or {}) if isinstance(cfg, dict) else {}).get(
|
||||
"require_restart_safe_scope", False
|
||||
)
|
||||
)
|
||||
multiplex_active = is_multiplex_active()
|
||||
scoped_command = restart_safe_gateway_child_argv(
|
||||
dispatch = restart_safe_gateway_child_argv(
|
||||
command,
|
||||
unit_suffix=f"cron-{job_id}-exec-{execution_id}",
|
||||
require_restart_safe_scope=require_restart_safe_scope,
|
||||
)
|
||||
if scoped_command == command:
|
||||
if dispatch.mode == "in_process":
|
||||
# Not a managed systemd gateway: keep the existing in-process path.
|
||||
return False
|
||||
# "scoped" AND "degraded" both launch an external worker with the same
|
||||
# #101940 ownership handoff below. Degraded only differs in isolation:
|
||||
# the direct command runs in the gateway cgroup (documented in the
|
||||
# warning), so a mid-job gateway restart kills it — but the execution
|
||||
# ledger still records exactly what happened (failed/unknown) instead of
|
||||
# the job silently never running. Never fall back to in-process here:
|
||||
# that would recreate the restart-interruption edge #101940 closed.
|
||||
launch_command = dispatch.argv
|
||||
|
||||
if mark_execution_handoff_pending(execution_id) is None:
|
||||
raise RuntimeError(
|
||||
@@ -3134,7 +3155,7 @@ def _launch_external_cron_worker(job: dict) -> bool:
|
||||
worker_env = systemd_user_bus_env(worker_env)
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
scoped_command,
|
||||
launch_command,
|
||||
cwd=str(Path(__file__).resolve().parent.parent),
|
||||
env=worker_env,
|
||||
stdin=subprocess.DEVNULL,
|
||||
|
||||
@@ -1705,6 +1705,13 @@ DEFAULT_CONFIG = {
|
||||
# (long TTS audio, big exports) need more than 30s. Env: HERMES_CRON_MEDIA_SEND_TIMEOUT.
|
||||
# Keep in sync with cron.scheduler._DEFAULT_MEDIA_SEND_TIMEOUT.
|
||||
"media_send_timeout_seconds": 300,
|
||||
# Restart-safety policy when the managed gateway has no user systemd session
|
||||
# (containers/LXCs without linger): false (default) degrades cron jobs to a direct
|
||||
# external subprocess with a once-per-process warning - process separation kept,
|
||||
# cgroup isolation lost, a mid-job gateway restart kills the worker. True fails
|
||||
# closed with the enable-linger remedy. The Kanban dispatcher always requires a
|
||||
# scope regardless (see #102431).
|
||||
"require_restart_safe_scope": False,
|
||||
},
|
||||
# Kanban multi-agent coordination. The dispatcher ticks every N seconds, reclaims stale claims,
|
||||
# promotes dependency-satisfied todos to ready, and fires `hermes -p <assignee> chat -q ...` per
|
||||
|
||||
@@ -2146,17 +2146,25 @@ def _open_worker_log(task: Task, board: Optional[str]):
|
||||
|
||||
|
||||
def _restart_safe_worker_argv(task: Task, command: list[str]) -> list[str]:
|
||||
"""Wrap a managed-gateway worker in the shared restart-safe scope."""
|
||||
"""Wrap a managed-gateway worker in the shared restart-safe scope.
|
||||
|
||||
Kanban workers are long-lived agentic runs, so unlike bounded cron jobs
|
||||
they are never dispatched in the degraded (unsupervised) mode: a managed
|
||||
gateway with no user systemd session fails closed through the shared
|
||||
helper's raise (remedy text included). See #102431.
|
||||
"""
|
||||
from tools.process_registry import restart_safe_gateway_child_argv
|
||||
|
||||
if task.current_run_id is None:
|
||||
# Outside managed systemd this is harmless, but a managed dispatch must
|
||||
# never mint an untraceable scope. Check topology through the shared
|
||||
# never mint an untraceable worker. Check topology through the shared
|
||||
# helper first, using a placeholder suffix that cannot be launched.
|
||||
scoped = restart_safe_gateway_child_argv(
|
||||
command, unit_suffix=f"kanban-{task.id}-run-missing"
|
||||
dispatch = restart_safe_gateway_child_argv(
|
||||
command,
|
||||
unit_suffix=f"kanban-{task.id}-run-missing",
|
||||
require_restart_safe_scope=True,
|
||||
)
|
||||
if scoped is not command:
|
||||
if dispatch.mode == "scoped":
|
||||
raise RuntimeError(
|
||||
"cannot create restart-safe systemd scope for Kanban worker: "
|
||||
"the claimed task has no current run id"
|
||||
@@ -2166,7 +2174,8 @@ def _restart_safe_worker_argv(task: Task, command: list[str]) -> list[str]:
|
||||
return restart_safe_gateway_child_argv(
|
||||
command,
|
||||
unit_suffix=f"kanban-{task.id}-run-{task.current_run_id}",
|
||||
)
|
||||
require_restart_safe_scope=True,
|
||||
).argv
|
||||
|
||||
|
||||
def _default_spawn(task: Task, workspace: str, *, board: Optional[str] = None) -> Optional[int]:
|
||||
|
||||
@@ -78,7 +78,7 @@ def test_genuine_external_worker_crash_is_recovered_unknown(
|
||||
|
||||
|
||||
@pytest.mark.linux_only
|
||||
def test_restart_safe_gateway_child_fails_closed_without_scope(monkeypatch):
|
||||
def test_restart_safe_gateway_child_fails_closed_when_required(monkeypatch):
|
||||
import tools.process_registry as process_registry
|
||||
|
||||
monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True)
|
||||
@@ -87,19 +87,73 @@ def test_restart_safe_gateway_child_fails_closed_without_scope(monkeypatch):
|
||||
|
||||
with pytest.raises(RuntimeError, match="systemd-run --user --scope is unavailable"):
|
||||
process_registry.restart_safe_gateway_child_argv(
|
||||
["python", "worker.py"], unit_suffix="cron-job-1"
|
||||
["python", "worker.py"],
|
||||
unit_suffix="cron-job-1",
|
||||
require_restart_safe_scope=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.linux_only
|
||||
def test_restart_safe_gateway_child_degrades_without_scope(monkeypatch):
|
||||
"""Managed gateway + no user bus degrades to a distinguishable state.
|
||||
|
||||
The degraded dispatch must NOT be identical to the in-process passthrough:
|
||||
the scheduler launches it as an external subprocess (with the #101940
|
||||
ownership handoff), never in-process. See #102431 review.
|
||||
"""
|
||||
import tools.process_registry as process_registry
|
||||
|
||||
monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True)
|
||||
monkeypatch.setenv("INVOCATION_ID", "managed-service")
|
||||
monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", lambda: False)
|
||||
|
||||
command = ["python", "worker.py"]
|
||||
dispatch = process_registry.restart_safe_gateway_child_argv(
|
||||
command, unit_suffix="cron-job-1", require_restart_safe_scope=False
|
||||
)
|
||||
assert dispatch.mode == "degraded"
|
||||
assert dispatch.argv == command
|
||||
assert dispatch.reason == "no-user-bus"
|
||||
|
||||
|
||||
@pytest.mark.linux_only
|
||||
def test_restart_safe_gateway_child_scoped_and_in_process_modes(monkeypatch):
|
||||
import tools.process_registry as process_registry
|
||||
|
||||
command = ["python", "worker.py"]
|
||||
# Managed + working bus -> scoped wrapper.
|
||||
monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True)
|
||||
monkeypatch.setenv("INVOCATION_ID", "managed-service")
|
||||
monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
process_registry, "_build_systemd_scope_argv",
|
||||
lambda cmd, unit_suffix: ["scope", "--", *cmd],
|
||||
)
|
||||
scoped = process_registry.restart_safe_gateway_child_argv(
|
||||
command, unit_suffix="cron-job-1", require_restart_safe_scope=False
|
||||
)
|
||||
assert scoped.mode == "scoped"
|
||||
assert scoped.argv == ["scope", "--", "python", "worker.py"]
|
||||
# Outside managed topology -> in-process passthrough (identity preserved).
|
||||
monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: False)
|
||||
passthrough = process_registry.restart_safe_gateway_child_argv(
|
||||
command, unit_suffix="cron-job-1", require_restart_safe_scope=False
|
||||
)
|
||||
assert passthrough.mode == "in_process"
|
||||
assert passthrough.argv is command
|
||||
|
||||
|
||||
def test_restart_safe_gateway_child_is_unchanged_outside_managed_gateway(monkeypatch):
|
||||
import tools.process_registry as process_registry
|
||||
|
||||
command = ["python", "worker.py"]
|
||||
monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: False)
|
||||
|
||||
assert process_registry.restart_safe_gateway_child_argv(
|
||||
command, unit_suffix="cron-job-1"
|
||||
) is command
|
||||
dispatch = process_registry.restart_safe_gateway_child_argv(
|
||||
command, unit_suffix="cron-job-1", require_restart_safe_scope=False
|
||||
)
|
||||
assert dispatch.mode == "in_process"
|
||||
assert dispatch.argv is command
|
||||
|
||||
|
||||
def test_restart_safe_gateway_child_never_probes_systemd_off_linux(monkeypatch):
|
||||
@@ -112,9 +166,11 @@ def test_restart_safe_gateway_child_never_probes_systemd_off_linux(monkeypatch):
|
||||
monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", probe)
|
||||
monkeypatch.setenv("INVOCATION_ID", "managed-service")
|
||||
|
||||
assert process_registry.restart_safe_gateway_child_argv(
|
||||
command, unit_suffix="cron-job-1"
|
||||
) is command
|
||||
dispatch = process_registry.restart_safe_gateway_child_argv(
|
||||
command, unit_suffix="cron-job-1", require_restart_safe_scope=False
|
||||
)
|
||||
assert dispatch.mode == "in_process"
|
||||
assert dispatch.argv is command
|
||||
probe.assert_not_called()
|
||||
|
||||
|
||||
@@ -197,10 +253,11 @@ def test_launch_external_worker_uses_restart_safe_scope_and_acknowledges(
|
||||
)
|
||||
register_env_passthrough(["SERVICE_TOKEN"])
|
||||
wrapped_commands = []
|
||||
from tools.process_registry import GatewayChildDispatch
|
||||
|
||||
def wrap(command, *, unit_suffix):
|
||||
def wrap(command, *, unit_suffix, require_restart_safe_scope=False):
|
||||
wrapped_commands.append((command, unit_suffix))
|
||||
return ["scope", "--", *command]
|
||||
return GatewayChildDispatch("scoped", ["scope", "--", *command])
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tools.process_registry.restart_safe_gateway_child_argv", wrap
|
||||
@@ -316,15 +373,16 @@ def test_launch_external_worker_stays_in_process_outside_managed_gateway(
|
||||
monkeypatch,
|
||||
):
|
||||
import cron.scheduler as scheduler
|
||||
from tools.process_registry import GatewayChildDispatch
|
||||
|
||||
command_calls = []
|
||||
|
||||
def unchanged(command, *, unit_suffix):
|
||||
def passthrough(command, *, unit_suffix, require_restart_safe_scope=False):
|
||||
command_calls.append((command, unit_suffix))
|
||||
return command
|
||||
return GatewayChildDispatch("in_process", command)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tools.process_registry.restart_safe_gateway_child_argv", unchanged
|
||||
"tools.process_registry.restart_safe_gateway_child_argv", passthrough
|
||||
)
|
||||
popen = Mock()
|
||||
monkeypatch.setattr(scheduler.subprocess, "Popen", popen)
|
||||
@@ -336,6 +394,163 @@ def test_launch_external_worker_stays_in_process_outside_managed_gateway(
|
||||
popen.assert_not_called()
|
||||
|
||||
|
||||
def test_launch_external_worker_dispatches_degraded_case_as_external_subprocess(
|
||||
tmp_path, monkeypatch,
|
||||
):
|
||||
"""Managed gateway + no user bus: the job MUST still Popen externally.
|
||||
|
||||
Regression for the #102431 review blocker: the degraded dispatch must
|
||||
take the external-worker path (payload written, handoff fenced, Popen
|
||||
called with the direct command) — never silently fall back in-process.
|
||||
"""
|
||||
import cron.scheduler as scheduler
|
||||
from tools.process_registry import GatewayChildDispatch
|
||||
|
||||
job = {"id": "job-1", "execution_id": "exec-1", "prompt": "work"}
|
||||
monkeypatch.setattr(scheduler, "_get_hermes_home", lambda: tmp_path)
|
||||
|
||||
def degraded(command, *, unit_suffix, require_restart_safe_scope=False):
|
||||
assert unit_suffix == "cron-job-1-exec-exec-1"
|
||||
return GatewayChildDispatch("degraded", command, "no-user-bus")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tools.process_registry.restart_safe_gateway_child_argv", degraded
|
||||
)
|
||||
|
||||
class FakeProcess:
|
||||
returncode = None
|
||||
|
||||
def poll(self):
|
||||
return self.returncode
|
||||
|
||||
def wait(self, timeout=None):
|
||||
if self.returncode is None:
|
||||
raise subprocess.TimeoutExpired(cmd="worker", timeout=timeout)
|
||||
return self.returncode
|
||||
|
||||
spawned = []
|
||||
|
||||
def popen(command, **kwargs):
|
||||
spawned.append((command, kwargs))
|
||||
assert command[0:2] == [sys.executable, "-m"], command[:3]
|
||||
assert "--external-worker-file" in command
|
||||
payload_index = command.index("--external-worker-file") + 1
|
||||
payload = json.loads(Path(command[payload_index]).read_text())
|
||||
assert payload["job"]["id"] == "job-1"
|
||||
ack_index = command.index("--ack-file") + 1
|
||||
Path(command[ack_index]).write_text(
|
||||
json.dumps({"pid": 4321, "execution_id": "exec-1"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return FakeProcess()
|
||||
|
||||
handoff = Mock(return_value={"id": "exec-1", "handoff_pending": 1})
|
||||
monkeypatch.setattr(scheduler, "mark_execution_handoff_pending", handoff)
|
||||
monkeypatch.setattr(scheduler.subprocess, "Popen", popen)
|
||||
observed_statuses = iter(
|
||||
[
|
||||
{"id": "exec-1", "status": "running"},
|
||||
{"id": "exec-1", "status": "completed"},
|
||||
]
|
||||
)
|
||||
get = Mock(side_effect=lambda _execution_id: next(observed_statuses))
|
||||
monkeypatch.setattr(scheduler, "get_execution", get)
|
||||
|
||||
assert scheduler._launch_external_cron_worker(job) is True
|
||||
# Direct command, NOT a systemd-run wrapper — but still an external Popen.
|
||||
assert "systemd-run" not in " ".join(spawned[0][0])
|
||||
assert spawned[0][1]["start_new_session"] is True
|
||||
handoff.assert_called_once_with("exec-1")
|
||||
assert not (tmp_path / "cron/external-workers/exec-1.json").exists()
|
||||
|
||||
|
||||
def test_launch_external_worker_fails_closed_when_config_requires_scope(monkeypatch):
|
||||
"""cron.require_restart_safe_scope=true restores the fail-closed raise
|
||||
through the real helper (config plumbing, not a stubbed dispatch)."""
|
||||
import cron.scheduler as scheduler
|
||||
import tools.process_registry as process_registry
|
||||
|
||||
monkeypatch.setattr(
|
||||
scheduler, "load_config",
|
||||
lambda: {"cron": {"require_restart_safe_scope": True}},
|
||||
)
|
||||
monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True)
|
||||
monkeypatch.setenv("INVOCATION_ID", "managed-service")
|
||||
monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", lambda: False)
|
||||
popen = Mock()
|
||||
monkeypatch.setattr(scheduler.subprocess, "Popen", popen)
|
||||
|
||||
with pytest.raises(RuntimeError, match="systemd-run --user --scope is unavailable"):
|
||||
scheduler._launch_external_cron_worker(
|
||||
{"id": "job-1", "execution_id": "exec-1", "prompt": "work"}
|
||||
)
|
||||
popen.assert_not_called()
|
||||
|
||||
|
||||
def test_launch_external_worker_degrades_by_default_with_real_helper(
|
||||
tmp_path, monkeypatch,
|
||||
):
|
||||
"""Managed gateway + no bus + default config (key absent): the real helper
|
||||
degrades and the job still Popens externally with the #101940 handoff.
|
||||
|
||||
Exercises the config default end-to-end (cron.require_restart_safe_scope
|
||||
defaults to false), not a stubbed dispatch.
|
||||
"""
|
||||
import cron.scheduler as scheduler
|
||||
import tools.process_registry as process_registry
|
||||
|
||||
job = {"id": "job-1", "execution_id": "exec-1", "prompt": "work"}
|
||||
monkeypatch.setattr(scheduler, "_get_hermes_home", lambda: tmp_path)
|
||||
monkeypatch.setattr(scheduler, "load_config", lambda: {})
|
||||
monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True)
|
||||
monkeypatch.setenv("INVOCATION_ID", "managed-service")
|
||||
monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", lambda: False)
|
||||
|
||||
class FakeProcess:
|
||||
returncode = None
|
||||
|
||||
def poll(self):
|
||||
return self.returncode
|
||||
|
||||
def wait(self, timeout=None):
|
||||
if self.returncode is None:
|
||||
raise subprocess.TimeoutExpired(cmd="worker", timeout=timeout)
|
||||
return self.returncode
|
||||
|
||||
spawned = []
|
||||
|
||||
def popen(command, **kwargs):
|
||||
spawned.append((command, kwargs))
|
||||
payload_index = command.index("--external-worker-file") + 1
|
||||
payload = json.loads(Path(command[payload_index]).read_text())
|
||||
assert payload["job"]["id"] == "job-1"
|
||||
ack_index = command.index("--ack-file") + 1
|
||||
Path(command[ack_index]).write_text(
|
||||
json.dumps({"pid": 4321, "execution_id": "exec-1"}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return FakeProcess()
|
||||
|
||||
handoff = Mock(return_value={"id": "exec-1", "handoff_pending": 1})
|
||||
monkeypatch.setattr(scheduler, "mark_execution_handoff_pending", handoff)
|
||||
monkeypatch.setattr(scheduler.subprocess, "Popen", popen)
|
||||
observed_statuses = iter(
|
||||
[
|
||||
{"id": "exec-1", "status": "running"},
|
||||
{"id": "exec-1", "status": "completed"},
|
||||
]
|
||||
)
|
||||
get = Mock(side_effect=lambda _execution_id: next(observed_statuses))
|
||||
monkeypatch.setattr(scheduler, "get_execution", get)
|
||||
|
||||
assert scheduler._launch_external_cron_worker(job) is True
|
||||
# Direct command, NOT a systemd-run wrapper — but still an external Popen.
|
||||
assert "systemd-run" not in " ".join(spawned[0][0])
|
||||
assert spawned[0][1]["start_new_session"] is True
|
||||
handoff.assert_called_once_with("exec-1")
|
||||
assert not (tmp_path / "cron/external-workers/exec-1.json").exists()
|
||||
|
||||
|
||||
def test_shared_run_path_hands_gateway_fire_to_external_worker(monkeypatch):
|
||||
import cron.scheduler as scheduler
|
||||
|
||||
|
||||
+99
-20
@@ -27,7 +27,7 @@ _IS_LINUX = platform.system() == "Linux"
|
||||
from tools.environments.local import _find_shell, _resolve_safe_cwd, _sanitize_subprocess_env
|
||||
from hermes_cli._subprocess_compat import windows_hide_flags
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Literal, NamedTuple, Optional
|
||||
|
||||
from hermes_cli.config import get_hermes_home
|
||||
|
||||
@@ -291,36 +291,115 @@ def _build_systemd_scope_argv(shell_argv: List[str], unit_suffix: str) -> List[s
|
||||
return _systemd_scope_argv(binary, f"hermes-worker-{unit_suffix}", *shell_argv)
|
||||
|
||||
|
||||
# --- restart-safe gateway child dispatch --------------------------------------
|
||||
# A systemd-supervised gateway restart kills every process in the service cgroup, so
|
||||
# children that must survive it are launched outside that cgroup via a transient user
|
||||
# scope (systemd-run --user --scope) when one can be created. Hosts with no user
|
||||
# systemd session at all (containers, LXCs without linger) cannot create scopes; the
|
||||
# callers set policy explicitly through ``require_restart_safe_scope`` (cron reads
|
||||
# ``cron.require_restart_safe_scope`` from config.yaml; kanban always requires a scope).
|
||||
|
||||
_scope_degraded_warned = False
|
||||
|
||||
|
||||
def _warn_scope_degraded_once(unit_suffix: str) -> None:
|
||||
"""Emit the degrade warning once per process.
|
||||
|
||||
The scope-availability verdict is cached, so without this guard the warning
|
||||
would fire on every dispatch (every cron fire on a bus-less host).
|
||||
"""
|
||||
global _scope_degraded_warned
|
||||
if _scope_degraded_warned:
|
||||
return
|
||||
_scope_degraded_warned = True
|
||||
logger.warning(
|
||||
"%s: systemd-run --user --scope is unavailable (no user D-Bus session at "
|
||||
"/run/user/%d/bus); dispatching the gateway child as a direct external "
|
||||
"subprocess without restart-safe cgroup isolation. The job still runs "
|
||||
"outside the gateway process, but will be killed if the gateway restarts "
|
||||
"mid-job. Remediate with `sudo loginctl enable-linger <gateway-user>` (plus "
|
||||
"XDG_RUNTIME_DIR/DBUS_SESSION_BUS_ADDRESS in the service unit), or set "
|
||||
"cron.require_restart_safe_scope=true in config.yaml to fail closed instead.",
|
||||
unit_suffix, os.getuid(),
|
||||
)
|
||||
|
||||
|
||||
class GatewayChildDispatch(NamedTuple):
|
||||
"""How a managed-gateway child should be launched.
|
||||
|
||||
Three mutually exclusive topologies - the whole point of this type is
|
||||
that (1) and (3) must never collapse into the same value:
|
||||
|
||||
- ``"in_process"`` - not a managed systemd gateway (standalone process,
|
||||
non-systemd supervisor, non-Linux host). The caller keeps its existing
|
||||
in-process path. ``argv is command`` holds, preserving the historical
|
||||
passthrough contract.
|
||||
- ``"scoped"`` - managed gateway with a working user bus. ``argv`` is the
|
||||
``systemd-run --user --scope`` wrapper; the caller launches it as an
|
||||
external worker with the #101940 ownership handoff.
|
||||
- ``"degraded"`` - managed gateway WITHOUT a user bus. ``argv`` is the
|
||||
direct command, but the caller MUST still launch it as an external
|
||||
subprocess (same ownership handoff as ``"scoped"``) - never fall back
|
||||
to the in-process path, which would recreate the restart-interruption
|
||||
edge #101940 closed. Isolation is lost but process separation is kept.
|
||||
"""
|
||||
|
||||
mode: Literal["in_process", "scoped", "degraded"]
|
||||
argv: List[str]
|
||||
reason: str = ""
|
||||
|
||||
|
||||
def restart_safe_gateway_child_argv(
|
||||
command: List[str], *, unit_suffix: str
|
||||
) -> List[str]:
|
||||
command: List[str], *, unit_suffix: str, require_restart_safe_scope: bool,
|
||||
) -> GatewayChildDispatch:
|
||||
"""Place a managed-systemd gateway child outside the gateway cgroup.
|
||||
|
||||
Returns a :class:`GatewayChildDispatch` distinguishing three topologies -
|
||||
never the bare command list, so callers cannot mistake a degraded dispatch
|
||||
for "not managed, stay in-process".
|
||||
|
||||
Children that must survive an intentional gateway restart cannot rely on
|
||||
``start_new_session`` alone: systemd still kills every process in the
|
||||
service cgroup. In that topology, require a transient user scope and fail
|
||||
closed if it cannot be established. Standalone processes, non-systemd
|
||||
supervisors, and non-Linux hosts retain the direct command.
|
||||
service cgroup. In that topology, prefer a transient user scope.
|
||||
|
||||
When a user systemd session is genuinely absent (containers, minimal LXCs,
|
||||
macOS-style supervisors) the scope cannot be created, but hard failing takes
|
||||
down every scheduled job on the host - a silent cron outage with no
|
||||
operator-visible symptom beyond skipped executions (the #101940 durability
|
||||
contract covers the restart case, not the never-had-a-bus case). Callers
|
||||
therefore state their policy explicitly: ``require_restart_safe_scope=True``
|
||||
raises when no scope can be established (fail-closed, the kanban contract);
|
||||
``False`` degrades to a direct external subprocess with a once-per-process
|
||||
warning (the cron default behind ``cron.require_restart_safe_scope``).
|
||||
|
||||
Standalone processes, non-systemd supervisors, and non-Linux hosts return
|
||||
``mode == "in_process"`` - the caller keeps its existing in-process path.
|
||||
"""
|
||||
if not _IS_LINUX:
|
||||
return command
|
||||
return GatewayChildDispatch("in_process", command)
|
||||
if not _is_supervised_gateway_process() or not os.environ.get("INVOCATION_ID"):
|
||||
return command
|
||||
return GatewayChildDispatch("in_process", command)
|
||||
if not _systemd_run_user_scope_available():
|
||||
# Stored as the cron execution's error and shown on the job row: name the remedy.
|
||||
raise RuntimeError(
|
||||
"cannot create restart-safe systemd scope for gateway child: "
|
||||
"systemd-run --user --scope is unavailable (usually no reachable user D-Bus session at "
|
||||
f"/run/user/{os.getuid()}/bus). On a system-level service install, run " # windows-footgun: ok — behind the _IS_LINUX return above
|
||||
"`sudo loginctl enable-linger <gateway-user>` and restart the gateway."
|
||||
)
|
||||
if require_restart_safe_scope:
|
||||
# Stored as the cron execution's error and shown on the job row: name the remedy.
|
||||
raise RuntimeError(
|
||||
"cannot create restart-safe systemd scope for gateway child: "
|
||||
"systemd-run --user --scope is unavailable (usually no reachable user D-Bus session at "
|
||||
f"/run/user/{os.getuid()}/bus). On a system-level service install, run " # windows-footgun: ok — behind the _IS_LINUX return above
|
||||
"`sudo loginctl enable-linger <gateway-user>` and restart the gateway."
|
||||
)
|
||||
_warn_scope_degraded_once(unit_suffix)
|
||||
return GatewayChildDispatch("degraded", command, "no-user-bus")
|
||||
scoped = _build_systemd_scope_argv(command, unit_suffix=unit_suffix)
|
||||
if scoped == command:
|
||||
raise RuntimeError(
|
||||
"cannot create restart-safe systemd scope for gateway child: "
|
||||
"systemd-run disappeared after the availability probe"
|
||||
)
|
||||
return scoped
|
||||
if require_restart_safe_scope:
|
||||
raise RuntimeError(
|
||||
"cannot create restart-safe systemd scope for gateway child: "
|
||||
"systemd-run disappeared after the availability probe"
|
||||
)
|
||||
_warn_scope_degraded_once(unit_suffix)
|
||||
return GatewayChildDispatch("degraded", command, "scope-binary-vanished")
|
||||
return GatewayChildDispatch("scoped", scoped)
|
||||
|
||||
|
||||
def _stop_systemd_unit(unit_name: str) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user