From 560b6d2e818924d576dcb300e220b5b4f9b25aad Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Sun, 6 Sep 2026 18:49:40 +1200 Subject: [PATCH] fix(cron): degrade gracefully when systemd user scopes are unavailable A systemd-supervised gateway (INVOCATION_ID set) with no user D-Bus session (containers, minimal LXCs, supervisors without linger) fails EVERY scheduled job at dispatch: restart_safe_gateway_child_argv() raises, run_one_job() records a failure, and the only symptom is silently skipped executions (a missed nightly backup, dead watchdogs, no alert). Cron now degrades to a direct external subprocess with a once-per-process warning instead of raising, unless cron.require_restart_safe_scope=true (config.yaml, default false) restores fail-closed. Degraded jobs keep process separation and the full #101940 ownership handoff - only cgroup isolation is lost, so a mid-job gateway restart kills the worker and the execution ledger records exactly that. The dispatch is a GatewayChildDispatch NamedTuple (in_process / scoped / degraded) so the degraded case can never collapse into the "not managed, stay in-process" sentinel - the failure mode that would recreate the restart-interruption edge #101940 closed. Kanban stays fail-closed (require_restart_safe_scope=True at its call sites): its workers are long-lived agentic runs, so the degrade policy is limited to bounded cron jobs in this PR. Addresses the #102431 review: the env-var flag became a config key per AGENTS.md (no new HERMES_* non-secret vars), Kanban keeps fail-closed instead of updating its tests to a degraded contract, main's enable-linger remedy message is preserved, and the degrade warning fires once per process. --- contributors/emails/paularobertson@gmail.com | 2 + cron/scheduler.py | 35 ++- hermes_cli/config_defaults.py | 7 + hermes_cli/kanban_db_dispatch.py | 21 +- tests/cron/test_restart_safe_worker.py | 241 ++++++++++++++++++- tools/process_registry.py | 119 +++++++-- 6 files changed, 379 insertions(+), 46 deletions(-) create mode 100644 contributors/emails/paularobertson@gmail.com diff --git a/contributors/emails/paularobertson@gmail.com b/contributors/emails/paularobertson@gmail.com new file mode 100644 index 0000000000..3d9dabcba6 --- /dev/null +++ b/contributors/emails/paularobertson@gmail.com @@ -0,0 +1,2 @@ +kiwipaulrob +# PR #102431 rework (cron scope graceful degrade) diff --git a/cron/scheduler.py b/cron/scheduler.py index 2133b0c028..22031b6bb2 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -3050,12 +3050,17 @@ def _wait_for_external_cron_worker( def _launch_external_cron_worker(job: dict) -> bool: - """Launch *job* outside a managed gateway cgroup when required. + """Launch *job* outside the managed gateway process when required. Returns ``False`` when the caller is not a managed systemd gateway and the - existing in-process path should be used. In managed topology, failure to - establish the transient scope raises: falling back would recreate the - restart interruption this handoff exists to prevent. + existing in-process path should be used. In managed topology the job is + always handed to an external worker with the #101940 ownership handoff: + either inside a transient user scope (isolated) or - when no user D-Bus + session exists and ``cron.require_restart_safe_scope`` is false (the + default) - as a direct subprocess (process separation without cgroup + isolation). Setting ``cron.require_restart_safe_scope: true`` restores + fail-closed. Falling back to in-process in managed topology would + recreate the restart interruption this handoff exists to prevent. """ execution_id = str(job["execution_id"]) job_id = str(job["id"]) @@ -3085,13 +3090,29 @@ def _launch_external_cron_worker(job: dict) -> bool: systemd_user_bus_env, ) + cfg = load_config() or {} + require_restart_safe_scope = bool( + ((cfg.get("cron") or {}) if isinstance(cfg, dict) else {}).get( + "require_restart_safe_scope", False + ) + ) multiplex_active = is_multiplex_active() - scoped_command = restart_safe_gateway_child_argv( + dispatch = restart_safe_gateway_child_argv( command, unit_suffix=f"cron-{job_id}-exec-{execution_id}", + require_restart_safe_scope=require_restart_safe_scope, ) - if scoped_command == command: + if dispatch.mode == "in_process": + # Not a managed systemd gateway: keep the existing in-process path. return False + # "scoped" AND "degraded" both launch an external worker with the same + # #101940 ownership handoff below. Degraded only differs in isolation: + # the direct command runs in the gateway cgroup (documented in the + # warning), so a mid-job gateway restart kills it — but the execution + # ledger still records exactly what happened (failed/unknown) instead of + # the job silently never running. Never fall back to in-process here: + # that would recreate the restart-interruption edge #101940 closed. + launch_command = dispatch.argv if mark_execution_handoff_pending(execution_id) is None: raise RuntimeError( @@ -3134,7 +3155,7 @@ def _launch_external_cron_worker(job: dict) -> bool: worker_env = systemd_user_bus_env(worker_env) try: process = subprocess.Popen( - scoped_command, + launch_command, cwd=str(Path(__file__).resolve().parent.parent), env=worker_env, stdin=subprocess.DEVNULL, diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 1cc1ea6ef5..d89e66d368 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -1705,6 +1705,13 @@ DEFAULT_CONFIG = { # (long TTS audio, big exports) need more than 30s. Env: HERMES_CRON_MEDIA_SEND_TIMEOUT. # Keep in sync with cron.scheduler._DEFAULT_MEDIA_SEND_TIMEOUT. "media_send_timeout_seconds": 300, + # Restart-safety policy when the managed gateway has no user systemd session + # (containers/LXCs without linger): false (default) degrades cron jobs to a direct + # external subprocess with a once-per-process warning - process separation kept, + # cgroup isolation lost, a mid-job gateway restart kills the worker. True fails + # closed with the enable-linger remedy. The Kanban dispatcher always requires a + # scope regardless (see #102431). + "require_restart_safe_scope": False, }, # Kanban multi-agent coordination. The dispatcher ticks every N seconds, reclaims stale claims, # promotes dependency-satisfied todos to ready, and fires `hermes -p chat -q ...` per diff --git a/hermes_cli/kanban_db_dispatch.py b/hermes_cli/kanban_db_dispatch.py index 407ab91f24..6ab9ea83cc 100644 --- a/hermes_cli/kanban_db_dispatch.py +++ b/hermes_cli/kanban_db_dispatch.py @@ -2146,17 +2146,25 @@ def _open_worker_log(task: Task, board: Optional[str]): def _restart_safe_worker_argv(task: Task, command: list[str]) -> list[str]: - """Wrap a managed-gateway worker in the shared restart-safe scope.""" + """Wrap a managed-gateway worker in the shared restart-safe scope. + + Kanban workers are long-lived agentic runs, so unlike bounded cron jobs + they are never dispatched in the degraded (unsupervised) mode: a managed + gateway with no user systemd session fails closed through the shared + helper's raise (remedy text included). See #102431. + """ from tools.process_registry import restart_safe_gateway_child_argv if task.current_run_id is None: # Outside managed systemd this is harmless, but a managed dispatch must - # never mint an untraceable scope. Check topology through the shared + # never mint an untraceable worker. Check topology through the shared # helper first, using a placeholder suffix that cannot be launched. - scoped = restart_safe_gateway_child_argv( - command, unit_suffix=f"kanban-{task.id}-run-missing" + dispatch = restart_safe_gateway_child_argv( + command, + unit_suffix=f"kanban-{task.id}-run-missing", + require_restart_safe_scope=True, ) - if scoped is not command: + if dispatch.mode == "scoped": raise RuntimeError( "cannot create restart-safe systemd scope for Kanban worker: " "the claimed task has no current run id" @@ -2166,7 +2174,8 @@ def _restart_safe_worker_argv(task: Task, command: list[str]) -> list[str]: return restart_safe_gateway_child_argv( command, unit_suffix=f"kanban-{task.id}-run-{task.current_run_id}", - ) + require_restart_safe_scope=True, + ).argv def _default_spawn(task: Task, workspace: str, *, board: Optional[str] = None) -> Optional[int]: diff --git a/tests/cron/test_restart_safe_worker.py b/tests/cron/test_restart_safe_worker.py index 4ff7b14839..6b6df0a381 100644 --- a/tests/cron/test_restart_safe_worker.py +++ b/tests/cron/test_restart_safe_worker.py @@ -78,7 +78,7 @@ def test_genuine_external_worker_crash_is_recovered_unknown( @pytest.mark.linux_only -def test_restart_safe_gateway_child_fails_closed_without_scope(monkeypatch): +def test_restart_safe_gateway_child_fails_closed_when_required(monkeypatch): import tools.process_registry as process_registry monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True) @@ -87,19 +87,73 @@ def test_restart_safe_gateway_child_fails_closed_without_scope(monkeypatch): with pytest.raises(RuntimeError, match="systemd-run --user --scope is unavailable"): process_registry.restart_safe_gateway_child_argv( - ["python", "worker.py"], unit_suffix="cron-job-1" + ["python", "worker.py"], + unit_suffix="cron-job-1", + require_restart_safe_scope=True, ) +@pytest.mark.linux_only +def test_restart_safe_gateway_child_degrades_without_scope(monkeypatch): + """Managed gateway + no user bus degrades to a distinguishable state. + + The degraded dispatch must NOT be identical to the in-process passthrough: + the scheduler launches it as an external subprocess (with the #101940 + ownership handoff), never in-process. See #102431 review. + """ + import tools.process_registry as process_registry + + monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True) + monkeypatch.setenv("INVOCATION_ID", "managed-service") + monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", lambda: False) + + command = ["python", "worker.py"] + dispatch = process_registry.restart_safe_gateway_child_argv( + command, unit_suffix="cron-job-1", require_restart_safe_scope=False + ) + assert dispatch.mode == "degraded" + assert dispatch.argv == command + assert dispatch.reason == "no-user-bus" + + +@pytest.mark.linux_only +def test_restart_safe_gateway_child_scoped_and_in_process_modes(monkeypatch): + import tools.process_registry as process_registry + + command = ["python", "worker.py"] + # Managed + working bus -> scoped wrapper. + monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True) + monkeypatch.setenv("INVOCATION_ID", "managed-service") + monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", lambda: True) + monkeypatch.setattr( + process_registry, "_build_systemd_scope_argv", + lambda cmd, unit_suffix: ["scope", "--", *cmd], + ) + scoped = process_registry.restart_safe_gateway_child_argv( + command, unit_suffix="cron-job-1", require_restart_safe_scope=False + ) + assert scoped.mode == "scoped" + assert scoped.argv == ["scope", "--", "python", "worker.py"] + # Outside managed topology -> in-process passthrough (identity preserved). + monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: False) + passthrough = process_registry.restart_safe_gateway_child_argv( + command, unit_suffix="cron-job-1", require_restart_safe_scope=False + ) + assert passthrough.mode == "in_process" + assert passthrough.argv is command + + def test_restart_safe_gateway_child_is_unchanged_outside_managed_gateway(monkeypatch): import tools.process_registry as process_registry command = ["python", "worker.py"] monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: False) - assert process_registry.restart_safe_gateway_child_argv( - command, unit_suffix="cron-job-1" - ) is command + dispatch = process_registry.restart_safe_gateway_child_argv( + command, unit_suffix="cron-job-1", require_restart_safe_scope=False + ) + assert dispatch.mode == "in_process" + assert dispatch.argv is command def test_restart_safe_gateway_child_never_probes_systemd_off_linux(monkeypatch): @@ -112,9 +166,11 @@ def test_restart_safe_gateway_child_never_probes_systemd_off_linux(monkeypatch): monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", probe) monkeypatch.setenv("INVOCATION_ID", "managed-service") - assert process_registry.restart_safe_gateway_child_argv( - command, unit_suffix="cron-job-1" - ) is command + dispatch = process_registry.restart_safe_gateway_child_argv( + command, unit_suffix="cron-job-1", require_restart_safe_scope=False + ) + assert dispatch.mode == "in_process" + assert dispatch.argv is command probe.assert_not_called() @@ -197,10 +253,11 @@ def test_launch_external_worker_uses_restart_safe_scope_and_acknowledges( ) register_env_passthrough(["SERVICE_TOKEN"]) wrapped_commands = [] + from tools.process_registry import GatewayChildDispatch - def wrap(command, *, unit_suffix): + def wrap(command, *, unit_suffix, require_restart_safe_scope=False): wrapped_commands.append((command, unit_suffix)) - return ["scope", "--", *command] + return GatewayChildDispatch("scoped", ["scope", "--", *command]) monkeypatch.setattr( "tools.process_registry.restart_safe_gateway_child_argv", wrap @@ -316,15 +373,16 @@ def test_launch_external_worker_stays_in_process_outside_managed_gateway( monkeypatch, ): import cron.scheduler as scheduler + from tools.process_registry import GatewayChildDispatch command_calls = [] - def unchanged(command, *, unit_suffix): + def passthrough(command, *, unit_suffix, require_restart_safe_scope=False): command_calls.append((command, unit_suffix)) - return command + return GatewayChildDispatch("in_process", command) monkeypatch.setattr( - "tools.process_registry.restart_safe_gateway_child_argv", unchanged + "tools.process_registry.restart_safe_gateway_child_argv", passthrough ) popen = Mock() monkeypatch.setattr(scheduler.subprocess, "Popen", popen) @@ -336,6 +394,163 @@ def test_launch_external_worker_stays_in_process_outside_managed_gateway( popen.assert_not_called() +def test_launch_external_worker_dispatches_degraded_case_as_external_subprocess( + tmp_path, monkeypatch, +): + """Managed gateway + no user bus: the job MUST still Popen externally. + + Regression for the #102431 review blocker: the degraded dispatch must + take the external-worker path (payload written, handoff fenced, Popen + called with the direct command) — never silently fall back in-process. + """ + import cron.scheduler as scheduler + from tools.process_registry import GatewayChildDispatch + + job = {"id": "job-1", "execution_id": "exec-1", "prompt": "work"} + monkeypatch.setattr(scheduler, "_get_hermes_home", lambda: tmp_path) + + def degraded(command, *, unit_suffix, require_restart_safe_scope=False): + assert unit_suffix == "cron-job-1-exec-exec-1" + return GatewayChildDispatch("degraded", command, "no-user-bus") + + monkeypatch.setattr( + "tools.process_registry.restart_safe_gateway_child_argv", degraded + ) + + class FakeProcess: + returncode = None + + def poll(self): + return self.returncode + + def wait(self, timeout=None): + if self.returncode is None: + raise subprocess.TimeoutExpired(cmd="worker", timeout=timeout) + return self.returncode + + spawned = [] + + def popen(command, **kwargs): + spawned.append((command, kwargs)) + assert command[0:2] == [sys.executable, "-m"], command[:3] + assert "--external-worker-file" in command + payload_index = command.index("--external-worker-file") + 1 + payload = json.loads(Path(command[payload_index]).read_text()) + assert payload["job"]["id"] == "job-1" + ack_index = command.index("--ack-file") + 1 + Path(command[ack_index]).write_text( + json.dumps({"pid": 4321, "execution_id": "exec-1"}), + encoding="utf-8", + ) + return FakeProcess() + + handoff = Mock(return_value={"id": "exec-1", "handoff_pending": 1}) + monkeypatch.setattr(scheduler, "mark_execution_handoff_pending", handoff) + monkeypatch.setattr(scheduler.subprocess, "Popen", popen) + observed_statuses = iter( + [ + {"id": "exec-1", "status": "running"}, + {"id": "exec-1", "status": "completed"}, + ] + ) + get = Mock(side_effect=lambda _execution_id: next(observed_statuses)) + monkeypatch.setattr(scheduler, "get_execution", get) + + assert scheduler._launch_external_cron_worker(job) is True + # Direct command, NOT a systemd-run wrapper — but still an external Popen. + assert "systemd-run" not in " ".join(spawned[0][0]) + assert spawned[0][1]["start_new_session"] is True + handoff.assert_called_once_with("exec-1") + assert not (tmp_path / "cron/external-workers/exec-1.json").exists() + + +def test_launch_external_worker_fails_closed_when_config_requires_scope(monkeypatch): + """cron.require_restart_safe_scope=true restores the fail-closed raise + through the real helper (config plumbing, not a stubbed dispatch).""" + import cron.scheduler as scheduler + import tools.process_registry as process_registry + + monkeypatch.setattr( + scheduler, "load_config", + lambda: {"cron": {"require_restart_safe_scope": True}}, + ) + monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True) + monkeypatch.setenv("INVOCATION_ID", "managed-service") + monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", lambda: False) + popen = Mock() + monkeypatch.setattr(scheduler.subprocess, "Popen", popen) + + with pytest.raises(RuntimeError, match="systemd-run --user --scope is unavailable"): + scheduler._launch_external_cron_worker( + {"id": "job-1", "execution_id": "exec-1", "prompt": "work"} + ) + popen.assert_not_called() + + +def test_launch_external_worker_degrades_by_default_with_real_helper( + tmp_path, monkeypatch, +): + """Managed gateway + no bus + default config (key absent): the real helper + degrades and the job still Popens externally with the #101940 handoff. + + Exercises the config default end-to-end (cron.require_restart_safe_scope + defaults to false), not a stubbed dispatch. + """ + import cron.scheduler as scheduler + import tools.process_registry as process_registry + + job = {"id": "job-1", "execution_id": "exec-1", "prompt": "work"} + monkeypatch.setattr(scheduler, "_get_hermes_home", lambda: tmp_path) + monkeypatch.setattr(scheduler, "load_config", lambda: {}) + monkeypatch.setattr(process_registry, "_is_supervised_gateway_process", lambda: True) + monkeypatch.setenv("INVOCATION_ID", "managed-service") + monkeypatch.setattr(process_registry, "_systemd_run_user_scope_available", lambda: False) + + class FakeProcess: + returncode = None + + def poll(self): + return self.returncode + + def wait(self, timeout=None): + if self.returncode is None: + raise subprocess.TimeoutExpired(cmd="worker", timeout=timeout) + return self.returncode + + spawned = [] + + def popen(command, **kwargs): + spawned.append((command, kwargs)) + payload_index = command.index("--external-worker-file") + 1 + payload = json.loads(Path(command[payload_index]).read_text()) + assert payload["job"]["id"] == "job-1" + ack_index = command.index("--ack-file") + 1 + Path(command[ack_index]).write_text( + json.dumps({"pid": 4321, "execution_id": "exec-1"}), + encoding="utf-8", + ) + return FakeProcess() + + handoff = Mock(return_value={"id": "exec-1", "handoff_pending": 1}) + monkeypatch.setattr(scheduler, "mark_execution_handoff_pending", handoff) + monkeypatch.setattr(scheduler.subprocess, "Popen", popen) + observed_statuses = iter( + [ + {"id": "exec-1", "status": "running"}, + {"id": "exec-1", "status": "completed"}, + ] + ) + get = Mock(side_effect=lambda _execution_id: next(observed_statuses)) + monkeypatch.setattr(scheduler, "get_execution", get) + + assert scheduler._launch_external_cron_worker(job) is True + # Direct command, NOT a systemd-run wrapper — but still an external Popen. + assert "systemd-run" not in " ".join(spawned[0][0]) + assert spawned[0][1]["start_new_session"] is True + handoff.assert_called_once_with("exec-1") + assert not (tmp_path / "cron/external-workers/exec-1.json").exists() + + def test_shared_run_path_hands_gateway_fire_to_external_worker(monkeypatch): import cron.scheduler as scheduler diff --git a/tools/process_registry.py b/tools/process_registry.py index 6487ebecbc..d1a9b2efa8 100644 --- a/tools/process_registry.py +++ b/tools/process_registry.py @@ -27,7 +27,7 @@ _IS_LINUX = platform.system() == "Linux" from tools.environments.local import _find_shell, _resolve_safe_cwd, _sanitize_subprocess_env from hermes_cli._subprocess_compat import windows_hide_flags from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Literal, NamedTuple, Optional from hermes_cli.config import get_hermes_home @@ -291,36 +291,115 @@ def _build_systemd_scope_argv(shell_argv: List[str], unit_suffix: str) -> List[s return _systemd_scope_argv(binary, f"hermes-worker-{unit_suffix}", *shell_argv) +# --- restart-safe gateway child dispatch -------------------------------------- +# A systemd-supervised gateway restart kills every process in the service cgroup, so +# children that must survive it are launched outside that cgroup via a transient user +# scope (systemd-run --user --scope) when one can be created. Hosts with no user +# systemd session at all (containers, LXCs without linger) cannot create scopes; the +# callers set policy explicitly through ``require_restart_safe_scope`` (cron reads +# ``cron.require_restart_safe_scope`` from config.yaml; kanban always requires a scope). + +_scope_degraded_warned = False + + +def _warn_scope_degraded_once(unit_suffix: str) -> None: + """Emit the degrade warning once per process. + + The scope-availability verdict is cached, so without this guard the warning + would fire on every dispatch (every cron fire on a bus-less host). + """ + global _scope_degraded_warned + if _scope_degraded_warned: + return + _scope_degraded_warned = True + logger.warning( + "%s: systemd-run --user --scope is unavailable (no user D-Bus session at " + "/run/user/%d/bus); dispatching the gateway child as a direct external " + "subprocess without restart-safe cgroup isolation. The job still runs " + "outside the gateway process, but will be killed if the gateway restarts " + "mid-job. Remediate with `sudo loginctl enable-linger ` (plus " + "XDG_RUNTIME_DIR/DBUS_SESSION_BUS_ADDRESS in the service unit), or set " + "cron.require_restart_safe_scope=true in config.yaml to fail closed instead.", + unit_suffix, os.getuid(), + ) + + +class GatewayChildDispatch(NamedTuple): + """How a managed-gateway child should be launched. + + Three mutually exclusive topologies - the whole point of this type is + that (1) and (3) must never collapse into the same value: + + - ``"in_process"`` - not a managed systemd gateway (standalone process, + non-systemd supervisor, non-Linux host). The caller keeps its existing + in-process path. ``argv is command`` holds, preserving the historical + passthrough contract. + - ``"scoped"`` - managed gateway with a working user bus. ``argv`` is the + ``systemd-run --user --scope`` wrapper; the caller launches it as an + external worker with the #101940 ownership handoff. + - ``"degraded"`` - managed gateway WITHOUT a user bus. ``argv`` is the + direct command, but the caller MUST still launch it as an external + subprocess (same ownership handoff as ``"scoped"``) - never fall back + to the in-process path, which would recreate the restart-interruption + edge #101940 closed. Isolation is lost but process separation is kept. + """ + + mode: Literal["in_process", "scoped", "degraded"] + argv: List[str] + reason: str = "" + + def restart_safe_gateway_child_argv( - command: List[str], *, unit_suffix: str -) -> List[str]: + command: List[str], *, unit_suffix: str, require_restart_safe_scope: bool, +) -> GatewayChildDispatch: """Place a managed-systemd gateway child outside the gateway cgroup. + Returns a :class:`GatewayChildDispatch` distinguishing three topologies - + never the bare command list, so callers cannot mistake a degraded dispatch + for "not managed, stay in-process". + Children that must survive an intentional gateway restart cannot rely on ``start_new_session`` alone: systemd still kills every process in the - service cgroup. In that topology, require a transient user scope and fail - closed if it cannot be established. Standalone processes, non-systemd - supervisors, and non-Linux hosts retain the direct command. + service cgroup. In that topology, prefer a transient user scope. + + When a user systemd session is genuinely absent (containers, minimal LXCs, + macOS-style supervisors) the scope cannot be created, but hard failing takes + down every scheduled job on the host - a silent cron outage with no + operator-visible symptom beyond skipped executions (the #101940 durability + contract covers the restart case, not the never-had-a-bus case). Callers + therefore state their policy explicitly: ``require_restart_safe_scope=True`` + raises when no scope can be established (fail-closed, the kanban contract); + ``False`` degrades to a direct external subprocess with a once-per-process + warning (the cron default behind ``cron.require_restart_safe_scope``). + + Standalone processes, non-systemd supervisors, and non-Linux hosts return + ``mode == "in_process"`` - the caller keeps its existing in-process path. """ if not _IS_LINUX: - return command + return GatewayChildDispatch("in_process", command) if not _is_supervised_gateway_process() or not os.environ.get("INVOCATION_ID"): - return command + return GatewayChildDispatch("in_process", command) if not _systemd_run_user_scope_available(): - # Stored as the cron execution's error and shown on the job row: name the remedy. - raise RuntimeError( - "cannot create restart-safe systemd scope for gateway child: " - "systemd-run --user --scope is unavailable (usually no reachable user D-Bus session at " - f"/run/user/{os.getuid()}/bus). On a system-level service install, run " # windows-footgun: ok — behind the _IS_LINUX return above - "`sudo loginctl enable-linger ` and restart the gateway." - ) + if require_restart_safe_scope: + # Stored as the cron execution's error and shown on the job row: name the remedy. + raise RuntimeError( + "cannot create restart-safe systemd scope for gateway child: " + "systemd-run --user --scope is unavailable (usually no reachable user D-Bus session at " + f"/run/user/{os.getuid()}/bus). On a system-level service install, run " # windows-footgun: ok — behind the _IS_LINUX return above + "`sudo loginctl enable-linger ` and restart the gateway." + ) + _warn_scope_degraded_once(unit_suffix) + return GatewayChildDispatch("degraded", command, "no-user-bus") scoped = _build_systemd_scope_argv(command, unit_suffix=unit_suffix) if scoped == command: - raise RuntimeError( - "cannot create restart-safe systemd scope for gateway child: " - "systemd-run disappeared after the availability probe" - ) - return scoped + if require_restart_safe_scope: + raise RuntimeError( + "cannot create restart-safe systemd scope for gateway child: " + "systemd-run disappeared after the availability probe" + ) + _warn_scope_degraded_once(unit_suffix) + return GatewayChildDispatch("degraded", command, "scope-binary-vanished") + return GatewayChildDispatch("scoped", scoped) def _stop_systemd_unit(unit_name: str) -> bool: