diff --git a/nix/checks.nix b/nix/checks.nix index 227f57d5e5..b47fe19c3d 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -810,6 +810,49 @@ json.dump(sorted(leaf_paths(DEFAULT_CONFIG)), sys.stdout, indent=2) '' ); + # ── The user manager that restart-safe cron workers need ───────── + # The cron scheduler launches every job in a transient `systemd-run + # --user --scope`, so that a gateway restart cannot kill a running job, + # and it fails the fire closed when the scope cannot be created. A + # system service has no user manager — and so no /run/user//bus — + # unless the uid lingers. That makes `linger` load-bearing for cron on + # this module, not cosmetic: without it every job dies before an agent + # starts. This check proves three properties: the uid the gateway runs + # as lingers, a lingering gateway does not start before that uid's bus + # exists, and a gateway whose uid does not linger waits for nothing. + cron-worker-user-scope = + let + configOf = settings: (evalNixosModule ({ enable = true; } // settings)).config; + + managed = configOf { }; + gateway = managed.systemd.services.hermes-agent; + gatewayUser = gateway.serviceConfig.User; + + # The operator declares the user themselves. Nothing here knows + # whether they lingered it, so nothing may assume a bus. + unmanaged = (configOf { createUser = false; }).systemd.services.hermes-agent; + + failures = + lib.optional (!((managed.users.users.${gatewayUser}.linger or false) == true)) + "the uid the gateway runs as (${gatewayUser}) must linger: `systemd-run --user --scope` has no user manager to ask without it, and cron dispatch fails closed" + ++ lib.optional (!lib.elem "linger-users.service" gateway.after) + "a lingering gateway must be ordered after linger-users.service, the unit that runs `loginctl enable-linger`" + ++ lib.optional (!lib.hasInfix "/run/user" gateway.preStart) + "a lingering gateway must wait for its user bus before ExecStart: the bus environment is resolved once at startup, so a bus that appears later is one this process never sees" + ++ lib.optional (lib.hasInfix "/run/user" unmanaged.preStart) + "a gateway whose uid is not known to linger must not block on a user bus that may never arrive"; + in + pkgs.runCommand "hermes-cron-worker-user-scope" { } ( + if failures != [ ] then + throw "cron worker user scope check failed:\n${lib.concatMapStringsSep "\n" (f: " - ${f}") failures}" + else + '' + echo "PASS: the gateway uid lingers and the unit waits for its user bus" + mkdir -p $out + echo "ok" > $out/result + '' + ); + # ── How .env is built ──────────────────────────────────────────── # This check runs the real script that both modules use to build # $HERMES_HOME/.env. The important property is that a second run diff --git a/nix/nixosModules.nix b/nix/nixosModules.nix index 0a3182d640..52ba773776 100644 --- a/nix/nixosModules.nix +++ b/nix/nixosModules.nix @@ -240,6 +240,13 @@ unitPath = common.processPath { inherit pkgs cfg; }; + # Whether the service uid gets a systemd user manager, and with it the + # user bus that restart-safe cron workers need (see the `linger` attribute + # under createUser). Read back off `config` rather than assumed, so an + # operator who declares the user themselves — or who turns the default off + # — does not pay for a bus that will never arrive. + lingerEnabled = ((config.users.users.${cfg.user} or { }).linger or false) == true; + in { options.services.hermes-agent = @@ -350,6 +357,15 @@ home = cfg.stateDir; createHome = true; shell = pkgs.bashInteractive; + + # The cron scheduler launches every job in a transient `systemd-run + # --user --scope` so a gateway restart cannot kill a running job, + # and that needs a systemd user manager for this uid. A system + # service has no /run/user/ unless the uid lingers, and the + # dispatch fails closed — without this, no cron job runs at all. + # + # Needs nixpkgs >= 25.05 (users.manageLingering). + linger = lib.mkDefault true; }; }) @@ -549,14 +565,42 @@ systemd.services.hermes-agent = { description = "Hermes Agent Gateway"; wantedBy = [ "multi-user.target" ]; - after = [ "network-online.target" ]; - wants = [ "network-online.target" ]; + # linger-users.service is the unit that runs `loginctl + # enable-linger` for a declared `users.users..linger`. + after = [ + "network-online.target" + ] + ++ lib.optional lingerEnabled "linger-users.service"; + wants = [ + "network-online.target" + ] + ++ lib.optional lingerEnabled "linger-users.service"; # cfg.environment and cfg.environmentFiles are written to # $HERMES_HOME/.env by the activation script. load_hermes_dotenv() # reads them at Python startup — no systemd EnvironmentFile needed. environment = commonUnitEnvironment; + # Wait for the user bus that `systemd-run --user --scope` connects + # to. Ordering after linger-users.service is not enough on its own: + # `loginctl enable-linger` returns before logind has finished + # starting user@.service, and run_gateway() resolves + # XDG_RUNTIME_DIR / DBUS_SESSION_BUS_ADDRESS exactly once at + # startup — so a bus that appears after ExecStart is a bus this + # process never sees, for its whole lifetime. + # + # Bounded and non-fatal: a gateway without cron beats no gateway. + preStart = lib.mkIf lingerEnabled '' + for _ in $(seq 1 50); do + [ -S "/run/user/$(id -u)/bus" ] && break + sleep 0.2 + done + if [ ! -S "/run/user/$(id -u)/bus" ]; then + echo "hermes-agent: no user bus at /run/user/$(id -u)/bus after 10s;" \ + "restart-safe cron dispatch will fail for the life of this process" >&2 + fi + ''; + serviceConfig = commonServiceConfig // { ExecStart = lib.escapeShellArgs (common.gatewayArgv cfg); };