diff --git a/nix/checks.nix b/nix/checks.nix index 4ec3b7713c..f353b6942b 100644 --- a/nix/checks.nix +++ b/nix/checks.nix @@ -435,6 +435,109 @@ json.dump(sorted(leaf_paths(DEFAULT_CONFIG)), sys.stdout, indent=2) '' ); + # ── How the backend waits for its bind target ──────────────────── + # The backend binds to `host` immediately by default. A unit that + # starts at boot can lose the race against the daemon that supplies + # the address, such as tailscaled. `backend.waitFor` puts a poll in + # front of the bind. This check proves three properties: the default + # keeps the direct command line, each wait mode makes a launcher that + # polls and then execs hermes, and the assertions reject a + # configuration that cannot work. + backend-bind-wait = + let + execOf = + settings: + (evalNixosModule ({ enable = true; } // settings)).config.systemd.services.hermes-backend.serviceConfig.ExecStart; + + direct = execOf { backend.mode = "serve"; }; + + hostnameWait = execOf { + backend = { + mode = "serve"; + host = "host.example.ts.net"; + waitFor = "hostname"; + }; + }; + + interfaceWait = execOf { + backend = { + mode = "dashboard"; + waitFor = "interface"; + interfaceName = "tailscale0"; + waitTimeout = 30; + }; + }; + + # The launcher is a store path. Read it to see what it runs. + hostnameScript = builtins.readFile hostnameWait; + interfaceScript = builtins.readFile interfaceWait; + + evalFails = + settings: + !(builtins.tryEval ( + lib.deepSeq + (evalNixosModule ({ enable = true; } // settings)).config.system.build.toplevel.drvPath + true + )).success; + + failures = + # The default must not change. + lib.optional (!lib.hasInfix "bin/hermes serve --host 127.0.0.1" direct) + "without waitFor the backend must exec hermes directly, got: ${direct}" + ++ lib.optional (lib.hasInfix "hermes-backend-launch" direct) + "without waitFor the backend must not use the launcher" + + # The hostname mode polls the resolver, then binds the name. + ++ lib.optional (!lib.hasInfix "hermes-backend-launch" hostnameWait) + "waitFor = hostname must run the launcher, got: ${hostnameWait}" + ++ lib.optional (!lib.hasInfix "getent hosts" hostnameScript) + "the hostname launcher must poll with getent" + ++ lib.optional (!lib.hasInfix "host.example.ts.net" hostnameScript) + "the hostname launcher must poll for backend.host" + ++ lib.optional (!lib.hasInfix "exec " hostnameScript) + "the launcher must exec hermes, so that it keeps the MainPID" + ++ lib.optional (!lib.hasInfix ''--host "$_target"'' hostnameScript) + "the launcher must bind the address that the poll resolved" + + # The interface mode reads an address off the interface. + ++ lib.optional (!lib.hasInfix "tailscale0" interfaceScript) + "the interface launcher must poll backend.interfaceName" + ++ lib.optional (!lib.hasInfix "_timeout=30" interfaceScript) + "the launcher must use backend.waitTimeout" + ++ lib.optional (!lib.hasInfix "bin/hermes dashboard" interfaceScript) + "the launcher must keep backend.mode" + + # The assertions reject what cannot work. + ++ + lib.optional + (!evalFails { + backend = { + mode = "serve"; + waitFor = "interface"; + }; + }) + "an assertion must reject waitFor = interface without interfaceName" + ++ + lib.optional + (!evalFails { + backend = { + mode = "serve"; + interfaceName = "tailscale0"; + }; + }) + "an assertion must reject interfaceName without waitFor = interface"; + in + pkgs.runCommand "hermes-backend-bind-wait" { } ( + if failures != [ ] then + throw "backend bind wait check failed:\n${lib.concatMapStringsSep "\n" (f: " - ${f}") failures}" + else + '' + echo "PASS: backend bind wait (default, hostname, interface)" + mkdir -p $out + echo "ok" > $out/result + '' + ); + # ── How .env is built ──────────────────────────────────────────── # This check runs the real script that both modules use to build # $HERMES_HOME/.env. The important property is that a second run @@ -523,6 +626,9 @@ json.dump(sorted(leaf_paths(DEFAULT_CONFIG)), sys.stdout, indent=2) host = "127.0.0.1"; port = 9119; extraArgs = [ ]; + waitFor = null; + interfaceName = null; + waitTimeout = 120; }; }; sentinel = "--hermes-nix-argv-probe"; @@ -555,8 +661,8 @@ json.dump(sorted(leaf_paths(DEFAULT_CONFIG)), sys.stdout, indent=2) } check "gateway" ${probe (common.gatewayArgv (cfgFor "none"))} - check "serve" ${probe (common.backendArgv (cfgFor "serve"))} - check "dashboard" ${probe (common.backendArgv (cfgFor "dashboard"))} + check "serve" ${probe (common.backendArgv { inherit pkgs; cfg = cfgFor "serve"; })} + check "dashboard" ${probe (common.backendArgv { inherit pkgs; cfg = cfgFor "dashboard"; })} mkdir -p $out echo "ok" > $out/result diff --git a/nix/homeManagerModules.nix b/nix/homeManagerModules.nix index 55b934672c..efaf9d7af4 100644 --- a/nix/homeManagerModules.nix +++ b/nix/homeManagerModules.nix @@ -186,7 +186,19 @@ inherit cfg; opt = options.services.hermes-agent.workingDirectory; optionPath = "services.hermes-agent"; - }; + } + ++ common.backendBindAssertions { + inherit cfg; + optionPath = "services.hermes-agent"; + } + ++ [ + { + # The interface poll reads `ip`, which iproute2 supplies on + # Linux only. + assertion = !isDarwin || cfg.backend.waitFor != "interface"; + message = "services.hermes-agent.backend.waitFor = \"interface\" works on Linux only. Use \"hostname\" on Darwin."; + } + ]; } # ── Packages and interactive-shell environment ───────────────── @@ -237,7 +249,7 @@ (lib.mkIf (isLinux && cfg.backend.mode != "none") { systemd.user.services.hermes-backend = mkUnit { description = common.backendDescription cfg; - argv = common.backendArgv cfg; + argv = common.backendArgv { inherit pkgs cfg; }; }; }) @@ -251,7 +263,7 @@ (lib.mkIf (isDarwin && cfg.backend.mode != "none") { launchd.agents.hermes-backend = mkAgent { - argv = common.backendArgv cfg; + argv = common.backendArgv { inherit pkgs cfg; }; logName = "hermes-backend"; }; }) diff --git a/nix/moduleCommon.nix b/nix/moduleCommon.nix index c021209123..5ea5b50e2c 100644 --- a/nix/moduleCommon.nix +++ b/nix/moduleCommon.nix @@ -540,6 +540,70 @@ let header that is different from the address that the server bound to. This is a defence against DNS rebinding. Bind to the name or the address that your clients use. + + If the name or the address is not available when the unit starts, + set `waitFor` as well. + ''; + }; + + waitFor = mkOption { + type = types.nullOr ( + types.enum [ + "hostname" + "interface" + ] + ); + default = null; + description = '' + Wait for the bind target before the backend starts. + + The backend binds to `host` immediately by default. The bind fails + when the target is not ready, because uvicorn cannot bind a name + that does not resolve, or an address that no interface holds. A + unit that starts at boot can lose this race against the daemon + that supplies the target, such as tailscaled or a VPN client. + + A systemd user unit cannot order itself after a system unit. + `After=` and `Requires=` are silent no-ops across that boundary. + Thus the wait is a poll, and not a dependency. + + The values are: + + - `null` — bind immediately. `Restart=on-failure` retries the unit + until the target is ready. + - `"hostname"` — poll until `host` resolves, then bind to `host`. + Use this for a name, such as a Tailscale MagicDNS name. + - `"interface"` — poll until `interfaceName` has an IPv4 address, + then bind to that address. Use this when the address changes, + and a name for it does not exist. + + CAUTION: The `"interface"` value ignores `host`. The unit binds to + the address of the interface. + ''; + example = "hostname"; + }; + + interfaceName = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + The interface to take the bind address from. + + This option is necessary when `waitFor` is `"interface"`, and it + has no effect for the other values. + ''; + example = "tailscale0"; + }; + + waitTimeout = mkOption { + type = types.ints.positive; + default = 120; + description = '' + The time in seconds to wait for the bind target. + + The unit stops with an error after this time. It does not bind to + a different address, because a fallback address can expose the + backend more widely than you intend. ''; }; @@ -783,13 +847,14 @@ let ] ++ cfg.extraArgs; - backendArgv = - cfg: + # The command line of the backend, without the wait. + backendCommand = + cfg: host: [ "${effectivePackage cfg}/bin/hermes" cfg.backend.mode "--host" - cfg.backend.host + host "--port" (toString cfg.backend.port) # CAUTION: A service must not try to open a browser when it starts. @@ -797,6 +862,89 @@ let ] ++ cfg.backend.extraArgs; + # The launcher that waits for the bind target, then starts the backend. + # + # `exec` on the last line keeps hermes as the MainPID of the unit. No shell + # stays in the cgroup, and the restart logic of systemd sees the real + # process. + backendLauncher = + { pkgs, cfg }: + # The bind address is known only at start time, but escapeShellArgs quotes + # each argument. Thus the command line is built with a placeholder, and the + # placeholder becomes the shell variable after the quoting. + pkgs.writeShellScript "hermes-backend-launch" ( + builtins.replaceStrings [ "@HOST@" ] [ ''"$_target"'' ] '' + set -euo pipefail + + _timeout=${toString cfg.backend.waitTimeout} + _waited=0 + + ${ + if cfg.backend.waitFor == "hostname" then + '' + _target=${lib.escapeShellArg cfg.backend.host} + _how="hostname" + + while :; do + if ${pkgs.getent}/bin/getent hosts "$_target" >/dev/null 2>&1; then + break + fi + + if [ "$_waited" -ge "$_timeout" ]; then + echo "hermes-backend: '$_target' did not resolve after ''${_timeout}s. The unit stops." >&2 + exit 1 + fi + + if [ "$_waited" = 0 ]; then + echo "hermes-backend: waits for '$_target' to resolve..." >&2 + fi + ${pkgs.coreutils}/bin/sleep 2 + _waited=$(( _waited + 2 )) + done + '' + else + '' + _iface=${lib.escapeShellArg cfg.backend.interfaceName} + _how="interface $_iface" + + while :; do + _target="$(${pkgs.iproute2}/bin/ip -4 -oneline addr show dev "$_iface" 2>/dev/null \ + | ${pkgs.gawk}/bin/awk '{print $4}' \ + | ${pkgs.coreutils}/bin/cut -d/ -f1 \ + | ${pkgs.coreutils}/bin/head -n1 || true)" + + if [ -n "''${_target:-}" ]; then + break + fi + + if [ "$_waited" -ge "$_timeout" ]; then + echo "hermes-backend: interface '$_iface' had no IPv4 address after ''${_timeout}s. The unit stops." >&2 + echo "hermes-backend: a fallback address can expose the backend more widely than you intend." >&2 + exit 1 + fi + + if [ "$_waited" = 0 ]; then + echo "hermes-backend: waits for an IPv4 address on '$_iface'..." >&2 + fi + ${pkgs.coreutils}/bin/sleep 2 + _waited=$(( _waited + 2 )) + done + '' + } + + echo "hermes-backend: binds to $_target:${toString cfg.backend.port} (from $_how)" >&2 + + exec ${lib.escapeShellArgs (backendCommand cfg "@HOST@")} + '' + ); + + backendArgv = + { pkgs, cfg }: + if cfg.backend.waitFor == null then + backendCommand cfg cfg.backend.host + else + [ "${backendLauncher { inherit pkgs cfg; }}" ]; + backendDescription = cfg: if cfg.backend.mode == "dashboard" then @@ -884,6 +1032,20 @@ let } ]; + # The backend wait needs an interface name when it polls an interface. + backendBindAssertions = + { cfg, optionPath }: + [ + { + assertion = cfg.backend.waitFor != "interface" || cfg.backend.interfaceName != null; + message = "${optionPath}.backend.interfaceName must be set when backend.waitFor is \"interface\"."; + } + { + assertion = cfg.backend.waitFor == "interface" || cfg.backend.interfaceName == null; + message = "${optionPath}.backend.interfaceName has no effect unless backend.waitFor is \"interface\"."; + } + ]; + # The subdirectories of HERMES_HOME that both modules make. stateSubdirs = [ "cron" @@ -896,6 +1058,7 @@ in { inherit backendArgv + backendBindAssertions backendDescription deepConfigType effectivePackage diff --git a/nix/nixosModules.nix b/nix/nixosModules.nix index 38af2ec5ed..0a3182d640 100644 --- a/nix/nixosModules.nix +++ b/nix/nixosModules.nix @@ -381,6 +381,10 @@ opt = options.services.hermes-agent.workingDirectory; optionPath = "services.hermes-agent"; } + ++ common.backendBindAssertions { + inherit cfg; + optionPath = "services.hermes-agent"; + } ++ [ { # Container mode runs one command in one container. A second @@ -574,7 +578,7 @@ environment = commonUnitEnvironment; serviceConfig = commonServiceConfig // { - ExecStart = lib.escapeShellArgs (common.backendArgv cfg); + ExecStart = lib.escapeShellArgs (common.backendArgv { inherit pkgs cfg; }); }; path = unitPath;