ouyangbo/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

feat(nix): wait for the backend bind target before it starts

Browse Source 
The backend binds to `backend.host` immediately. The bind fails when the
target is not ready, because uvicorn cannot bind a name that does not
resolve, or an address that no interface holds. A unit that starts at boot
loses this race against the daemon that supplies the target, such as
tailscaled.

A bind to a Tailscale MagicDNS name shows the problem. The name is the
correct bind target, because the dashboard refuses each request with a Host
header that is different from the address that the server bound to, and a
shared machine has a different address in each tailnet. But the name does
not resolve until tailscaled is up, so the unit fails at each boot until
`Restart=on-failure` finds the moment when the name works.

A systemd user unit cannot order itself after a system unit. `After=` and
`Requires=` are silent no-ops across that boundary. Thus the wait is a poll,
and not a dependency.

This change adds three options to `services.hermes-agent.backend` on both
the NixOS module and the Home Manager module:

- `waitFor` — `null` (the default, unchanged behavior), `"hostname"`, or
  `"interface"`
- `interfaceName` — the interface to take the address from
- `waitTimeout` — the time in seconds before the unit stops

With `waitFor`, ExecStart becomes a launcher that polls for the target and
then execs hermes. `exec` keeps hermes as the MainPID, so the restart logic
of systemd sees the real process. A timeout stops the unit with an error. It
does not bind a fallback address, because a fallback can expose the backend
more widely than the user intends.

The default is not changed. Without `waitFor`, ExecStart is the same
command line as before.
This commit is contained in:
ethernet
2026-08-21 14:05:18 -04:00
parent f7a5c9e520
commit fd3a783a3e
4 changed files with 294 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nix/checks.nix
+108 -2
View File
@@ -435,6 +435,109 @@ json.dump(sorted(leaf_paths(DEFAULT_CONFIG)), sys.stdout, indent=2)
''
);
# ── How the backend waits for its bind target ────────────────────
# The backend binds to `host` immediately by default. A unit that
# starts at boot can lose the race against the daemon that supplies
# the address, such as tailscaled. `backend.waitFor` puts a poll in
# front of the bind. This check proves three properties: the default
# keeps the direct command line, each wait mode makes a launcher that
# polls and then execs hermes, and the assertions reject a
# configuration that cannot work.
backend-bind-wait =
let
execOf =
settings:
(evalNixosModule ({ enable = true; } // settings)).config.systemd.services.hermes-backend.serviceConfig.ExecStart;
direct = execOf { backend.mode = "serve"; };
hostnameWait = execOf {
backend = {
mode = "serve";
host = "host.example.ts.net";
waitFor = "hostname";
};
};
interfaceWait = execOf {
backend = {
mode = "dashboard";
waitFor = "interface";
interfaceName = "tailscale0";
waitTimeout = 30;
};
};
# The launcher is a store path. Read it to see what it runs.
hostnameScript = builtins.readFile hostnameWait;
interfaceScript = builtins.readFile interfaceWait;
evalFails =
settings:
!(builtins.tryEval (
lib.deepSeq
(evalNixosModule ({ enable = true; } // settings)).config.system.build.toplevel.drvPath
true
)).success;
failures =
# The default must not change.
lib.optional (!lib.hasInfix "bin/hermes serve --host 127.0.0.1" direct)
"without waitFor the backend must exec hermes directly, got: ${direct}"
++ lib.optional (lib.hasInfix "hermes-backend-launch" direct)
"without waitFor the backend must not use the launcher"
# The hostname mode polls the resolver, then binds the name.
++ lib.optional (!lib.hasInfix "hermes-backend-launch" hostnameWait)
"waitFor = hostname must run the launcher, got: ${hostnameWait}"
++ lib.optional (!lib.hasInfix "getent hosts" hostnameScript)
"the hostname launcher must poll with getent"
++ lib.optional (!lib.hasInfix "host.example.ts.net" hostnameScript)
"the hostname launcher must poll for backend.host"
++ lib.optional (!lib.hasInfix "exec " hostnameScript)
"the launcher must exec hermes, so that it keeps the MainPID"
++ lib.optional (!lib.hasInfix ''--host "$_target"'' hostnameScript)
"the launcher must bind the address that the poll resolved"
# The interface mode reads an address off the interface.
++ lib.optional (!lib.hasInfix "tailscale0" interfaceScript)
"the interface launcher must poll backend.interfaceName"
++ lib.optional (!lib.hasInfix "_timeout=30" interfaceScript)
"the launcher must use backend.waitTimeout"
++ lib.optional (!lib.hasInfix "bin/hermes dashboard" interfaceScript)
"the launcher must keep backend.mode"
# The assertions reject what cannot work.
++
lib.optional
(!evalFails {
backend = {
mode = "serve";
waitFor = "interface";
};
})
"an assertion must reject waitFor = interface without interfaceName"
++
lib.optional
(!evalFails {
backend = {
mode = "serve";
interfaceName = "tailscale0";
};
})
"an assertion must reject interfaceName without waitFor = interface";
in
pkgs.runCommand "hermes-backend-bind-wait" { } (
if failures != [ ] then
throw "backend bind wait check failed:\n${lib.concatMapStringsSep "\n" (f: " - ${f}") failures}"
else
''
echo "PASS: backend bind wait (default, hostname, interface)"
mkdir -p $out
echo "ok" > $out/result
''
);
# ── How .env is built ────────────────────────────────────────────
# This check runs the real script that both modules use to build
# $HERMES_HOME/.env. The important property is that a second run
@@ -523,6 +626,9 @@ json.dump(sorted(leaf_paths(DEFAULT_CONFIG)), sys.stdout, indent=2)
host = "127.0.0.1";
port = 9119;
extraArgs = [ ];
waitFor = null;
interfaceName = null;
waitTimeout = 120;
};
};
sentinel = "--hermes-nix-argv-probe";
@@ -555,8 +661,8 @@ json.dump(sorted(leaf_paths(DEFAULT_CONFIG)), sys.stdout, indent=2)
}
check "gateway" ${probe (common.gatewayArgv (cfgFor "none"))}
check "serve" ${probe (common.backendArgv (cfgFor "serve"))}
check "dashboard" ${probe (common.backendArgv (cfgFor "dashboard"))}
check "serve" ${probe (common.backendArgv { inherit pkgs; cfg = cfgFor "serve"; })}
check "dashboard" ${probe (common.backendArgv { inherit pkgs; cfg = cfgFor "dashboard"; })}
mkdir -p $out
echo "ok" > $out/result
nix/homeManagerModules.nix
+15 -3
View File
@@ -186,7 +186,19 @@
inherit cfg;
opt = options.services.hermes-agent.workingDirectory;
optionPath = "services.hermes-agent";
};
}
++ common.backendBindAssertions {
inherit cfg;
optionPath = "services.hermes-agent";
}
++ [
{
# The interface poll reads `ip`, which iproute2 supplies on
# Linux only.
assertion = !isDarwin || cfg.backend.waitFor != "interface";
message = "services.hermes-agent.backend.waitFor = \"interface\" works on Linux only. Use \"hostname\" on Darwin.";
}
];
}
# ── Packages and interactive-shell environment ─────────────────
@@ -237,7 +249,7 @@
(lib.mkIf (isLinux && cfg.backend.mode != "none") {
systemd.user.services.hermes-backend = mkUnit {
description = common.backendDescription cfg;
argv = common.backendArgv cfg;
argv = common.backendArgv { inherit pkgs cfg; };
};
})
@@ -251,7 +263,7 @@
(lib.mkIf (isDarwin && cfg.backend.mode != "none") {
launchd.agents.hermes-backend = mkAgent {
argv = common.backendArgv cfg;
argv = common.backendArgv { inherit pkgs cfg; };
logName = "hermes-backend";
};
})
nix/moduleCommon.nix
+166 -3
View File
@@ -540,6 +540,70 @@ let
header that is different from the address that the server bound
to. This is a defence against DNS rebinding. Bind to the name or
the address that your clients use.
If the name or the address is not available when the unit starts,
set `waitFor` as well.
'';
};
waitFor = mkOption {
type = types.nullOr (
types.enum [
"hostname"
"interface"
]
);
default = null;
description = ''
Wait for the bind target before the backend starts.
The backend binds to `host` immediately by default. The bind fails
when the target is not ready, because uvicorn cannot bind a name
that does not resolve, or an address that no interface holds. A
unit that starts at boot can lose this race against the daemon
that supplies the target, such as tailscaled or a VPN client.
A systemd user unit cannot order itself after a system unit.
`After=` and `Requires=` are silent no-ops across that boundary.
Thus the wait is a poll, and not a dependency.
The values are:
- `null` — bind immediately. `Restart=on-failure` retries the unit
until the target is ready.
- `"hostname"` — poll until `host` resolves, then bind to `host`.
Use this for a name, such as a Tailscale MagicDNS name.
- `"interface"` — poll until `interfaceName` has an IPv4 address,
then bind to that address. Use this when the address changes,
and a name for it does not exist.
CAUTION: The `"interface"` value ignores `host`. The unit binds to
the address of the interface.
'';
example = "hostname";
};
interfaceName = mkOption {
type = types.nullOr types.str;
default = null;
description = ''
The interface to take the bind address from.
This option is necessary when `waitFor` is `"interface"`, and it
has no effect for the other values.
'';
example = "tailscale0";
};
waitTimeout = mkOption {
type = types.ints.positive;
default = 120;
description = ''
The time in seconds to wait for the bind target.
The unit stops with an error after this time. It does not bind to
a different address, because a fallback address can expose the
backend more widely than you intend.
'';
};
@@ -783,13 +847,14 @@ let
]
++ cfg.extraArgs;
backendArgv =
cfg:
# The command line of the backend, without the wait.
backendCommand =
cfg: host:
[
"${effectivePackage cfg}/bin/hermes"
cfg.backend.mode
"--host"
cfg.backend.host
host
"--port"
(toString cfg.backend.port)
# CAUTION: A service must not try to open a browser when it starts.
@@ -797,6 +862,89 @@ let
]
++ cfg.backend.extraArgs;
# The launcher that waits for the bind target, then starts the backend.
#
# `exec` on the last line keeps hermes as the MainPID of the unit. No shell
# stays in the cgroup, and the restart logic of systemd sees the real
# process.
backendLauncher =
{ pkgs, cfg }:
# The bind address is known only at start time, but escapeShellArgs quotes
# each argument. Thus the command line is built with a placeholder, and the
# placeholder becomes the shell variable after the quoting.
pkgs.writeShellScript "hermes-backend-launch" (
builtins.replaceStrings [ "@HOST@" ] [ ''"$_target"'' ] ''
set -euo pipefail
_timeout=${toString cfg.backend.waitTimeout}
_waited=0
${
if cfg.backend.waitFor == "hostname" then
''
_target=${lib.escapeShellArg cfg.backend.host}
_how="hostname"
while :; do
if ${pkgs.getent}/bin/getent hosts "$_target" >/dev/null 2>&1; then
break
fi
if [ "$_waited" -ge "$_timeout" ]; then
echo "hermes-backend: '$_target' did not resolve after ''${_timeout}s. The unit stops." >&2
exit 1
fi
if [ "$_waited" = 0 ]; then
echo "hermes-backend: waits for '$_target' to resolve..." >&2
fi
${pkgs.coreutils}/bin/sleep 2
_waited=$(( _waited + 2 ))
done
''
else
''
_iface=${lib.escapeShellArg cfg.backend.interfaceName}
_how="interface $_iface"
while :; do
_target="$(${pkgs.iproute2}/bin/ip -4 -oneline addr show dev "$_iface" 2>/dev/null \
| ${pkgs.gawk}/bin/awk '{print $4}' \
| ${pkgs.coreutils}/bin/cut -d/ -f1 \
| ${pkgs.coreutils}/bin/head -n1 || true)"
if [ -n "''${_target:-}" ]; then
break
fi
if [ "$_waited" -ge "$_timeout" ]; then
echo "hermes-backend: interface '$_iface' had no IPv4 address after ''${_timeout}s. The unit stops." >&2
echo "hermes-backend: a fallback address can expose the backend more widely than you intend." >&2
exit 1
fi
if [ "$_waited" = 0 ]; then
echo "hermes-backend: waits for an IPv4 address on '$_iface'..." >&2
fi
${pkgs.coreutils}/bin/sleep 2
_waited=$(( _waited + 2 ))
done
''
}
echo "hermes-backend: binds to $_target:${toString cfg.backend.port} (from $_how)" >&2
exec ${lib.escapeShellArgs (backendCommand cfg "@HOST@")}
''
);
backendArgv =
{ pkgs, cfg }:
if cfg.backend.waitFor == null then
backendCommand cfg cfg.backend.host
else
[ "${backendLauncher { inherit pkgs cfg; }}" ];
backendDescription =
cfg:
if cfg.backend.mode == "dashboard" then
@@ -884,6 +1032,20 @@ let
}
];
# The backend wait needs an interface name when it polls an interface.
backendBindAssertions =
{ cfg, optionPath }:
[
{
assertion = cfg.backend.waitFor != "interface" || cfg.backend.interfaceName != null;
message = "${optionPath}.backend.interfaceName must be set when backend.waitFor is \"interface\".";
}
{
assertion = cfg.backend.waitFor == "interface" || cfg.backend.interfaceName == null;
message = "${optionPath}.backend.interfaceName has no effect unless backend.waitFor is \"interface\".";
}
];
# The subdirectories of HERMES_HOME that both modules make.
stateSubdirs = [
"cron"
@@ -896,6 +1058,7 @@ in
{
inherit
backendArgv
backendBindAssertions
backendDescription
deepConfigType
effectivePackage
nix/nixosModules.nix
+5 -1
View File
@@ -381,6 +381,10 @@
opt = options.services.hermes-agent.workingDirectory;
optionPath = "services.hermes-agent";
}
++ common.backendBindAssertions {
inherit cfg;
optionPath = "services.hermes-agent";
}
++ [
{
# Container mode runs one command in one container. A second
@@ -574,7 +578,7 @@
environment = commonUnitEnvironment;
serviceConfig = commonServiceConfig // {
ExecStart = lib.escapeShellArgs (common.backendArgv cfg);
ExecStart = lib.escapeShellArgs (common.backendArgv { inherit pkgs cfg; });
};
path = unitPath;