From 87ce653d1d05a841c2ea1ad95b64f4eb01a6b503 Mon Sep 17 00:00:00 2001 From: teknium1 <127238744+teknium1@users.noreply.github.com> Date: Mon, 14 Sep 2026 21:35:57 -0700 Subject: [PATCH] feat(relay): migrate legacy HERMES_NEMO_RELAY_ATIF_*/ATOF_* vars into a validated relay-plugins.toml 3fad83df319e6 (Aug 11) moved Relay exporter config to a plugins.toml selected by HERMES_NEMO_RELAY_PLUGINS_TOML. A .env still carrying the legacy exporter vars and no TOML logs ONE warning and initialises no exporters, so users who followed the earlier docs lost every trace silently (the maintainer's stopped Aug 20, noticed Sep 14; five multiplexed profiles on the same box carry the same eight vars today). - `hermes_cli/relay_plugin_migrate.py`: build the document from the `nemo_relay.observability` dataclasses (`ComponentSpec(...).to_dict()`, so the `type = "file"` sink discriminator is emitted), validate it by activating it through `nemo_relay.plugin.initialize` + `clear_async`, write `/relay-plugins.toml` (tomli_w when installed, minimal emitter otherwise), set HERMES_NEMO_RELAY_PLUGINS_TOML in that .env, and comment the legacy lines out (never delete). Defaults mirror the removed plugin so files land where they used to. - `hermes update` runs it for the default home AND every live named profile (each writes its own TOML) as a best-effort post-update step, with a loud notice; `hermes migrate relay [--all-profiles] [--no-validate]` runs it on demand. - The runtime WARNING and the `hermes doctor` finding now say "NO traces are being exported" and name the exact command and file path. - Docs: environment-variables.md + built-in-plugins.md carry the migration note and a complete plugins.toml example including `type = "file"`. --- agent/relay_runtime.py | 6 ++- hermes_cli/doctor_config.py | 3 +- hermes_cli/subcommands/migrate.py | 19 +++++++++ hermes_cli/update_cmd_maint.py | 8 ++++ tests/hermes_cli/test_relay_plugin_migrate.py | 16 ++++---- .../docs/reference/environment-variables.md | 2 +- .../user-guide/features/built-in-plugins.md | 39 ++++++++++++++++++- 7 files changed, 80 insertions(+), 13 deletions(-) diff --git a/agent/relay_runtime.py b/agent/relay_runtime.py index 7c0b78d2ea..e8bd26e7d0 100644 --- a/agent/relay_runtime.py +++ b/agent/relay_runtime.py @@ -1219,9 +1219,11 @@ def _configured_plugin_inputs(relay: Any) -> tuple[dict[str, Any], list[Any]] | if not configured: if legacy_vars := configured_legacy_relay_env_vars(os.environ): logger.warning( - "Legacy NeMo Relay exporter variables are set but no %s was provided. %s no longer activate " - "Relay exporters; migrate the exporter configuration to a Relay plugins.toml file.", + "Legacy NeMo Relay exporter variables are set but no %s was provided — NO traces are being " + "exported. %s no longer activate Relay exporters. Run `hermes migrate relay` (or `hermes update`, " + "which runs it for every profile) to generate %s from them and select it in .env.", RELAY_PLUGINS_CONFIG_ENV, ", ".join(legacy_vars), + get_hermes_home() / "relay-plugins.toml", ) return None config_path = Path(configured).expanduser() diff --git a/hermes_cli/doctor_config.py b/hermes_cli/doctor_config.py index ffe30b665c..ddc0e4fb46 100644 --- a/hermes_cli/doctor_config.py +++ b/hermes_cli/doctor_config.py @@ -72,7 +72,8 @@ def collect_relay_plugin_cutover_findings(raw_config: dict | None, env_map: dict if name not in effective_env and os.environ.get(name) is not None: effective_env[name] = os.environ[name] if not str(effective_env.get(RELAY_PLUGINS_CONFIG_ENV, "")).strip(): - findings += [(name, f"move exporter settings to {RELAY_PLUGINS_CONFIG_ENV}; this variable is now ignored") + findings += [(name, f"run `hermes migrate relay` to generate relay-plugins.toml and set {RELAY_PLUGINS_CONFIG_ENV}; " + "this variable is now ignored and no traces are exported") for name in configured_legacy_relay_env_vars(effective_env)] return findings diff --git a/hermes_cli/subcommands/migrate.py b/hermes_cli/subcommands/migrate.py index 432cda08cb..f65be647cf 100644 --- a/hermes_cli/subcommands/migrate.py +++ b/hermes_cli/subcommands/migrate.py @@ -26,4 +26,23 @@ def build_migrate_parser(subparsers) -> None: "--no-backup", action="store_true", help="Skip the timestamped backup of config.yaml when applying") migrate_xai.set_defaults(func=cmd_migrate_xai) + + migrate_relay = migrate_subparsers.add_parser( + "relay", help="Convert legacy HERMES_NEMO_RELAY_ATIF_*/ATOF_* exporter vars into relay-plugins.toml", + description="The NeMo Relay cutover stopped reading the legacy exporter variables; a .env that still " + "carries them (and no HERMES_NEMO_RELAY_PLUGINS_TOML) exports nothing. Generate " + "/relay-plugins.toml from them, point HERMES_NEMO_RELAY_PLUGINS_TOML at it, " + "and comment the legacy lines out. `hermes update` runs this for every profile automatically.") + migrate_relay.add_argument( + "--all-profiles", action="store_true", + help="Migrate the default home and every named profile (what `hermes update` does)") + migrate_relay.add_argument( + "--no-validate", action="store_true", + help="Skip activating the generated file through Relay's validator before writing it") + migrate_relay.set_defaults(func=_cmd_migrate_relay) migrate_parser.set_defaults(func=cmd_migrate) + + +def _cmd_migrate_relay(args) -> None: + from hermes_cli.relay_plugin_migrate import cmd_migrate_relay + cmd_migrate_relay(args) diff --git a/hermes_cli/update_cmd_maint.py b/hermes_cli/update_cmd_maint.py index a2579f28a9..1107863617 100644 --- a/hermes_cli/update_cmd_maint.py +++ b/hermes_cli/update_cmd_maint.py @@ -964,11 +964,19 @@ def _print_post_update_notices_and_self_heals() -> None: ('Windows bin launcher migration failed: %s', _migrate_windows_bin_path), ('cua-driver refresh failed: %s', _refresh_cua_driver_after_update), ('Plugin compat notice failed: %s', _print_plugin_compat_notice), + # Legacy HERMES_NEMO_RELAY_ATIF_*/ATOF_* vars produce no traces since the Relay cutover; + # generate each profile's relay-plugins.toml instead of leaving exports silently dead. + ('Relay exporter migration failed: %s', _migrate_relay_exporter_env), ): with _best_effort(message): step() +def _migrate_relay_exporter_env() -> None: + from hermes_cli.relay_plugin_migrate import run_relay_migration_after_update + run_relay_migration_after_update() + + def _run_post_update_maintenance( *, assume_yes, gateway_mode, pre_update_snapshot_id, had_desktop_app_before_update, node_failures, desktop_build_ok, pre_update_version, diff --git a/tests/hermes_cli/test_relay_plugin_migrate.py b/tests/hermes_cli/test_relay_plugin_migrate.py index 73209df1a3..c5cd2bbc6f 100644 --- a/tests/hermes_cli/test_relay_plugin_migrate.py +++ b/tests/hermes_cli/test_relay_plugin_migrate.py @@ -29,7 +29,7 @@ HERMES_NEMO_RELAY_ATIF_SUBAGENT_EXPORT_MODE=all def _parse_env(path: Path) -> dict[str, str]: out = {} - for line in path.read_text().splitlines(): + for line in path.read_text(encoding="utf-8").splitlines(): if line.strip() and not line.startswith("#") and "=" in line: k, _, v = line.partition("=") out[k.strip()] = v.strip().strip('"') @@ -46,14 +46,14 @@ def profile_env(tmp_path, monkeypatch): def test_legacy_env_becomes_validated_toml_selected_from_env(profile_env): - (profile_env / ".env").write_text(LEGACY_ENV.format(home=profile_env)) + (profile_env / ".env").write_text(LEGACY_ENV.format(home=profile_env), encoding="utf-8") result = migrate_profile_relay_env(profile_env) assert result.migrated, (result.skipped_reason, result.validation_error) assert result.diagnostics == [] toml_path = profile_env / RELAY_PLUGINS_TOML_NAME - document = tomllib.loads(toml_path.read_text()) + document = tomllib.loads(toml_path.read_text(encoding="utf-8")) sink = document["components"][0]["config"]["atof"]["sinks"][0] assert sink["type"] == "file" and sink["filename"] == "hermes-atof.jsonl" assert document["components"][0]["config"]["atif"]["filename_template"] == "trajectory-{session_id}.json" @@ -67,19 +67,19 @@ def test_legacy_env_becomes_validated_toml_selected_from_env(profile_env): assert env[RELAY_PLUGINS_CONFIG_ENV] == str(toml_path) assert env["OPENAI_API_KEY"] == "sk-test" assert configured_legacy_relay_env_vars(env) == () - assert "# migrated to relay-plugins.toml: HERMES_NEMO_RELAY_ATOF_ENABLED=1" in (profile_env / ".env").read_text() + assert "# migrated to relay-plugins.toml: HERMES_NEMO_RELAY_ATOF_ENABLED=1" in (profile_env / ".env").read_text(encoding="utf-8") assert migrate_profile_relay_env(profile_env).skipped_reason == "no legacy exporter variables" def test_update_migrates_every_profile_home_separately(profile_env): """Multiplex: each profile keeps its own TOML; a profile without legacy vars is untouched.""" - (profile_env / ".env").write_text("OPENAI_API_KEY=x\n") + (profile_env / ".env").write_text("OPENAI_API_KEY=x\n", encoding="utf-8") work = profile_env / "profiles" / "work" idle = profile_env / "profiles" / "idle" for p in (work, idle): p.mkdir(parents=True) - (work / ".env").write_text(LEGACY_ENV.format(home=work)) - (idle / ".env").write_text("SLACK_BOT_TOKEN=y\n") + (work / ".env").write_text(LEGACY_ENV.format(home=work), encoding="utf-8") + (idle / ".env").write_text("SLACK_BOT_TOKEN=y\n", encoding="utf-8") results = {r.home: r for r in migrate_all_profile_relay_envs(validate=False)} @@ -87,4 +87,4 @@ def test_update_migrates_every_profile_home_separately(profile_env): assert not results[idle].migrated and not (idle / RELAY_PLUGINS_TOML_NAME).exists() assert not results[profile_env].migrated and not (profile_env / RELAY_PLUGINS_TOML_NAME).exists() assert _parse_env(work / ".env")[RELAY_PLUGINS_CONFIG_ENV] == str(work / RELAY_PLUGINS_TOML_NAME) - assert (idle / ".env").read_text() == "SLACK_BOT_TOKEN=y\n" + assert (idle / ".env").read_text(encoding="utf-8") == "SLACK_BOT_TOKEN=y\n" diff --git a/website/docs/reference/environment-variables.md b/website/docs/reference/environment-variables.md index 4844ab579c..b7c6cdef9e 100644 --- a/website/docs/reference/environment-variables.md +++ b/website/docs/reference/environment-variables.md @@ -797,7 +797,7 @@ Advanced per-platform knobs for throttling the outbound message batcher. Most us | Variable | Description | |----------|-------------| -| `HERMES_NEMO_RELAY_PLUGINS_TOML` | Explicit path to the standard NeMo Relay `plugins.toml` loaded process-wide by Hermes core. When unset, Hermes does not initialize Relay middleware, dynamic plugins, or exporters. The removed `HERMES_NEMO_RELAY_ATOF_*` and `HERMES_NEMO_RELAY_ATIF_*` variables are ignored; configure those outputs in the selected file instead. See [NeMo Relay observability configuration](https://docs.nvidia.com/nemo/relay/configure-plugins/observability/about). | +| `HERMES_NEMO_RELAY_PLUGINS_TOML` | Explicit path to the standard NeMo Relay `plugins.toml` loaded process-wide by Hermes core. When unset, Hermes does not initialize Relay middleware, dynamic plugins, or exporters. The removed `HERMES_NEMO_RELAY_ATOF_*` and `HERMES_NEMO_RELAY_ATIF_*` variables are ignored (a `.env` that still carries them exports nothing); `hermes update` / `hermes migrate relay` converts them into `/relay-plugins.toml` and sets this variable — see the [migration note and full example](../user-guide/features/built-in-plugins.md#nemo-relay-native-integration-migration-note). See [NeMo Relay observability configuration](https://docs.nvidia.com/nemo/relay/configure-plugins/observability/about). | ## Agent Behavior diff --git a/website/docs/user-guide/features/built-in-plugins.md b/website/docs/user-guide/features/built-in-plugins.md index 2114963402..7f7a24475e 100644 --- a/website/docs/user-guide/features/built-in-plugins.md +++ b/website/docs/user-guide/features/built-in-plugins.md @@ -208,7 +208,44 @@ NeMo Relay is no longer a bundled Hermes plugin. Do not run `hermes plugins enab To opt into Relay middleware or exporters, create a standard Relay `plugins.toml`, then set `HERMES_NEMO_RELAY_PLUGINS_TOML` to that file before starting Hermes. The policy is process-wide for every profile hosted by that Hermes process. See the [NeMo Relay observability configuration](https://docs.nvidia.com/nemo/relay/configure-plugins/observability/about) for ATOF, ATIF, and OpenTelemetry options. -The old `HERMES_NEMO_RELAY_ATOF_*` and `HERMES_NEMO_RELAY_ATIF_*` settings no longer activate exporters. `hermes doctor` reports these stale settings when no replacement `plugins.toml` is selected. +The old `HERMES_NEMO_RELAY_ATOF_*` and `HERMES_NEMO_RELAY_ATIF_*` settings no longer activate exporters — a `.env` that still carries them (and no `HERMES_NEMO_RELAY_PLUGINS_TOML`) exports **nothing**, and the gateway logs one warning saying so. `hermes doctor` reports these stale settings when no replacement `plugins.toml` is selected. + +**Automatic migration.** `hermes update` (and `hermes migrate relay`, or `hermes migrate relay --all-profiles` for every profile home) converts the legacy variables into `/relay-plugins.toml`, sets `HERMES_NEMO_RELAY_PLUGINS_TOML` in that profile's `.env`, and comments the legacy lines out (nothing is deleted). Under a multiplexed gateway every profile home gets its own file. The generated file is validated through Relay before it is written; this is the shape it produces (note the `type = "file"` sink discriminator — a sink without it is rejected): + +```toml +version = 1 + +[[components]] +kind = "observability" +enabled = true + +[components.config] +version = 4 +enable_full_payloads = false + +[components.config.atof] +enabled = true + +[[components.config.atof.sinks]] +type = "file" +output_directory = "/home/you/.hermes/telemetry/nemo-relay/atof" +filename = "hermes-atof.jsonl" +mode = "append" + +[components.config.atif] +enabled = true +agent_name = "Hermes Agent" +model_name = "unknown" +output_directory = "/home/you/.hermes/telemetry/nemo-relay/atif" +filename_template = "trajectory-{session_id}.json" + +[components.config.policy] +unknown_component = "warn" +unknown_field = "warn" +unsupported_value = "error" +``` + +Then add `HERMES_NEMO_RELAY_PLUGINS_TOML=/home/you/.hermes/relay-plugins.toml` to `.env` and restart the gateway. #### Session-span segmentation (continuous sessions)