From 43d29a37c8f040bea4936beacc43e7059d772f4b Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Fri, 31 Jul 2026 06:58:41 -0700 Subject: [PATCH] fix(observability): include auxiliary model routes Signed-off-by: Alex Fournier --- agent/auxiliary_client.py | 21 +++ agent/relay_llm.py | 87 +++++++++-- .../observability/shared_metrics_contract.py | 58 ++++++- tests/agent/test_auxiliary_relay.py | 146 ++++++++++++++++-- tests/hermes_cli/test_relay_shared_metrics.py | 38 +++++ 5 files changed, 324 insertions(+), 26 deletions(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index acd22d8848..4694de9131 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -2522,6 +2522,7 @@ def _relay_auxiliary_call(callback): "attempt_count": 0, "provider": "", "model": "", + "response_model": None, "api_mode": "chat_completions", }) try: @@ -2547,6 +2548,7 @@ def _relay_auxiliary_call_async(callback): "attempt_count": 0, "provider": "", "model": "", + "response_model": None, "api_mode": "chat_completions", }) try: @@ -2570,6 +2572,7 @@ def _set_relay_auxiliary_route( return context["provider"] = str(provider or "auxiliary") context["model"] = str(model or "unknown") + context["response_model"] = None context["api_mode"] = str(api_mode or "chat_completions") @@ -7474,6 +7477,7 @@ def _validate_llm_response( except (AttributeError, TypeError, IndexError) as exc: recovered = _recover_aux_response_message(response) if recovered is not None: + _record_relay_auxiliary_response_model(response) _complete_relay_auxiliary_call() return recovered response_type = type(response).__name__ @@ -7484,6 +7488,7 @@ def _validate_llm_response( f"Expected object with .choices[0].message — check provider " f"adapter or custom endpoint compatibility." ) from exc + _record_relay_auxiliary_response_model(response) _complete_relay_auxiliary_call() return response @@ -7498,9 +7503,25 @@ def _complete_relay_auxiliary_call(*, outcome: str = "success") -> None: relay_llm.complete_logical_call( str(context.get("request_id") or ""), outcome=outcome, + model_name=str(context.get("model") or "unknown"), + provider_name=str(context.get("provider") or "auxiliary"), + response_model_name=context.get("response_model"), ) +def _record_relay_auxiliary_response_model(response: Any) -> None: + """Retain the provider-reported model for terminal route attribution.""" + context = _RELAY_AUX_CALL_CONTEXT.get() + if context is None: + return + if isinstance(response, dict): + model = response.get("model") + else: + model = getattr(response, "model", None) + if isinstance(model, str) and model.strip(): + context["response_model"] = model + + def _fail_relay_auxiliary_call() -> None: """Close a terminally failed call without replacing its original error.""" try: diff --git a/agent/relay_llm.py b/agent/relay_llm.py index 3481ca0723..535925f9a6 100644 --- a/agent/relay_llm.py +++ b/agent/relay_llm.py @@ -336,6 +336,14 @@ class ManagedLlmStream(Iterator[Any]): self._callback_error: BaseException | None = None self._logical: tuple[relay_runtime.RelayTurnContext, Any, str] | None = None self._defer_logical_completion = defer_logical_completion + if str((metadata or {}).get("call_role") or "").startswith("auxiliary:"): + self._logical_model_name: str | None = model_name + self._logical_provider_name: str | None = name + self._logical_response_model_name: str | None = None + else: + self._logical_model_name = None + self._logical_provider_name = None + self._logical_response_model_name = None self._on_chunk = on_chunk self._chunk_adapter = chunk_adapter or _namespace self._accept_chunk = accept_chunk @@ -445,8 +453,12 @@ class ManagedLlmStream(Iterator[Any]): return None try: if self.final_response is not None: - return _jsonable(self.final_response) - return _jsonable(run_callback(finalizer)) + response = self.final_response + else: + response = run_callback(finalizer) + if self._logical_model_name is not None: + self._logical_response_model_name = _response_model_name(response) + return _jsonable(response) except BaseException as exc: self._callback_error = exc raise @@ -488,6 +500,9 @@ class ManagedLlmStream(Iterator[Any]): _complete_logical( self._logical, outcome="cancelled" if _is_cancellation(exc) else "failed", + model_name=self._logical_model_name, + provider_name=self._logical_provider_name, + response_model_name=self._logical_response_model_name, ) self._logical = None loop.close() @@ -520,7 +535,13 @@ class ManagedLlmStream(Iterator[Any]): if self._raw_chunks: self.output_modified = True if not self._defer_logical_completion: - _complete_logical(self._logical, outcome="success") + _complete_logical( + self._logical, + outcome="success", + model_name=self._logical_model_name, + provider_name=self._logical_provider_name, + response_model_name=self._logical_response_model_name, + ) self._logical = None self._close(logical_outcome="cancelled") raise StopIteration from None @@ -593,7 +614,13 @@ class ManagedLlmStream(Iterator[Any]): ) loop.close() if not self._defer_logical_completion: - _complete_logical(self._logical, outcome="success") + _complete_logical( + self._logical, + outcome="success", + model_name=self._logical_model_name, + provider_name=self._logical_provider_name, + response_model_name=self._logical_response_model_name, + ) self._logical = None def _close(self, *, logical_outcome: str) -> None: @@ -623,7 +650,13 @@ class ManagedLlmStream(Iterator[Any]): exc_info=True, ) if not self._defer_logical_completion: - _complete_logical(self._logical, outcome=logical_outcome) + _complete_logical( + self._logical, + outcome=logical_outcome, + model_name=self._logical_model_name, + provider_name=self._logical_provider_name, + response_model_name=self._logical_response_model_name, + ) self._logical = None return close = getattr(self._stream, "aclose", None) @@ -638,7 +671,13 @@ class ManagedLlmStream(Iterator[Any]): if self._close_error is None: self._close_error = exc if not self._defer_logical_completion: - _complete_logical(self._logical, outcome=logical_outcome) + _complete_logical( + self._logical, + outcome=logical_outcome, + model_name=self._logical_model_name, + provider_name=self._logical_provider_name, + response_model_name=self._logical_response_model_name, + ) self._logical = None loop.close() @@ -777,6 +816,9 @@ def _complete_logical( logical: tuple[relay_runtime.RelayTurnContext, Any, str] | None, *, outcome: str, + model_name: str | None = None, + provider_name: str | None = None, + response_model_name: str | None = None, ) -> None: if logical is None: return @@ -791,11 +833,16 @@ def _complete_logical( if lease.session is None: return try: + output = {"outcome": outcome} + if model_name is not None and provider_name is not None: + output.update({"model": model_name, "provider": provider_name}) + if response_model_name is not None: + output["response_model"] = response_model_name lease.host.run_in_session( lease.session, lease.host.relay.scope.pop, handle, - output={"outcome": outcome}, + output=output, metadata={ relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION, relay_runtime.RUNTIME_INSTANCE_KEY: lease.host.runtime_id, @@ -845,7 +892,14 @@ def _is_cancellation(error: BaseException) -> bool: ) -def complete_logical_call(api_request_id: str, *, outcome: str) -> None: +def complete_logical_call( + api_request_id: str, + *, + outcome: str, + model_name: str | None = None, + provider_name: str | None = None, + response_model_name: str | None = None, +) -> None: """Complete the active turn's logical LLM call after caller validation.""" turn = relay_runtime.active_turn() if turn is None or not api_request_id: @@ -853,7 +907,22 @@ def complete_logical_call(api_request_id: str, *, outcome: str) -> None: with turn.logical_llm_lock: handle = turn.logical_llm_calls.get(api_request_id) if handle is not None: - _complete_logical((turn, handle, api_request_id), outcome=outcome) + _complete_logical( + (turn, handle, api_request_id), + outcome=outcome, + model_name=model_name, + provider_name=provider_name, + response_model_name=response_model_name, + ) + + +def _response_model_name(response: Any) -> str | None: + """Return a provider-reported model name when one is available.""" + if isinstance(response, dict): + value = response.get("model") + else: + value = getattr(response, "model", None) + return value if isinstance(value, str) and value.strip() else None def _provider_request( diff --git a/hermes_cli/observability/shared_metrics_contract.py b/hermes_cli/observability/shared_metrics_contract.py index 7284ec3463..bb48952cb8 100644 --- a/hermes_cli/observability/shared_metrics_contract.py +++ b/hermes_cli/observability/shared_metrics_contract.py @@ -4,7 +4,12 @@ from __future__ import annotations from typing import Any -from agent.relay_runtime import RUNTIME_INSTANCE_KEY +from agent.relay_runtime import ( + LOGICAL_LLM_SCOPE, + RUNTIME_INSTANCE_KEY, + RUNTIME_SCHEMA_KEY, + RUNTIME_SCHEMA_VERSION, +) SCHEMA_KEY = "hermes.metrics.schema_version" SCHEMA_VERSION = "hermes.metrics.event.v2" @@ -186,7 +191,11 @@ def counter_dimensions_are_valid( def model_call_dimensions(event: Any) -> dict[str, str] | None: - """Return package dimensions for one valid primary model-call end event.""" + """Return package dimensions for one valid logical model-call end event.""" + auxiliary = _auxiliary_model_call_dimensions(event) + if auxiliary is not None: + return auxiliary + metadata = getattr(event, "metadata", None) if not isinstance(metadata, dict) or metadata.get(SCHEMA_KEY) != SCHEMA_VERSION: return None @@ -221,6 +230,51 @@ def model_call_dimensions(event: Any) -> dict[str, str] | None: return dimensions +def _auxiliary_model_call_dimensions(event: Any) -> dict[str, str] | None: + """Project a terminal auxiliary route from its Hermes logical scope.""" + metadata = getattr(event, "metadata", None) + if ( + not isinstance(metadata, dict) + or metadata.get(RUNTIME_SCHEMA_KEY) != RUNTIME_SCHEMA_VERSION + ): + return None + relay_metadata = set(metadata) - { + RUNTIME_INSTANCE_KEY, + RUNTIME_SCHEMA_KEY, + "hermes.call_role", + } + if relay_metadata - {"otel.status_code"} or metadata.get( + "otel.status_code", "OK" + ) not in {"OK", "ERROR"}: + return None + call_role = metadata.get("hermes.call_role") + if not isinstance(call_role, str) or not call_role.startswith("auxiliary:"): + return None + if ( + str(getattr(event, "kind", "") or "") != "scope" + or str(getattr(event, "category", "") or "") != "function" + or str(getattr(event, "name", "") or "") != LOGICAL_LLM_SCOPE + or str(getattr(event, "scope_category", "") or "") != "end" + or getattr(event, "category_profile", None) is not None + ): + return None + data = getattr(event, "data", None) + if ( + not isinstance(data, dict) + or set(data) + not in ( + {"model", "outcome", "provider"}, + {"model", "outcome", "provider", "response_model"}, + ) + or data.get("outcome") not in {"cancelled", "failed", "success"} + ): + return None + dimensions = model_call_fields(data) + if not counter_dimensions_are_valid(MODEL_ROUTE_METRIC, dimensions): + return None + return dimensions + + def task_counter(event: Any) -> tuple[str, dict[str, str]] | None: """Return one validated task counter from a task scope event.""" metadata = getattr(event, "metadata", None) diff --git a/tests/agent/test_auxiliary_relay.py b/tests/agent/test_auxiliary_relay.py index da4d6323c6..5af8ef0c60 100644 --- a/tests/agent/test_auxiliary_relay.py +++ b/tests/agent/test_auxiliary_relay.py @@ -5,6 +5,9 @@ import pytest pytest.importorskip("nemo_relay") from agent import auxiliary_client, relay_llm, relay_runtime +from hermes_cli.observability.shared_metrics import SharedMetricsStore +from hermes_cli.observability.shared_metrics_contract import MODEL_ROUTE_METRIC +from hermes_cli.observability.shared_metrics_subscriber import SharedMetricsSubscriber @pytest.fixture() @@ -54,8 +57,8 @@ def test_auxiliary_retries_share_logical_relay_identity(monkeypatch): monkeypatch.setattr( relay_llm, "complete_logical_call", - lambda request_id, *, outcome: logical_completions.append( - (request_id, outcome) + lambda request_id, *, outcome, model_name, provider_name, response_model_name: logical_completions.append( + (request_id, outcome, model_name, provider_name, response_model_name) ), ) @@ -92,7 +95,13 @@ def test_auxiliary_retries_share_logical_relay_identity(monkeypatch): assert attempts[0]["metadata"]["call_role"] == "auxiliary:compression" assert all(attempt["defer_logical_completion"] is True for attempt in attempts) assert logical_completions == [ - (attempts[0]["metadata"]["api_request_id"], "success") + ( + attempts[0]["metadata"]["api_request_id"], + "success", + "test-model", + "openrouter", + None, + ) ] @@ -103,11 +112,11 @@ def test_auxiliary_provider_fallback_closes_one_real_logical_call( relay, turn = relay_turn consumer = "test.auxiliary-provider-fallback" turn.lease.host.retain_managed_execution(consumer) - outcomes = [] + logical_outputs = [] original_pop = relay.scope.pop def record_pop(*args, **kwargs): - outcomes.append((kwargs.get("output") or {}).get("outcome")) + logical_outputs.append(kwargs.get("output") or {}) return original_pop(*args, **kwargs) monkeypatch.setattr(relay.scope, "pop", record_pop) @@ -162,7 +171,93 @@ def test_auxiliary_provider_fallback_closes_one_real_logical_call( assert result.choices[0].message.content == "recovered" assert turn.logical_llm_calls == {} - assert outcomes == ["success"] + assert logical_outputs == [ + { + "model": "openrouter/test-model", + "outcome": "success", + "provider": "openrouter", + } + ] + + +def test_auxiliary_provider_fallback_records_one_terminal_model_route( + relay_turn, + tmp_path, +): + relay, turn = relay_turn + store = SharedMetricsStore( + tmp_path / "metrics.sqlite3", + tmp_path / "outbox", + ) + subscriber = SharedMetricsSubscriber( + store, + "test-version", + runtime_id=turn.lease.host.runtime_id, + ) + subscriber_name = "test.auxiliary-model-route" + relay.subscribers.register(subscriber_name, subscriber) + turn.lease.host.retain_managed_execution(subscriber_name) + responses = iter([ + SimpleNamespace(model="failed/model", choices=[]), + SimpleNamespace( + model="Accepted/Model", + choices=[SimpleNamespace(message=SimpleNamespace(content="recovered"))], + ), + ]) + client = SimpleNamespace( + chat=SimpleNamespace( + completions=SimpleNamespace( + create=lambda **_kwargs: next(responses), + ) + ) + ) + + @auxiliary_client._relay_auxiliary_call + def run(task): + auxiliary_client._set_relay_auxiliary_route( + "nvidia", + "failed/configured-model", + "chat_completions", + ) + with pytest.raises(RuntimeError, match="invalid response"): + auxiliary_client._validate_llm_response( + auxiliary_client._relay_sync_completion( + client, + {"model": "failed/configured-model", "messages": []}, + ), + task, + ) + auxiliary_client._set_relay_auxiliary_route( + "OpenRouter", + "fallback/configured-model", + "chat_completions", + ) + return auxiliary_client._validate_llm_response( + auxiliary_client._relay_sync_completion( + client, + {"model": "fallback/configured-model", "messages": []}, + ), + task, + ) + + try: + result = run("compression") + relay.subscribers.flush() + finally: + turn.lease.host.release_managed_execution(subscriber_name) + relay.subscribers.deregister(subscriber_name) + + assert result.choices[0].message.content == "recovered" + snapshot = store.counter_snapshot() + assert len(snapshot) == 1 + assert snapshot[0]["metric_name"] == MODEL_ROUTE_METRIC + assert snapshot[0]["hermes_version"] == "test-version" + assert snapshot[0]["dimensions"] == { + "model": "accepted/model", + "provider": "openrouter", + } + assert snapshot[0]["value"] == 1 + assert snapshot[0]["packaged_value"] == 0 @pytest.mark.asyncio @@ -192,8 +287,8 @@ async def test_async_auxiliary_attempt_uses_inherited_relay_adapter(monkeypatch) monkeypatch.setattr( relay_llm, "complete_logical_call", - lambda request_id, *, outcome: logical_completions.append( - (request_id, outcome) + lambda request_id, *, outcome, model_name, provider_name, response_model_name: logical_completions.append( + (request_id, outcome, model_name, provider_name, response_model_name) ), ) @@ -219,7 +314,13 @@ async def test_async_auxiliary_attempt_uses_inherited_relay_adapter(monkeypatch) assert captured["metadata"]["call_role"] == "auxiliary:title_generation" assert captured["defer_logical_completion"] is True assert logical_completions == [ - (captured["metadata"]["api_request_id"], "success") + ( + captured["metadata"]["api_request_id"], + "success", + "claude-test", + "anthropic", + None, + ) ] @@ -235,11 +336,11 @@ def test_partial_auxiliary_stream_failure_closes_before_recovery( _relay, turn = relay_turn consumer = "test.partial-auxiliary-stream-failure" turn.lease.host.retain_managed_execution(consumer) - outcomes = [] + logical_outputs = [] original_pop = turn.lease.host.relay.scope.pop def record_pop(*args, **kwargs): - outcomes.append((kwargs.get("output") or {}).get("outcome")) + logical_outputs.append(kwargs.get("output") or {}) return original_pop(*args, **kwargs) monkeypatch.setattr(turn.lease.host.relay.scope, "pop", record_pop) @@ -317,15 +418,30 @@ def test_partial_auxiliary_stream_failure_closes_before_recovery( next(stream) assert caught.value is provider_error - assert outcomes == ["failed"] + assert logical_outputs == [ + { + "model": "test-model", + "outcome": "failed", + "provider": "openrouter", + } + ] assert turn.logical_llm_calls == {} result = recover("moa") assert result.choices[0].message.content == "recovered" - assert outcomes == ["failed", "success"] + assert logical_outputs == [ + { + "model": "test-model", + "outcome": "failed", + "provider": "openrouter", + }, + { + "model": "test-model", + "outcome": "success", + "provider": "openrouter", + }, + ] assert turn.logical_llm_calls == {} finally: turn.lease.host.release_managed_execution(consumer) - - diff --git a/tests/hermes_cli/test_relay_shared_metrics.py b/tests/hermes_cli/test_relay_shared_metrics.py index d4d83c1d4b..71abad092d 100644 --- a/tests/hermes_cli/test_relay_shared_metrics.py +++ b/tests/hermes_cli/test_relay_shared_metrics.py @@ -18,6 +18,7 @@ from types import SimpleNamespace from typing import Any import pytest +from agent import relay_runtime from hermes_cli.observability import shared_metrics as shared_metrics_module from hermes_cli.observability.shared_metrics import SharedMetricsStore from hermes_cli.observability.shared_metrics_contract import ( @@ -347,6 +348,43 @@ def test_model_call_fields_report_terminal_model_and_provider_without_a_catalog( } +def test_auxiliary_logical_scope_projects_one_normalized_terminal_route(): + event = SimpleNamespace( + kind="scope", + category="function", + name=relay_runtime.LOGICAL_LLM_SCOPE, + scope_category="end", + category_profile=None, + data={ + "model": "Accepted/Model", + "outcome": "success", + "provider": "OpenRouter", + }, + metadata={ + relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION, + relay_runtime.RUNTIME_INSTANCE_KEY: "runtime-1", + "hermes.call_role": "auxiliary:compression", + }, + ) + + assert model_call_dimensions(event) == { + "model": "accepted/model", + "provider": "openrouter", + } + + event.data.update({ + "model": "configured/model", + "response_model": "malformed response model", + }) + assert model_call_dimensions(event) == { + "model": "configured/model", + "provider": "openrouter", + } + + event.metadata["hermes.call_role"] = "primary" + assert model_call_dimensions(event) is None + + @pytest.mark.parametrize( "response_model", [