fix(observability): include auxiliary model routes

Signed-off-by: Alex Fournier <afournier@nvidia.com>
This commit is contained in:
Alex Fournier
2026-07-31 06:58:41 -07:00
parent c0369f0891
commit 43d29a37c8
5 changed files with 324 additions and 26 deletions
+21
View File
@@ -2522,6 +2522,7 @@ def _relay_auxiliary_call(callback):
"attempt_count": 0,
"provider": "",
"model": "",
"response_model": None,
"api_mode": "chat_completions",
})
try:
@@ -2547,6 +2548,7 @@ def _relay_auxiliary_call_async(callback):
"attempt_count": 0,
"provider": "",
"model": "",
"response_model": None,
"api_mode": "chat_completions",
})
try:
@@ -2570,6 +2572,7 @@ def _set_relay_auxiliary_route(
return
context["provider"] = str(provider or "auxiliary")
context["model"] = str(model or "unknown")
context["response_model"] = None
context["api_mode"] = str(api_mode or "chat_completions")
@@ -7474,6 +7477,7 @@ def _validate_llm_response(
except (AttributeError, TypeError, IndexError) as exc:
recovered = _recover_aux_response_message(response)
if recovered is not None:
_record_relay_auxiliary_response_model(response)
_complete_relay_auxiliary_call()
return recovered
response_type = type(response).__name__
@@ -7484,6 +7488,7 @@ def _validate_llm_response(
f"Expected object with .choices[0].message — check provider "
f"adapter or custom endpoint compatibility."
) from exc
_record_relay_auxiliary_response_model(response)
_complete_relay_auxiliary_call()
return response
@@ -7498,9 +7503,25 @@ def _complete_relay_auxiliary_call(*, outcome: str = "success") -> None:
relay_llm.complete_logical_call(
str(context.get("request_id") or ""),
outcome=outcome,
model_name=str(context.get("model") or "unknown"),
provider_name=str(context.get("provider") or "auxiliary"),
response_model_name=context.get("response_model"),
)
def _record_relay_auxiliary_response_model(response: Any) -> None:
"""Retain the provider-reported model for terminal route attribution."""
context = _RELAY_AUX_CALL_CONTEXT.get()
if context is None:
return
if isinstance(response, dict):
model = response.get("model")
else:
model = getattr(response, "model", None)
if isinstance(model, str) and model.strip():
context["response_model"] = model
def _fail_relay_auxiliary_call() -> None:
"""Close a terminally failed call without replacing its original error."""
try:
+78 -9
View File
@@ -336,6 +336,14 @@ class ManagedLlmStream(Iterator[Any]):
self._callback_error: BaseException | None = None
self._logical: tuple[relay_runtime.RelayTurnContext, Any, str] | None = None
self._defer_logical_completion = defer_logical_completion
if str((metadata or {}).get("call_role") or "").startswith("auxiliary:"):
self._logical_model_name: str | None = model_name
self._logical_provider_name: str | None = name
self._logical_response_model_name: str | None = None
else:
self._logical_model_name = None
self._logical_provider_name = None
self._logical_response_model_name = None
self._on_chunk = on_chunk
self._chunk_adapter = chunk_adapter or _namespace
self._accept_chunk = accept_chunk
@@ -445,8 +453,12 @@ class ManagedLlmStream(Iterator[Any]):
return None
try:
if self.final_response is not None:
return _jsonable(self.final_response)
return _jsonable(run_callback(finalizer))
response = self.final_response
else:
response = run_callback(finalizer)
if self._logical_model_name is not None:
self._logical_response_model_name = _response_model_name(response)
return _jsonable(response)
except BaseException as exc:
self._callback_error = exc
raise
@@ -488,6 +500,9 @@ class ManagedLlmStream(Iterator[Any]):
_complete_logical(
self._logical,
outcome="cancelled" if _is_cancellation(exc) else "failed",
model_name=self._logical_model_name,
provider_name=self._logical_provider_name,
response_model_name=self._logical_response_model_name,
)
self._logical = None
loop.close()
@@ -520,7 +535,13 @@ class ManagedLlmStream(Iterator[Any]):
if self._raw_chunks:
self.output_modified = True
if not self._defer_logical_completion:
_complete_logical(self._logical, outcome="success")
_complete_logical(
self._logical,
outcome="success",
model_name=self._logical_model_name,
provider_name=self._logical_provider_name,
response_model_name=self._logical_response_model_name,
)
self._logical = None
self._close(logical_outcome="cancelled")
raise StopIteration from None
@@ -593,7 +614,13 @@ class ManagedLlmStream(Iterator[Any]):
)
loop.close()
if not self._defer_logical_completion:
_complete_logical(self._logical, outcome="success")
_complete_logical(
self._logical,
outcome="success",
model_name=self._logical_model_name,
provider_name=self._logical_provider_name,
response_model_name=self._logical_response_model_name,
)
self._logical = None
def _close(self, *, logical_outcome: str) -> None:
@@ -623,7 +650,13 @@ class ManagedLlmStream(Iterator[Any]):
exc_info=True,
)
if not self._defer_logical_completion:
_complete_logical(self._logical, outcome=logical_outcome)
_complete_logical(
self._logical,
outcome=logical_outcome,
model_name=self._logical_model_name,
provider_name=self._logical_provider_name,
response_model_name=self._logical_response_model_name,
)
self._logical = None
return
close = getattr(self._stream, "aclose", None)
@@ -638,7 +671,13 @@ class ManagedLlmStream(Iterator[Any]):
if self._close_error is None:
self._close_error = exc
if not self._defer_logical_completion:
_complete_logical(self._logical, outcome=logical_outcome)
_complete_logical(
self._logical,
outcome=logical_outcome,
model_name=self._logical_model_name,
provider_name=self._logical_provider_name,
response_model_name=self._logical_response_model_name,
)
self._logical = None
loop.close()
@@ -777,6 +816,9 @@ def _complete_logical(
logical: tuple[relay_runtime.RelayTurnContext, Any, str] | None,
*,
outcome: str,
model_name: str | None = None,
provider_name: str | None = None,
response_model_name: str | None = None,
) -> None:
if logical is None:
return
@@ -791,11 +833,16 @@ def _complete_logical(
if lease.session is None:
return
try:
output = {"outcome": outcome}
if model_name is not None and provider_name is not None:
output.update({"model": model_name, "provider": provider_name})
if response_model_name is not None:
output["response_model"] = response_model_name
lease.host.run_in_session(
lease.session,
lease.host.relay.scope.pop,
handle,
output={"outcome": outcome},
output=output,
metadata={
relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION,
relay_runtime.RUNTIME_INSTANCE_KEY: lease.host.runtime_id,
@@ -845,7 +892,14 @@ def _is_cancellation(error: BaseException) -> bool:
)
def complete_logical_call(api_request_id: str, *, outcome: str) -> None:
def complete_logical_call(
api_request_id: str,
*,
outcome: str,
model_name: str | None = None,
provider_name: str | None = None,
response_model_name: str | None = None,
) -> None:
"""Complete the active turn's logical LLM call after caller validation."""
turn = relay_runtime.active_turn()
if turn is None or not api_request_id:
@@ -853,7 +907,22 @@ def complete_logical_call(api_request_id: str, *, outcome: str) -> None:
with turn.logical_llm_lock:
handle = turn.logical_llm_calls.get(api_request_id)
if handle is not None:
_complete_logical((turn, handle, api_request_id), outcome=outcome)
_complete_logical(
(turn, handle, api_request_id),
outcome=outcome,
model_name=model_name,
provider_name=provider_name,
response_model_name=response_model_name,
)
def _response_model_name(response: Any) -> str | None:
"""Return a provider-reported model name when one is available."""
if isinstance(response, dict):
value = response.get("model")
else:
value = getattr(response, "model", None)
return value if isinstance(value, str) and value.strip() else None
def _provider_request(
@@ -4,7 +4,12 @@ from __future__ import annotations
from typing import Any
from agent.relay_runtime import RUNTIME_INSTANCE_KEY
from agent.relay_runtime import (
LOGICAL_LLM_SCOPE,
RUNTIME_INSTANCE_KEY,
RUNTIME_SCHEMA_KEY,
RUNTIME_SCHEMA_VERSION,
)
SCHEMA_KEY = "hermes.metrics.schema_version"
SCHEMA_VERSION = "hermes.metrics.event.v2"
@@ -186,7 +191,11 @@ def counter_dimensions_are_valid(
def model_call_dimensions(event: Any) -> dict[str, str] | None:
"""Return package dimensions for one valid primary model-call end event."""
"""Return package dimensions for one valid logical model-call end event."""
auxiliary = _auxiliary_model_call_dimensions(event)
if auxiliary is not None:
return auxiliary
metadata = getattr(event, "metadata", None)
if not isinstance(metadata, dict) or metadata.get(SCHEMA_KEY) != SCHEMA_VERSION:
return None
@@ -221,6 +230,51 @@ def model_call_dimensions(event: Any) -> dict[str, str] | None:
return dimensions
def _auxiliary_model_call_dimensions(event: Any) -> dict[str, str] | None:
"""Project a terminal auxiliary route from its Hermes logical scope."""
metadata = getattr(event, "metadata", None)
if (
not isinstance(metadata, dict)
or metadata.get(RUNTIME_SCHEMA_KEY) != RUNTIME_SCHEMA_VERSION
):
return None
relay_metadata = set(metadata) - {
RUNTIME_INSTANCE_KEY,
RUNTIME_SCHEMA_KEY,
"hermes.call_role",
}
if relay_metadata - {"otel.status_code"} or metadata.get(
"otel.status_code", "OK"
) not in {"OK", "ERROR"}:
return None
call_role = metadata.get("hermes.call_role")
if not isinstance(call_role, str) or not call_role.startswith("auxiliary:"):
return None
if (
str(getattr(event, "kind", "") or "") != "scope"
or str(getattr(event, "category", "") or "") != "function"
or str(getattr(event, "name", "") or "") != LOGICAL_LLM_SCOPE
or str(getattr(event, "scope_category", "") or "") != "end"
or getattr(event, "category_profile", None) is not None
):
return None
data = getattr(event, "data", None)
if (
not isinstance(data, dict)
or set(data)
not in (
{"model", "outcome", "provider"},
{"model", "outcome", "provider", "response_model"},
)
or data.get("outcome") not in {"cancelled", "failed", "success"}
):
return None
dimensions = model_call_fields(data)
if not counter_dimensions_are_valid(MODEL_ROUTE_METRIC, dimensions):
return None
return dimensions
def task_counter(event: Any) -> tuple[str, dict[str, str]] | None:
"""Return one validated task counter from a task scope event."""
metadata = getattr(event, "metadata", None)
+131 -15
View File
@@ -5,6 +5,9 @@ import pytest
pytest.importorskip("nemo_relay")
from agent import auxiliary_client, relay_llm, relay_runtime
from hermes_cli.observability.shared_metrics import SharedMetricsStore
from hermes_cli.observability.shared_metrics_contract import MODEL_ROUTE_METRIC
from hermes_cli.observability.shared_metrics_subscriber import SharedMetricsSubscriber
@pytest.fixture()
@@ -54,8 +57,8 @@ def test_auxiliary_retries_share_logical_relay_identity(monkeypatch):
monkeypatch.setattr(
relay_llm,
"complete_logical_call",
lambda request_id, *, outcome: logical_completions.append(
(request_id, outcome)
lambda request_id, *, outcome, model_name, provider_name, response_model_name: logical_completions.append(
(request_id, outcome, model_name, provider_name, response_model_name)
),
)
@@ -92,7 +95,13 @@ def test_auxiliary_retries_share_logical_relay_identity(monkeypatch):
assert attempts[0]["metadata"]["call_role"] == "auxiliary:compression"
assert all(attempt["defer_logical_completion"] is True for attempt in attempts)
assert logical_completions == [
(attempts[0]["metadata"]["api_request_id"], "success")
(
attempts[0]["metadata"]["api_request_id"],
"success",
"test-model",
"openrouter",
None,
)
]
@@ -103,11 +112,11 @@ def test_auxiliary_provider_fallback_closes_one_real_logical_call(
relay, turn = relay_turn
consumer = "test.auxiliary-provider-fallback"
turn.lease.host.retain_managed_execution(consumer)
outcomes = []
logical_outputs = []
original_pop = relay.scope.pop
def record_pop(*args, **kwargs):
outcomes.append((kwargs.get("output") or {}).get("outcome"))
logical_outputs.append(kwargs.get("output") or {})
return original_pop(*args, **kwargs)
monkeypatch.setattr(relay.scope, "pop", record_pop)
@@ -162,7 +171,93 @@ def test_auxiliary_provider_fallback_closes_one_real_logical_call(
assert result.choices[0].message.content == "recovered"
assert turn.logical_llm_calls == {}
assert outcomes == ["success"]
assert logical_outputs == [
{
"model": "openrouter/test-model",
"outcome": "success",
"provider": "openrouter",
}
]
def test_auxiliary_provider_fallback_records_one_terminal_model_route(
relay_turn,
tmp_path,
):
relay, turn = relay_turn
store = SharedMetricsStore(
tmp_path / "metrics.sqlite3",
tmp_path / "outbox",
)
subscriber = SharedMetricsSubscriber(
store,
"test-version",
runtime_id=turn.lease.host.runtime_id,
)
subscriber_name = "test.auxiliary-model-route"
relay.subscribers.register(subscriber_name, subscriber)
turn.lease.host.retain_managed_execution(subscriber_name)
responses = iter([
SimpleNamespace(model="failed/model", choices=[]),
SimpleNamespace(
model="Accepted/Model",
choices=[SimpleNamespace(message=SimpleNamespace(content="recovered"))],
),
])
client = SimpleNamespace(
chat=SimpleNamespace(
completions=SimpleNamespace(
create=lambda **_kwargs: next(responses),
)
)
)
@auxiliary_client._relay_auxiliary_call
def run(task):
auxiliary_client._set_relay_auxiliary_route(
"nvidia",
"failed/configured-model",
"chat_completions",
)
with pytest.raises(RuntimeError, match="invalid response"):
auxiliary_client._validate_llm_response(
auxiliary_client._relay_sync_completion(
client,
{"model": "failed/configured-model", "messages": []},
),
task,
)
auxiliary_client._set_relay_auxiliary_route(
"OpenRouter",
"fallback/configured-model",
"chat_completions",
)
return auxiliary_client._validate_llm_response(
auxiliary_client._relay_sync_completion(
client,
{"model": "fallback/configured-model", "messages": []},
),
task,
)
try:
result = run("compression")
relay.subscribers.flush()
finally:
turn.lease.host.release_managed_execution(subscriber_name)
relay.subscribers.deregister(subscriber_name)
assert result.choices[0].message.content == "recovered"
snapshot = store.counter_snapshot()
assert len(snapshot) == 1
assert snapshot[0]["metric_name"] == MODEL_ROUTE_METRIC
assert snapshot[0]["hermes_version"] == "test-version"
assert snapshot[0]["dimensions"] == {
"model": "accepted/model",
"provider": "openrouter",
}
assert snapshot[0]["value"] == 1
assert snapshot[0]["packaged_value"] == 0
@pytest.mark.asyncio
@@ -192,8 +287,8 @@ async def test_async_auxiliary_attempt_uses_inherited_relay_adapter(monkeypatch)
monkeypatch.setattr(
relay_llm,
"complete_logical_call",
lambda request_id, *, outcome: logical_completions.append(
(request_id, outcome)
lambda request_id, *, outcome, model_name, provider_name, response_model_name: logical_completions.append(
(request_id, outcome, model_name, provider_name, response_model_name)
),
)
@@ -219,7 +314,13 @@ async def test_async_auxiliary_attempt_uses_inherited_relay_adapter(monkeypatch)
assert captured["metadata"]["call_role"] == "auxiliary:title_generation"
assert captured["defer_logical_completion"] is True
assert logical_completions == [
(captured["metadata"]["api_request_id"], "success")
(
captured["metadata"]["api_request_id"],
"success",
"claude-test",
"anthropic",
None,
)
]
@@ -235,11 +336,11 @@ def test_partial_auxiliary_stream_failure_closes_before_recovery(
_relay, turn = relay_turn
consumer = "test.partial-auxiliary-stream-failure"
turn.lease.host.retain_managed_execution(consumer)
outcomes = []
logical_outputs = []
original_pop = turn.lease.host.relay.scope.pop
def record_pop(*args, **kwargs):
outcomes.append((kwargs.get("output") or {}).get("outcome"))
logical_outputs.append(kwargs.get("output") or {})
return original_pop(*args, **kwargs)
monkeypatch.setattr(turn.lease.host.relay.scope, "pop", record_pop)
@@ -317,15 +418,30 @@ def test_partial_auxiliary_stream_failure_closes_before_recovery(
next(stream)
assert caught.value is provider_error
assert outcomes == ["failed"]
assert logical_outputs == [
{
"model": "test-model",
"outcome": "failed",
"provider": "openrouter",
}
]
assert turn.logical_llm_calls == {}
result = recover("moa")
assert result.choices[0].message.content == "recovered"
assert outcomes == ["failed", "success"]
assert logical_outputs == [
{
"model": "test-model",
"outcome": "failed",
"provider": "openrouter",
},
{
"model": "test-model",
"outcome": "success",
"provider": "openrouter",
},
]
assert turn.logical_llm_calls == {}
finally:
turn.lease.host.release_managed_execution(consumer)
@@ -18,6 +18,7 @@ from types import SimpleNamespace
from typing import Any
import pytest
from agent import relay_runtime
from hermes_cli.observability import shared_metrics as shared_metrics_module
from hermes_cli.observability.shared_metrics import SharedMetricsStore
from hermes_cli.observability.shared_metrics_contract import (
@@ -347,6 +348,43 @@ def test_model_call_fields_report_terminal_model_and_provider_without_a_catalog(
}
def test_auxiliary_logical_scope_projects_one_normalized_terminal_route():
event = SimpleNamespace(
kind="scope",
category="function",
name=relay_runtime.LOGICAL_LLM_SCOPE,
scope_category="end",
category_profile=None,
data={
"model": "Accepted/Model",
"outcome": "success",
"provider": "OpenRouter",
},
metadata={
relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION,
relay_runtime.RUNTIME_INSTANCE_KEY: "runtime-1",
"hermes.call_role": "auxiliary:compression",
},
)
assert model_call_dimensions(event) == {
"model": "accepted/model",
"provider": "openrouter",
}
event.data.update({
"model": "configured/model",
"response_model": "malformed response model",
})
assert model_call_dimensions(event) == {
"model": "configured/model",
"provider": "openrouter",
}
event.metadata["hermes.call_role"] = "primary"
assert model_call_dimensions(event) is None
@pytest.mark.parametrize(
"response_model",
[