fix(observability): include auxiliary model routes
Signed-off-by: Alex Fournier <afournier@nvidia.com>
This commit is contained in:
@@ -2522,6 +2522,7 @@ def _relay_auxiliary_call(callback):
|
||||
"attempt_count": 0,
|
||||
"provider": "",
|
||||
"model": "",
|
||||
"response_model": None,
|
||||
"api_mode": "chat_completions",
|
||||
})
|
||||
try:
|
||||
@@ -2547,6 +2548,7 @@ def _relay_auxiliary_call_async(callback):
|
||||
"attempt_count": 0,
|
||||
"provider": "",
|
||||
"model": "",
|
||||
"response_model": None,
|
||||
"api_mode": "chat_completions",
|
||||
})
|
||||
try:
|
||||
@@ -2570,6 +2572,7 @@ def _set_relay_auxiliary_route(
|
||||
return
|
||||
context["provider"] = str(provider or "auxiliary")
|
||||
context["model"] = str(model or "unknown")
|
||||
context["response_model"] = None
|
||||
context["api_mode"] = str(api_mode or "chat_completions")
|
||||
|
||||
|
||||
@@ -7474,6 +7477,7 @@ def _validate_llm_response(
|
||||
except (AttributeError, TypeError, IndexError) as exc:
|
||||
recovered = _recover_aux_response_message(response)
|
||||
if recovered is not None:
|
||||
_record_relay_auxiliary_response_model(response)
|
||||
_complete_relay_auxiliary_call()
|
||||
return recovered
|
||||
response_type = type(response).__name__
|
||||
@@ -7484,6 +7488,7 @@ def _validate_llm_response(
|
||||
f"Expected object with .choices[0].message — check provider "
|
||||
f"adapter or custom endpoint compatibility."
|
||||
) from exc
|
||||
_record_relay_auxiliary_response_model(response)
|
||||
_complete_relay_auxiliary_call()
|
||||
return response
|
||||
|
||||
@@ -7498,9 +7503,25 @@ def _complete_relay_auxiliary_call(*, outcome: str = "success") -> None:
|
||||
relay_llm.complete_logical_call(
|
||||
str(context.get("request_id") or ""),
|
||||
outcome=outcome,
|
||||
model_name=str(context.get("model") or "unknown"),
|
||||
provider_name=str(context.get("provider") or "auxiliary"),
|
||||
response_model_name=context.get("response_model"),
|
||||
)
|
||||
|
||||
|
||||
def _record_relay_auxiliary_response_model(response: Any) -> None:
|
||||
"""Retain the provider-reported model for terminal route attribution."""
|
||||
context = _RELAY_AUX_CALL_CONTEXT.get()
|
||||
if context is None:
|
||||
return
|
||||
if isinstance(response, dict):
|
||||
model = response.get("model")
|
||||
else:
|
||||
model = getattr(response, "model", None)
|
||||
if isinstance(model, str) and model.strip():
|
||||
context["response_model"] = model
|
||||
|
||||
|
||||
def _fail_relay_auxiliary_call() -> None:
|
||||
"""Close a terminally failed call without replacing its original error."""
|
||||
try:
|
||||
|
||||
+78
-9
@@ -336,6 +336,14 @@ class ManagedLlmStream(Iterator[Any]):
|
||||
self._callback_error: BaseException | None = None
|
||||
self._logical: tuple[relay_runtime.RelayTurnContext, Any, str] | None = None
|
||||
self._defer_logical_completion = defer_logical_completion
|
||||
if str((metadata or {}).get("call_role") or "").startswith("auxiliary:"):
|
||||
self._logical_model_name: str | None = model_name
|
||||
self._logical_provider_name: str | None = name
|
||||
self._logical_response_model_name: str | None = None
|
||||
else:
|
||||
self._logical_model_name = None
|
||||
self._logical_provider_name = None
|
||||
self._logical_response_model_name = None
|
||||
self._on_chunk = on_chunk
|
||||
self._chunk_adapter = chunk_adapter or _namespace
|
||||
self._accept_chunk = accept_chunk
|
||||
@@ -445,8 +453,12 @@ class ManagedLlmStream(Iterator[Any]):
|
||||
return None
|
||||
try:
|
||||
if self.final_response is not None:
|
||||
return _jsonable(self.final_response)
|
||||
return _jsonable(run_callback(finalizer))
|
||||
response = self.final_response
|
||||
else:
|
||||
response = run_callback(finalizer)
|
||||
if self._logical_model_name is not None:
|
||||
self._logical_response_model_name = _response_model_name(response)
|
||||
return _jsonable(response)
|
||||
except BaseException as exc:
|
||||
self._callback_error = exc
|
||||
raise
|
||||
@@ -488,6 +500,9 @@ class ManagedLlmStream(Iterator[Any]):
|
||||
_complete_logical(
|
||||
self._logical,
|
||||
outcome="cancelled" if _is_cancellation(exc) else "failed",
|
||||
model_name=self._logical_model_name,
|
||||
provider_name=self._logical_provider_name,
|
||||
response_model_name=self._logical_response_model_name,
|
||||
)
|
||||
self._logical = None
|
||||
loop.close()
|
||||
@@ -520,7 +535,13 @@ class ManagedLlmStream(Iterator[Any]):
|
||||
if self._raw_chunks:
|
||||
self.output_modified = True
|
||||
if not self._defer_logical_completion:
|
||||
_complete_logical(self._logical, outcome="success")
|
||||
_complete_logical(
|
||||
self._logical,
|
||||
outcome="success",
|
||||
model_name=self._logical_model_name,
|
||||
provider_name=self._logical_provider_name,
|
||||
response_model_name=self._logical_response_model_name,
|
||||
)
|
||||
self._logical = None
|
||||
self._close(logical_outcome="cancelled")
|
||||
raise StopIteration from None
|
||||
@@ -593,7 +614,13 @@ class ManagedLlmStream(Iterator[Any]):
|
||||
)
|
||||
loop.close()
|
||||
if not self._defer_logical_completion:
|
||||
_complete_logical(self._logical, outcome="success")
|
||||
_complete_logical(
|
||||
self._logical,
|
||||
outcome="success",
|
||||
model_name=self._logical_model_name,
|
||||
provider_name=self._logical_provider_name,
|
||||
response_model_name=self._logical_response_model_name,
|
||||
)
|
||||
self._logical = None
|
||||
|
||||
def _close(self, *, logical_outcome: str) -> None:
|
||||
@@ -623,7 +650,13 @@ class ManagedLlmStream(Iterator[Any]):
|
||||
exc_info=True,
|
||||
)
|
||||
if not self._defer_logical_completion:
|
||||
_complete_logical(self._logical, outcome=logical_outcome)
|
||||
_complete_logical(
|
||||
self._logical,
|
||||
outcome=logical_outcome,
|
||||
model_name=self._logical_model_name,
|
||||
provider_name=self._logical_provider_name,
|
||||
response_model_name=self._logical_response_model_name,
|
||||
)
|
||||
self._logical = None
|
||||
return
|
||||
close = getattr(self._stream, "aclose", None)
|
||||
@@ -638,7 +671,13 @@ class ManagedLlmStream(Iterator[Any]):
|
||||
if self._close_error is None:
|
||||
self._close_error = exc
|
||||
if not self._defer_logical_completion:
|
||||
_complete_logical(self._logical, outcome=logical_outcome)
|
||||
_complete_logical(
|
||||
self._logical,
|
||||
outcome=logical_outcome,
|
||||
model_name=self._logical_model_name,
|
||||
provider_name=self._logical_provider_name,
|
||||
response_model_name=self._logical_response_model_name,
|
||||
)
|
||||
self._logical = None
|
||||
loop.close()
|
||||
|
||||
@@ -777,6 +816,9 @@ def _complete_logical(
|
||||
logical: tuple[relay_runtime.RelayTurnContext, Any, str] | None,
|
||||
*,
|
||||
outcome: str,
|
||||
model_name: str | None = None,
|
||||
provider_name: str | None = None,
|
||||
response_model_name: str | None = None,
|
||||
) -> None:
|
||||
if logical is None:
|
||||
return
|
||||
@@ -791,11 +833,16 @@ def _complete_logical(
|
||||
if lease.session is None:
|
||||
return
|
||||
try:
|
||||
output = {"outcome": outcome}
|
||||
if model_name is not None and provider_name is not None:
|
||||
output.update({"model": model_name, "provider": provider_name})
|
||||
if response_model_name is not None:
|
||||
output["response_model"] = response_model_name
|
||||
lease.host.run_in_session(
|
||||
lease.session,
|
||||
lease.host.relay.scope.pop,
|
||||
handle,
|
||||
output={"outcome": outcome},
|
||||
output=output,
|
||||
metadata={
|
||||
relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION,
|
||||
relay_runtime.RUNTIME_INSTANCE_KEY: lease.host.runtime_id,
|
||||
@@ -845,7 +892,14 @@ def _is_cancellation(error: BaseException) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def complete_logical_call(api_request_id: str, *, outcome: str) -> None:
|
||||
def complete_logical_call(
|
||||
api_request_id: str,
|
||||
*,
|
||||
outcome: str,
|
||||
model_name: str | None = None,
|
||||
provider_name: str | None = None,
|
||||
response_model_name: str | None = None,
|
||||
) -> None:
|
||||
"""Complete the active turn's logical LLM call after caller validation."""
|
||||
turn = relay_runtime.active_turn()
|
||||
if turn is None or not api_request_id:
|
||||
@@ -853,7 +907,22 @@ def complete_logical_call(api_request_id: str, *, outcome: str) -> None:
|
||||
with turn.logical_llm_lock:
|
||||
handle = turn.logical_llm_calls.get(api_request_id)
|
||||
if handle is not None:
|
||||
_complete_logical((turn, handle, api_request_id), outcome=outcome)
|
||||
_complete_logical(
|
||||
(turn, handle, api_request_id),
|
||||
outcome=outcome,
|
||||
model_name=model_name,
|
||||
provider_name=provider_name,
|
||||
response_model_name=response_model_name,
|
||||
)
|
||||
|
||||
|
||||
def _response_model_name(response: Any) -> str | None:
|
||||
"""Return a provider-reported model name when one is available."""
|
||||
if isinstance(response, dict):
|
||||
value = response.get("model")
|
||||
else:
|
||||
value = getattr(response, "model", None)
|
||||
return value if isinstance(value, str) and value.strip() else None
|
||||
|
||||
|
||||
def _provider_request(
|
||||
|
||||
@@ -4,7 +4,12 @@ from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from agent.relay_runtime import RUNTIME_INSTANCE_KEY
|
||||
from agent.relay_runtime import (
|
||||
LOGICAL_LLM_SCOPE,
|
||||
RUNTIME_INSTANCE_KEY,
|
||||
RUNTIME_SCHEMA_KEY,
|
||||
RUNTIME_SCHEMA_VERSION,
|
||||
)
|
||||
|
||||
SCHEMA_KEY = "hermes.metrics.schema_version"
|
||||
SCHEMA_VERSION = "hermes.metrics.event.v2"
|
||||
@@ -186,7 +191,11 @@ def counter_dimensions_are_valid(
|
||||
|
||||
|
||||
def model_call_dimensions(event: Any) -> dict[str, str] | None:
|
||||
"""Return package dimensions for one valid primary model-call end event."""
|
||||
"""Return package dimensions for one valid logical model-call end event."""
|
||||
auxiliary = _auxiliary_model_call_dimensions(event)
|
||||
if auxiliary is not None:
|
||||
return auxiliary
|
||||
|
||||
metadata = getattr(event, "metadata", None)
|
||||
if not isinstance(metadata, dict) or metadata.get(SCHEMA_KEY) != SCHEMA_VERSION:
|
||||
return None
|
||||
@@ -221,6 +230,51 @@ def model_call_dimensions(event: Any) -> dict[str, str] | None:
|
||||
return dimensions
|
||||
|
||||
|
||||
def _auxiliary_model_call_dimensions(event: Any) -> dict[str, str] | None:
|
||||
"""Project a terminal auxiliary route from its Hermes logical scope."""
|
||||
metadata = getattr(event, "metadata", None)
|
||||
if (
|
||||
not isinstance(metadata, dict)
|
||||
or metadata.get(RUNTIME_SCHEMA_KEY) != RUNTIME_SCHEMA_VERSION
|
||||
):
|
||||
return None
|
||||
relay_metadata = set(metadata) - {
|
||||
RUNTIME_INSTANCE_KEY,
|
||||
RUNTIME_SCHEMA_KEY,
|
||||
"hermes.call_role",
|
||||
}
|
||||
if relay_metadata - {"otel.status_code"} or metadata.get(
|
||||
"otel.status_code", "OK"
|
||||
) not in {"OK", "ERROR"}:
|
||||
return None
|
||||
call_role = metadata.get("hermes.call_role")
|
||||
if not isinstance(call_role, str) or not call_role.startswith("auxiliary:"):
|
||||
return None
|
||||
if (
|
||||
str(getattr(event, "kind", "") or "") != "scope"
|
||||
or str(getattr(event, "category", "") or "") != "function"
|
||||
or str(getattr(event, "name", "") or "") != LOGICAL_LLM_SCOPE
|
||||
or str(getattr(event, "scope_category", "") or "") != "end"
|
||||
or getattr(event, "category_profile", None) is not None
|
||||
):
|
||||
return None
|
||||
data = getattr(event, "data", None)
|
||||
if (
|
||||
not isinstance(data, dict)
|
||||
or set(data)
|
||||
not in (
|
||||
{"model", "outcome", "provider"},
|
||||
{"model", "outcome", "provider", "response_model"},
|
||||
)
|
||||
or data.get("outcome") not in {"cancelled", "failed", "success"}
|
||||
):
|
||||
return None
|
||||
dimensions = model_call_fields(data)
|
||||
if not counter_dimensions_are_valid(MODEL_ROUTE_METRIC, dimensions):
|
||||
return None
|
||||
return dimensions
|
||||
|
||||
|
||||
def task_counter(event: Any) -> tuple[str, dict[str, str]] | None:
|
||||
"""Return one validated task counter from a task scope event."""
|
||||
metadata = getattr(event, "metadata", None)
|
||||
|
||||
@@ -5,6 +5,9 @@ import pytest
|
||||
pytest.importorskip("nemo_relay")
|
||||
|
||||
from agent import auxiliary_client, relay_llm, relay_runtime
|
||||
from hermes_cli.observability.shared_metrics import SharedMetricsStore
|
||||
from hermes_cli.observability.shared_metrics_contract import MODEL_ROUTE_METRIC
|
||||
from hermes_cli.observability.shared_metrics_subscriber import SharedMetricsSubscriber
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
@@ -54,8 +57,8 @@ def test_auxiliary_retries_share_logical_relay_identity(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
relay_llm,
|
||||
"complete_logical_call",
|
||||
lambda request_id, *, outcome: logical_completions.append(
|
||||
(request_id, outcome)
|
||||
lambda request_id, *, outcome, model_name, provider_name, response_model_name: logical_completions.append(
|
||||
(request_id, outcome, model_name, provider_name, response_model_name)
|
||||
),
|
||||
)
|
||||
|
||||
@@ -92,7 +95,13 @@ def test_auxiliary_retries_share_logical_relay_identity(monkeypatch):
|
||||
assert attempts[0]["metadata"]["call_role"] == "auxiliary:compression"
|
||||
assert all(attempt["defer_logical_completion"] is True for attempt in attempts)
|
||||
assert logical_completions == [
|
||||
(attempts[0]["metadata"]["api_request_id"], "success")
|
||||
(
|
||||
attempts[0]["metadata"]["api_request_id"],
|
||||
"success",
|
||||
"test-model",
|
||||
"openrouter",
|
||||
None,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@@ -103,11 +112,11 @@ def test_auxiliary_provider_fallback_closes_one_real_logical_call(
|
||||
relay, turn = relay_turn
|
||||
consumer = "test.auxiliary-provider-fallback"
|
||||
turn.lease.host.retain_managed_execution(consumer)
|
||||
outcomes = []
|
||||
logical_outputs = []
|
||||
original_pop = relay.scope.pop
|
||||
|
||||
def record_pop(*args, **kwargs):
|
||||
outcomes.append((kwargs.get("output") or {}).get("outcome"))
|
||||
logical_outputs.append(kwargs.get("output") or {})
|
||||
return original_pop(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(relay.scope, "pop", record_pop)
|
||||
@@ -162,7 +171,93 @@ def test_auxiliary_provider_fallback_closes_one_real_logical_call(
|
||||
|
||||
assert result.choices[0].message.content == "recovered"
|
||||
assert turn.logical_llm_calls == {}
|
||||
assert outcomes == ["success"]
|
||||
assert logical_outputs == [
|
||||
{
|
||||
"model": "openrouter/test-model",
|
||||
"outcome": "success",
|
||||
"provider": "openrouter",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_auxiliary_provider_fallback_records_one_terminal_model_route(
|
||||
relay_turn,
|
||||
tmp_path,
|
||||
):
|
||||
relay, turn = relay_turn
|
||||
store = SharedMetricsStore(
|
||||
tmp_path / "metrics.sqlite3",
|
||||
tmp_path / "outbox",
|
||||
)
|
||||
subscriber = SharedMetricsSubscriber(
|
||||
store,
|
||||
"test-version",
|
||||
runtime_id=turn.lease.host.runtime_id,
|
||||
)
|
||||
subscriber_name = "test.auxiliary-model-route"
|
||||
relay.subscribers.register(subscriber_name, subscriber)
|
||||
turn.lease.host.retain_managed_execution(subscriber_name)
|
||||
responses = iter([
|
||||
SimpleNamespace(model="failed/model", choices=[]),
|
||||
SimpleNamespace(
|
||||
model="Accepted/Model",
|
||||
choices=[SimpleNamespace(message=SimpleNamespace(content="recovered"))],
|
||||
),
|
||||
])
|
||||
client = SimpleNamespace(
|
||||
chat=SimpleNamespace(
|
||||
completions=SimpleNamespace(
|
||||
create=lambda **_kwargs: next(responses),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
@auxiliary_client._relay_auxiliary_call
|
||||
def run(task):
|
||||
auxiliary_client._set_relay_auxiliary_route(
|
||||
"nvidia",
|
||||
"failed/configured-model",
|
||||
"chat_completions",
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="invalid response"):
|
||||
auxiliary_client._validate_llm_response(
|
||||
auxiliary_client._relay_sync_completion(
|
||||
client,
|
||||
{"model": "failed/configured-model", "messages": []},
|
||||
),
|
||||
task,
|
||||
)
|
||||
auxiliary_client._set_relay_auxiliary_route(
|
||||
"OpenRouter",
|
||||
"fallback/configured-model",
|
||||
"chat_completions",
|
||||
)
|
||||
return auxiliary_client._validate_llm_response(
|
||||
auxiliary_client._relay_sync_completion(
|
||||
client,
|
||||
{"model": "fallback/configured-model", "messages": []},
|
||||
),
|
||||
task,
|
||||
)
|
||||
|
||||
try:
|
||||
result = run("compression")
|
||||
relay.subscribers.flush()
|
||||
finally:
|
||||
turn.lease.host.release_managed_execution(subscriber_name)
|
||||
relay.subscribers.deregister(subscriber_name)
|
||||
|
||||
assert result.choices[0].message.content == "recovered"
|
||||
snapshot = store.counter_snapshot()
|
||||
assert len(snapshot) == 1
|
||||
assert snapshot[0]["metric_name"] == MODEL_ROUTE_METRIC
|
||||
assert snapshot[0]["hermes_version"] == "test-version"
|
||||
assert snapshot[0]["dimensions"] == {
|
||||
"model": "accepted/model",
|
||||
"provider": "openrouter",
|
||||
}
|
||||
assert snapshot[0]["value"] == 1
|
||||
assert snapshot[0]["packaged_value"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -192,8 +287,8 @@ async def test_async_auxiliary_attempt_uses_inherited_relay_adapter(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
relay_llm,
|
||||
"complete_logical_call",
|
||||
lambda request_id, *, outcome: logical_completions.append(
|
||||
(request_id, outcome)
|
||||
lambda request_id, *, outcome, model_name, provider_name, response_model_name: logical_completions.append(
|
||||
(request_id, outcome, model_name, provider_name, response_model_name)
|
||||
),
|
||||
)
|
||||
|
||||
@@ -219,7 +314,13 @@ async def test_async_auxiliary_attempt_uses_inherited_relay_adapter(monkeypatch)
|
||||
assert captured["metadata"]["call_role"] == "auxiliary:title_generation"
|
||||
assert captured["defer_logical_completion"] is True
|
||||
assert logical_completions == [
|
||||
(captured["metadata"]["api_request_id"], "success")
|
||||
(
|
||||
captured["metadata"]["api_request_id"],
|
||||
"success",
|
||||
"claude-test",
|
||||
"anthropic",
|
||||
None,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@@ -235,11 +336,11 @@ def test_partial_auxiliary_stream_failure_closes_before_recovery(
|
||||
_relay, turn = relay_turn
|
||||
consumer = "test.partial-auxiliary-stream-failure"
|
||||
turn.lease.host.retain_managed_execution(consumer)
|
||||
outcomes = []
|
||||
logical_outputs = []
|
||||
original_pop = turn.lease.host.relay.scope.pop
|
||||
|
||||
def record_pop(*args, **kwargs):
|
||||
outcomes.append((kwargs.get("output") or {}).get("outcome"))
|
||||
logical_outputs.append(kwargs.get("output") or {})
|
||||
return original_pop(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(turn.lease.host.relay.scope, "pop", record_pop)
|
||||
@@ -317,15 +418,30 @@ def test_partial_auxiliary_stream_failure_closes_before_recovery(
|
||||
next(stream)
|
||||
|
||||
assert caught.value is provider_error
|
||||
assert outcomes == ["failed"]
|
||||
assert logical_outputs == [
|
||||
{
|
||||
"model": "test-model",
|
||||
"outcome": "failed",
|
||||
"provider": "openrouter",
|
||||
}
|
||||
]
|
||||
assert turn.logical_llm_calls == {}
|
||||
|
||||
result = recover("moa")
|
||||
|
||||
assert result.choices[0].message.content == "recovered"
|
||||
assert outcomes == ["failed", "success"]
|
||||
assert logical_outputs == [
|
||||
{
|
||||
"model": "test-model",
|
||||
"outcome": "failed",
|
||||
"provider": "openrouter",
|
||||
},
|
||||
{
|
||||
"model": "test-model",
|
||||
"outcome": "success",
|
||||
"provider": "openrouter",
|
||||
},
|
||||
]
|
||||
assert turn.logical_llm_calls == {}
|
||||
finally:
|
||||
turn.lease.host.release_managed_execution(consumer)
|
||||
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from agent import relay_runtime
|
||||
from hermes_cli.observability import shared_metrics as shared_metrics_module
|
||||
from hermes_cli.observability.shared_metrics import SharedMetricsStore
|
||||
from hermes_cli.observability.shared_metrics_contract import (
|
||||
@@ -347,6 +348,43 @@ def test_model_call_fields_report_terminal_model_and_provider_without_a_catalog(
|
||||
}
|
||||
|
||||
|
||||
def test_auxiliary_logical_scope_projects_one_normalized_terminal_route():
|
||||
event = SimpleNamespace(
|
||||
kind="scope",
|
||||
category="function",
|
||||
name=relay_runtime.LOGICAL_LLM_SCOPE,
|
||||
scope_category="end",
|
||||
category_profile=None,
|
||||
data={
|
||||
"model": "Accepted/Model",
|
||||
"outcome": "success",
|
||||
"provider": "OpenRouter",
|
||||
},
|
||||
metadata={
|
||||
relay_runtime.RUNTIME_SCHEMA_KEY: relay_runtime.RUNTIME_SCHEMA_VERSION,
|
||||
relay_runtime.RUNTIME_INSTANCE_KEY: "runtime-1",
|
||||
"hermes.call_role": "auxiliary:compression",
|
||||
},
|
||||
)
|
||||
|
||||
assert model_call_dimensions(event) == {
|
||||
"model": "accepted/model",
|
||||
"provider": "openrouter",
|
||||
}
|
||||
|
||||
event.data.update({
|
||||
"model": "configured/model",
|
||||
"response_model": "malformed response model",
|
||||
})
|
||||
assert model_call_dimensions(event) == {
|
||||
"model": "configured/model",
|
||||
"provider": "openrouter",
|
||||
}
|
||||
|
||||
event.metadata["hermes.call_role"] = "primary"
|
||||
assert model_call_dimensions(event) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response_model",
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user