From dfb8c1bd4d0c690ccd5eea653e17e3d94f5ea480 Mon Sep 17 00:00:00 2001 From: Alex Fournier Date: Wed, 29 Jul 2026 14:09:47 -0700 Subject: [PATCH] fix(observability): preserve shared metrics compatibility Signed-off-by: Alex Fournier --- docs/observability/relay-shared-metrics.md | 6 + .../hermes.shared_metrics.v1.schema.json | 66 ++- .../hermes.shared_metrics.v2.schema.json | 385 ++++++++++++++++++ hermes_cli/observability/shared_metrics.py | 6 +- .../observability/shared_metrics_contract.py | 58 ++- .../shared_metrics_subscriber.py | 4 +- scripts/smoke_nemo_relay_shared_metrics.py | 16 +- tests/hermes_cli/test_relay_shared_metrics.py | 181 +++++++- .../test_relay_shared_metrics_runtime.py | 12 +- tests/plugins/test_nemo_relay_plugin.py | 2 +- 10 files changed, 686 insertions(+), 50 deletions(-) create mode 100644 hermes_cli/observability/schemas/hermes.shared_metrics.v2.schema.json diff --git a/docs/observability/relay-shared-metrics.md b/docs/observability/relay-shared-metrics.md index 4e2ea24fe3..c32430c215 100644 --- a/docs/observability/relay-shared-metrics.md +++ b/docs/observability/relay-shared-metrics.md @@ -74,6 +74,9 @@ lowercased and structurally bounded, but they are not normalized through a checked-in model catalog. Pricing and model-family classification belong to the metrics backend. Prompts, responses, endpoints, errors, session IDs, task IDs, and request IDs are not included in the metrics event or package. +New calls use `hermes.model_route.count`. The previous +`hermes.model_call.count` contract remains readable only so pending local +counters created by older builds can be exported without losing data. Each task run is a Relay `Function` scope named `hermes.task_run`, parented to the owning Hermes session. The start counter contains only bounded execution @@ -99,6 +102,9 @@ files are immutable delta documents that conform to a closed JSON schema and are written with atomic replacement. Fully packaged aggregate rows and successfully exported package rows and files are retained locally for 30 days. Pending package rows and counters with unexported deltas are never pruned. +Package schema v1 remains unchanged for existing outbox files. New packages +use v2, which accepts both the retired model-call contract and the current +model-route contract so upgrades can drain pending counters safely. Each package contains an `install_id` generated as a random UUID. Despite the schema field name, its current scope is one `HERMES_HOME`, so it is more diff --git a/hermes_cli/observability/schemas/hermes.shared_metrics.v1.schema.json b/hermes_cli/observability/schemas/hermes.shared_metrics.v1.schema.json index 2e03a2e6f5..68496e5ab6 100644 --- a/hermes_cli/observability/schemas/hermes.shared_metrics.v1.schema.json +++ b/hermes_cli/observability/schemas/hermes.shared_metrics.v1.schema.json @@ -94,21 +94,63 @@ "type": "object", "additionalProperties": false, "required": [ - "model", - "provider" + "call_role", + "locality", + "model_family", + "outcome", + "provider_family" ], "properties": { - "model": { - "type": "string", - "minLength": 1, - "maxLength": 256, - "pattern": "^[a-z0-9][a-z0-9._:/@+\\-]*$" + "call_role": { + "const": "primary" }, - "provider": { - "type": "string", - "minLength": 1, - "maxLength": 64, - "pattern": "^[a-z0-9][a-z0-9._:/@+\\-]*$" + "locality": { + "enum": [ + "local", + "remote", + "unknown" + ] + }, + "model_family": { + "enum": [ + "claude", + "deepseek", + "gemini", + "gemma", + "glm", + "gpt", + "grok", + "kimi", + "llama", + "minimax", + "mimo", + "mistral", + "nemotron", + "nova", + "o1", + "o3", + "o4", + "qwen", + "step", + "trinity", + "unknown" + ] + }, + "outcome": { + "enum": [ + "cancelled", + "failed", + "success" + ] + }, + "provider_family": { + "enum": [ + "aggregator", + "custom", + "direct", + "local", + "unknown" + ] } } }, diff --git a/hermes_cli/observability/schemas/hermes.shared_metrics.v2.schema.json b/hermes_cli/observability/schemas/hermes.shared_metrics.v2.schema.json new file mode 100644 index 0000000000..112b9f3ce8 --- /dev/null +++ b/hermes_cli/observability/schemas/hermes.shared_metrics.v2.schema.json @@ -0,0 +1,385 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:hermes-agent:schema:shared-metrics:v2", + "title": "Hermes Shared Metrics Package v2", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "package_id", + "install_id", + "period_start", + "period_end", + "generated_at", + "resource", + "metrics" + ], + "properties": { + "schema_version": { + "const": "hermes.shared_metrics.v2" + }, + "package_id": { + "$ref": "#/$defs/uuid" + }, + "install_id": { + "description": "Random persistent identifier scoped to one HERMES_HOME; local-only in schema v2.", + "$ref": "#/$defs/uuid" + }, + "period_start": { + "type": "string", + "format": "date-time" + }, + "period_end": { + "type": "string", + "format": "date-time" + }, + "generated_at": { + "type": "string", + "format": "date-time" + }, + "resource": { + "type": "object", + "additionalProperties": false, + "required": [ + "hermes_version" + ], + "properties": { + "hermes_version": { + "type": "string", + "minLength": 1, + "maxLength": 64 + } + } + }, + "metrics": { + "type": "array", + "minItems": 1, + "items": { + "oneOf": [ + { + "$ref": "#/$defs/model_call_counter" + }, + { + "$ref": "#/$defs/model_route_counter" + }, + { + "$ref": "#/$defs/task_started_counter" + }, + { + "$ref": "#/$defs/task_finished_counter" + } + ] + } + } + }, + "$defs": { + "uuid": { + "type": "string", + "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$" + }, + "model_call_counter": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type", + "dimensions", + "value" + ], + "properties": { + "name": { + "const": "hermes.model_call.count" + }, + "type": { + "const": "counter" + }, + "dimensions": { + "type": "object", + "additionalProperties": false, + "required": [ + "call_role", + "locality", + "model_family", + "outcome", + "provider_family" + ], + "properties": { + "call_role": { + "const": "primary" + }, + "locality": { + "enum": [ + "local", + "remote", + "unknown" + ] + }, + "model_family": { + "enum": [ + "claude", + "deepseek", + "gemini", + "gemma", + "glm", + "gpt", + "grok", + "kimi", + "llama", + "minimax", + "mimo", + "mistral", + "nemotron", + "nova", + "o1", + "o3", + "o4", + "qwen", + "step", + "trinity", + "unknown" + ] + }, + "outcome": { + "enum": [ + "cancelled", + "failed", + "success" + ] + }, + "provider_family": { + "enum": [ + "aggregator", + "custom", + "direct", + "local", + "unknown" + ] + } + } + }, + "value": { + "type": "integer", + "minimum": 1 + } + } + }, + "model_route_counter": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type", + "dimensions", + "value" + ], + "properties": { + "name": { + "const": "hermes.model_route.count" + }, + "type": { + "const": "counter" + }, + "dimensions": { + "type": "object", + "additionalProperties": false, + "required": [ + "model", + "provider" + ], + "properties": { + "model": { + "type": "string", + "minLength": 1, + "maxLength": 256, + "pattern": "^[a-z0-9][a-z0-9._:/@+\\-]*$" + }, + "provider": { + "type": "string", + "minLength": 1, + "maxLength": 64, + "pattern": "^[a-z0-9][a-z0-9._:/@+\\-]*$" + } + } + }, + "value": { + "type": "integer", + "minimum": 1 + } + } + }, + "task_started_counter": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type", + "dimensions", + "value" + ], + "properties": { + "name": { + "const": "hermes.task_run.started" + }, + "type": { + "const": "counter" + }, + "dimensions": { + "type": "object", + "additionalProperties": false, + "required": [ + "entrypoint", + "execution_surface" + ], + "properties": { + "entrypoint": { + "$ref": "#/$defs/task_entrypoint" + }, + "execution_surface": { + "$ref": "#/$defs/execution_surface" + } + } + }, + "value": { + "type": "integer", + "minimum": 1 + } + } + }, + "task_finished_counter": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "type", + "dimensions", + "value" + ], + "properties": { + "name": { + "const": "hermes.task_run.finished" + }, + "type": { + "const": "counter" + }, + "dimensions": { + "type": "object", + "additionalProperties": false, + "required": [ + "duration_bucket", + "end_reason", + "entrypoint", + "execution_surface", + "model_call_count_bucket", + "outcome", + "retry_count_bucket", + "termination", + "tool_call_count_bucket" + ], + "properties": { + "duration_bucket": { + "$ref": "#/$defs/duration_bucket" + }, + "end_reason": { + "enum": [ + "approval_denied", + "completed", + "failed", + "guardrail_blocked", + "iteration_limit", + "system_aborted", + "timed_out", + "unknown", + "user_cancelled" + ] + }, + "entrypoint": { + "$ref": "#/$defs/task_entrypoint" + }, + "execution_surface": { + "$ref": "#/$defs/execution_surface" + }, + "model_call_count_bucket": { + "$ref": "#/$defs/count_bucket" + }, + "outcome": { + "enum": [ + "cancelled", + "failed", + "success", + "timed_out", + "unknown" + ] + }, + "retry_count_bucket": { + "$ref": "#/$defs/count_bucket" + }, + "termination": { + "enum": [ + "none", + "system_aborted", + "timed_out", + "unknown", + "user_cancelled" + ] + }, + "tool_call_count_bucket": { + "$ref": "#/$defs/count_bucket" + } + } + }, + "value": { + "type": "integer", + "minimum": 1 + } + } + }, + "execution_surface": { + "enum": [ + "api", + "batch", + "cli", + "desktop", + "gateway", + "other", + "python", + "scheduled_task", + "tui", + "unknown" + ] + }, + "task_entrypoint": { + "enum": [ + "api", + "background", + "batch", + "delegated", + "gateway_message", + "interactive", + "other", + "python", + "scheduled_task", + "unknown" + ] + }, + "duration_bucket": { + "enum": [ + "1s_to_5s", + "2m_to_10m", + "30s_to_2m", + "5s_to_30s", + "gte_10m", + "lt_1s" + ] + }, + "count_bucket": { + "enum": [ + "0", + "1", + "2", + "3_to_5", + "6_to_10", + "gte_11" + ] + } + } +} diff --git a/hermes_cli/observability/shared_metrics.py b/hermes_cli/observability/shared_metrics.py index 666053e0d1..186924576d 100644 --- a/hermes_cli/observability/shared_metrics.py +++ b/hermes_cli/observability/shared_metrics.py @@ -18,12 +18,12 @@ from utils import atomic_json_write from .shared_metrics_contract import ( COUNTER_METRICS, - MODEL_CALL_METRIC, + MODEL_ROUTE_METRIC, counter_dimensions_are_valid, ) -_PACKAGE_SCHEMA_VERSION = "hermes.shared_metrics.v1" +_PACKAGE_SCHEMA_VERSION = "hermes.shared_metrics.v2" _STORE_SCHEMA_VERSION = "1" _BUSY_TIMEOUT_MS = 250 _SCHEMA_BUSY_TIMEOUT_MS = 5_000 @@ -62,7 +62,7 @@ class SharedMetricsStore: hermes_version: str, ) -> None: """Increment the terminal model-call counter for the current UTC day.""" - self.record_counter(MODEL_CALL_METRIC, dimensions, hermes_version) + self.record_counter(MODEL_ROUTE_METRIC, dimensions, hermes_version) def record_counter( self, diff --git a/hermes_cli/observability/shared_metrics_contract.py b/hermes_cli/observability/shared_metrics_contract.py index 63c6a70698..7284ec3463 100644 --- a/hermes_cli/observability/shared_metrics_contract.py +++ b/hermes_cli/observability/shared_metrics_contract.py @@ -7,12 +7,13 @@ from typing import Any from agent.relay_runtime import RUNTIME_INSTANCE_KEY SCHEMA_KEY = "hermes.metrics.schema_version" -SCHEMA_VERSION = "hermes.metrics.event.v1" +SCHEMA_VERSION = "hermes.metrics.event.v2" MODEL_CALL_SCOPE = "hermes.model_call" MODEL_CALL_PROFILE_MODEL = "unknown" TASK_SCOPE = "hermes.task_run" SUBSCRIBER_NAME = "hermes.nemo_relay.shared_metrics" -MODEL_CALL_METRIC = "hermes.model_call.count" +LEGACY_MODEL_CALL_METRIC = "hermes.model_call.count" +MODEL_ROUTE_METRIC = "hermes.model_route.count" TASK_STARTED_METRIC = "hermes.task_run.started" TASK_FINISHED_METRIC = "hermes.task_run.finished" MODEL_IDENTIFIER_MAX_LENGTH = 256 @@ -90,7 +91,48 @@ COUNT_BUCKETS: frozenset[str] = frozenset({ "gte_11", }) +_LEGACY_PROVIDER_FAMILIES = frozenset({ + "aggregator", + "custom", + "direct", + "local", + "unknown", +}) +_LEGACY_MODEL_LOCALITIES = frozenset({"local", "remote", "unknown"}) +_LEGACY_MODEL_OUTCOMES = frozenset({"cancelled", "failed", "success"}) +_LEGACY_MODEL_FAMILIES = frozenset({ + "claude", + "deepseek", + "gemini", + "gemma", + "glm", + "gpt", + "grok", + "kimi", + "llama", + "minimax", + "mimo", + "mistral", + "nemotron", + "nova", + "o1", + "o3", + "o4", + "qwen", + "step", + "trinity", + "unknown", +}) + _COUNTER_DIMENSION_VALUES: dict[str, dict[str, frozenset[str]]] = { + # Retained only so pre-v2 pending rows remain packageable. + LEGACY_MODEL_CALL_METRIC: { + "call_role": frozenset({"primary"}), + "locality": _LEGACY_MODEL_LOCALITIES, + "model_family": _LEGACY_MODEL_FAMILIES, + "outcome": _LEGACY_MODEL_OUTCOMES, + "provider_family": _LEGACY_PROVIDER_FAMILIES, + }, TASK_STARTED_METRIC: { "entrypoint": TASK_ENTRYPOINTS, "execution_surface": EXECUTION_SURFACES, @@ -107,9 +149,11 @@ _COUNTER_DIMENSION_VALUES: dict[str, dict[str, frozenset[str]]] = { "tool_call_count_bucket": COUNT_BUCKETS, }, } -COUNTER_METRICS: frozenset[str] = frozenset( - {*_COUNTER_DIMENSION_VALUES, MODEL_CALL_METRIC} -) +COUNTER_METRICS: frozenset[str] = frozenset({ + MODEL_ROUTE_METRIC, + TASK_FINISHED_METRIC, + TASK_STARTED_METRIC, +}) def counter_dimensions_are_valid( @@ -117,7 +161,7 @@ def counter_dimensions_are_valid( dimensions: dict[str, Any], ) -> bool: """Return whether dimensions match one closed shared-metric contract.""" - if metric_name == MODEL_CALL_METRIC: + if metric_name == MODEL_ROUTE_METRIC: return ( set(dimensions) == {"model", "provider"} and dimensions["model"] @@ -172,7 +216,7 @@ def model_call_dimensions(event: Any) -> dict[str, str] | None: if not isinstance(data, dict) or set(data) != expected_fields: return None dimensions = {field: data.get(field) for field in sorted(expected_fields)} - if not counter_dimensions_are_valid(MODEL_CALL_METRIC, dimensions): + if not counter_dimensions_are_valid(MODEL_ROUTE_METRIC, dimensions): return None return dimensions diff --git a/hermes_cli/observability/shared_metrics_subscriber.py b/hermes_cli/observability/shared_metrics_subscriber.py index 677308503a..259a6f6b4a 100644 --- a/hermes_cli/observability/shared_metrics_subscriber.py +++ b/hermes_cli/observability/shared_metrics_subscriber.py @@ -9,7 +9,7 @@ from typing import Any from agent.relay_runtime import RUNTIME_INSTANCE_KEY from .shared_metrics import SharedMetricsStore -from .shared_metrics_contract import MODEL_CALL_METRIC, model_call_dimensions, task_counter +from .shared_metrics_contract import MODEL_ROUTE_METRIC, model_call_dimensions, task_counter logger = logging.getLogger(__name__) @@ -44,7 +44,7 @@ class SharedMetricsSubscriber: ): return dimensions = model_call_dimensions(event) - metric_name = MODEL_CALL_METRIC + metric_name = MODEL_ROUTE_METRIC if dimensions is None: task_metric = task_counter(event) if task_metric is None: diff --git a/scripts/smoke_nemo_relay_shared_metrics.py b/scripts/smoke_nemo_relay_shared_metrics.py index 9e611ae215..a8bf4adb01 100644 --- a/scripts/smoke_nemo_relay_shared_metrics.py +++ b/scripts/smoke_nemo_relay_shared_metrics.py @@ -222,7 +222,7 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]: ] by_name = {counter["name"]: counter for counter in counters} if set(by_name) != { - "hermes.model_call.count", + "hermes.model_route.count", "hermes.task_run.finished", "hermes.task_run.started", }: @@ -230,7 +230,7 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]: f"Unexpected SQLite counters:\n{json.dumps(counters, indent=2)}" ) expected_model = { - "name": "hermes.model_call.count", + "name": "hermes.model_route.count", "dimensions": { "model": MODEL_CANARY, "provider": "custom", @@ -238,9 +238,9 @@ def _validate_store(database_path: Path) -> list[dict[str, Any]]: "value": 1, "packaged_value": 1, } - if by_name["hermes.model_call.count"] != expected_model: + if by_name["hermes.model_route.count"] != expected_model: raise AssertionError( - f"Unexpected model counter: {by_name['hermes.model_call.count']}" + f"Unexpected model counter: {by_name['hermes.model_route.count']}" ) expected_start = { "name": "hermes.task_run.started", @@ -299,20 +299,20 @@ def _validate_package(outbox: Path, schema_path: Path) -> tuple[Path, dict[str, ) metrics = {metric["name"]: metric for metric in package.get("metrics", [])} if set(metrics) != { - "hermes.model_call.count", + "hermes.model_route.count", "hermes.task_run.finished", "hermes.task_run.started", }: raise AssertionError( f"Unexpected package metrics:\n{json.dumps(package.get('metrics'), indent=2)}" ) - model_dimensions = metrics["hermes.model_call.count"]["dimensions"] + model_dimensions = metrics["hermes.model_route.count"]["dimensions"] if model_dimensions != { "model": MODEL_CANARY, "provider": "custom", }: raise AssertionError( - f"Unexpected model metric: {metrics['hermes.model_call.count']}" + f"Unexpected model metric: {metrics['hermes.model_route.count']}" ) terminal = metrics["hermes.task_run.finished"] if terminal["dimensions"] != { @@ -422,7 +422,7 @@ def main() -> int: / "hermes_cli" / "observability" / "schemas" - / "hermes.shared_metrics.v1.schema.json", + / "hermes.shared_metrics.v2.schema.json", ) print("Hermes -> NeMo Relay shared-metrics smoke test passed") diff --git a/tests/hermes_cli/test_relay_shared_metrics.py b/tests/hermes_cli/test_relay_shared_metrics.py index 9cfd36cd6e..f193ff2f44 100644 --- a/tests/hermes_cli/test_relay_shared_metrics.py +++ b/tests/hermes_cli/test_relay_shared_metrics.py @@ -24,9 +24,13 @@ from hermes_cli.observability.shared_metrics_contract import ( COUNT_BUCKETS, DURATION_BUCKETS, EXECUTION_SURFACES, + LEGACY_MODEL_CALL_METRIC, MODEL_CALL_PROFILE_MODEL, MODEL_IDENTIFIER_MAX_LENGTH, + MODEL_ROUTE_METRIC, PROVIDER_IDENTIFIER_MAX_LENGTH, + SCHEMA_KEY, + SCHEMA_VERSION, TASK_END_REASONS, TASK_ENTRYPOINTS, TASK_OUTCOMES, @@ -48,13 +52,14 @@ SCHEMA_PATH = ( / "hermes_cli" / "observability" / "schemas" - / "hermes.shared_metrics.v1.schema.json" + / "hermes.shared_metrics.v2.schema.json" ) +LEGACY_SCHEMA_PATH = SCHEMA_PATH.with_name("hermes.shared_metrics.v1.schema.json") -def _schema_validator(): +def _schema_validator(path: Path = SCHEMA_PATH): jsonschema = pytest.importorskip("jsonschema") - schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) + schema = json.loads(path.read_text(encoding="utf-8")) jsonschema.Draft202012Validator.check_schema(schema) return jsonschema.Draft202012Validator( schema, @@ -64,7 +69,7 @@ def _schema_validator(): def _package_dimension_schema() -> dict[str, object]: schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) - return schema["$defs"]["model_call_counter"]["properties"]["dimensions"] + return schema["$defs"]["model_route_counter"]["properties"]["dimensions"] def _task_dimension_schema(kind: str) -> dict[str, object]: @@ -79,6 +84,16 @@ def _dimensions() -> dict[str, str]: } +def _legacy_dimensions() -> dict[str, str]: + return { + "call_role": "primary", + "locality": "remote", + "model_family": "claude", + "outcome": "success", + "provider_family": "direct", + } + + def _record_model_calls_in_process( database_path: str, outbox_directory: str, @@ -106,11 +121,11 @@ def test_model_call_counter_survives_restart_and_exports_only_new_deltas(tmp_pat _schema_validator().validate(first_package) uuid.UUID(first_package["package_id"]) uuid.UUID(first_package["install_id"]) - assert first_package["schema_version"] == "hermes.shared_metrics.v1" + assert first_package["schema_version"] == "hermes.shared_metrics.v2" assert first_package["resource"] == {"hermes_version": "test-version"} assert first_package["metrics"] == [ { - "name": "hermes.model_call.count", + "name": MODEL_ROUTE_METRIC, "type": "counter", "dimensions": _dimensions(), "value": 2, @@ -135,6 +150,120 @@ def test_model_call_counter_survives_restart_and_exports_only_new_deltas(tmp_pat assert restarted.counter_snapshot()[0]["packaged_value"] == 3 +def test_v2_package_preserves_pending_v1_model_counters(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + period_start = shared_metrics_module._utc_now().date().isoformat() + legacy_dimensions_json = json.dumps( + _legacy_dimensions(), + sort_keys=True, + separators=(",", ":"), + ) + with sqlite3.connect(database_path) as connection: + connection.execute( + """ + INSERT INTO counter_aggregates( + period_start, + metric_name, + hermes_version, + dimensions_json, + value, + packaged_value + ) VALUES (?, ?, ?, ?, 3, 0) + """, + ( + period_start, + LEGACY_MODEL_CALL_METRIC, + "test-version", + legacy_dimensions_json, + ), + ) + store.record_model_call(_dimensions(), "test-version") + + [package_path] = store.create_and_export_package() + package = json.loads(package_path.read_text(encoding="utf-8")) + _schema_validator().validate(package) + + assert package["schema_version"] == "hermes.shared_metrics.v2" + assert package["metrics"] == [ + { + "name": LEGACY_MODEL_CALL_METRIC, + "type": "counter", + "dimensions": _legacy_dimensions(), + "value": 3, + }, + { + "name": MODEL_ROUTE_METRIC, + "type": "counter", + "dimensions": _dimensions(), + "value": 1, + }, + ] + assert all( + row["value"] == row["packaged_value"] for row in store.counter_snapshot() + ) + + +def test_v1_outbox_package_exports_unchanged_after_upgrade(tmp_path): + database_path = tmp_path / "metrics.sqlite3" + outbox_directory = tmp_path / "outbox" + store = SharedMetricsStore(database_path, outbox_directory) + package_id = str(uuid.uuid4()) + payload = { + "schema_version": "hermes.shared_metrics.v1", + "package_id": package_id, + "install_id": str(uuid.uuid4()), + "period_start": "2026-07-28T00:00:00Z", + "period_end": "2026-07-29T00:00:00Z", + "generated_at": "2026-07-29T01:00:00Z", + "resource": {"hermes_version": "legacy-version"}, + "metrics": [ + { + "name": LEGACY_MODEL_CALL_METRIC, + "type": "counter", + "dimensions": _legacy_dimensions(), + "value": 2, + } + ], + } + payload_json = json.dumps(payload, sort_keys=True, separators=(",", ":")) + _schema_validator(LEGACY_SCHEMA_PATH).validate(payload) + with sqlite3.connect(database_path) as connection: + connection.execute( + """ + INSERT INTO package_outbox( + package_id, + period_start, + period_end, + payload_json, + created_at + ) VALUES (?, ?, ?, ?, ?) + """, + ( + package_id, + payload["period_start"], + payload["period_end"], + payload_json, + payload["generated_at"], + ), + ) + + assert store.create_and_export_package() == [ + outbox_directory / f"{package_id}.json" + ] + exported = json.loads( + (outbox_directory / f"{package_id}.json").read_text(encoding="utf-8") + ) + assert exported == payload + with sqlite3.connect(database_path) as connection: + [persisted_payload_json] = connection.execute( + "SELECT payload_json FROM package_outbox WHERE package_id = ?", + (package_id,), + ).fetchone() + assert persisted_payload_json == payload_json + + def test_due_export_runs_once_per_utc_day_and_catches_up_pending_deltas( tmp_path, monkeypatch ): @@ -152,7 +281,7 @@ def test_due_export_runs_once_per_utc_day_and_catches_up_pending_deltas( assert len(list((tmp_path / "outbox").glob("*.json"))) == 1 assert store.counter_snapshot()[0] == { "period_start": "2026-07-28", - "metric_name": "hermes.model_call.count", + "metric_name": MODEL_ROUTE_METRIC, "hermes_version": "test-version", "dimensions": _dimensions(), "value": 2, @@ -173,8 +302,10 @@ def test_due_export_runs_once_per_utc_day_and_catches_up_pending_deltas( def test_package_schema_matches_the_model_call_contract(): + schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) properties = _package_dimension_schema()["properties"] + assert schema["properties"]["schema_version"]["const"] == "hermes.shared_metrics.v2" assert set(properties) == {"model", "provider"} assert properties["model"]["maxLength"] == MODEL_IDENTIFIER_MAX_LENGTH assert properties["provider"]["maxLength"] == PROVIDER_IDENTIFIER_MAX_LENGTH @@ -182,6 +313,21 @@ def test_package_schema_matches_the_model_call_contract(): assert "enum" not in properties["provider"] +def test_v1_package_schema_retains_the_legacy_model_contract(): + schema = json.loads(LEGACY_SCHEMA_PATH.read_text(encoding="utf-8")) + model_counter = schema["$defs"]["model_call_counter"] + + assert schema["properties"]["schema_version"]["const"] == "hermes.shared_metrics.v1" + assert model_counter["properties"]["name"]["const"] == LEGACY_MODEL_CALL_METRIC + assert set(model_counter["properties"]["dimensions"]["properties"]) == { + "call_role", + "locality", + "model_family", + "outcome", + "provider_family", + } + + def test_package_schema_matches_the_task_contract(): schema = json.loads(SCHEMA_PATH.read_text(encoding="utf-8")) start = _task_dimension_schema("task_started_counter")["properties"] @@ -377,7 +523,7 @@ def test_subscriber_contract_rejects_unknown_fields_and_dimension_values(): category_profile={"model_name": MODEL_CALL_PROFILE_MODEL}, name="hermes.model_call", scope_category="end", - metadata={"hermes.metrics.schema_version": "hermes.metrics.event.v1"}, + metadata={SCHEMA_KEY: SCHEMA_VERSION}, data=_dimensions(), ) @@ -410,7 +556,7 @@ def test_task_subscriber_contract_accepts_only_bounded_scope_events(): category_profile=None, name="hermes.task_run", scope_category="start", - metadata={"hermes.metrics.schema_version": "hermes.metrics.event.v1"}, + metadata={SCHEMA_KEY: SCHEMA_VERSION}, data={"entrypoint": "interactive", "execution_surface": "cli"}, ) assert task_counter(start) == ( @@ -534,7 +680,7 @@ def test_store_rejects_dimensions_outside_the_metric_contract(tmp_path): with pytest.raises(ValueError, match="Unsupported dimensions"): store.record_counter( - "hermes.model_call.count", + MODEL_ROUTE_METRIC, {"prompt": "must-not-be-persisted"}, "test-version", ) @@ -542,6 +688,19 @@ def test_store_rejects_dimensions_outside_the_metric_contract(tmp_path): assert store.counter_snapshot() == [] +def test_store_does_not_record_the_retired_model_metric(tmp_path): + store = SharedMetricsStore(tmp_path / "metrics.sqlite3", tmp_path / "outbox") + + with pytest.raises(ValueError, match="Unsupported shared metric"): + store.record_counter( + LEGACY_MODEL_CALL_METRIC, + _legacy_dimensions(), + "test-version", + ) + + assert store.counter_snapshot() == [] + + def test_package_builder_rejects_tampered_dimensions(tmp_path): database_path = tmp_path / "metrics.sqlite3" outbox_directory = tmp_path / "outbox" @@ -708,7 +867,7 @@ def test_package_export_does_not_chase_concurrent_updates(tmp_path, monkeypatch) assert create_calls == 1 assert len(first_paths) == 1 [counter] = store.counter_snapshot() - assert counter["metric_name"] == "hermes.model_call.count" + assert counter["metric_name"] == MODEL_ROUTE_METRIC assert counter["dimensions"] == _dimensions() assert counter["value"] == 2 assert counter["packaged_value"] == 1 diff --git a/tests/hermes_cli/test_relay_shared_metrics_runtime.py b/tests/hermes_cli/test_relay_shared_metrics_runtime.py index d5332ef691..ae3f8b3650 100644 --- a/tests/hermes_cli/test_relay_shared_metrics_runtime.py +++ b/tests/hermes_cli/test_relay_shared_metrics_runtime.py @@ -279,15 +279,15 @@ def test_direct_runtime_records_without_enabling_a_plugin(direct_runtime, tmp_pa package = json.loads(packages[0].read_text(encoding="utf-8")) metrics = {metric["name"]: metric for metric in package["metrics"]} assert set(metrics) == { - "hermes.model_call.count", + "hermes.model_route.count", "hermes.task_run.finished", "hermes.task_run.started", } - assert metrics["hermes.model_call.count"]["dimensions"] == { + assert metrics["hermes.model_route.count"]["dimensions"] == { "model": "claude-sonnet", "provider": "anthropic", } - assert metrics["hermes.model_call.count"]["value"] == 1 + assert metrics["hermes.model_route.count"]["value"] == 1 assert metrics["hermes.task_run.started"] == { "name": "hermes.task_run.started", "type": "counter", @@ -432,8 +432,8 @@ def test_real_binding_drives_lifecycle_aggregation_export_and_snapshot( assert len(by_metric["hermes.task_run.started"]) == 1 assert by_metric["hermes.task_run.started"][0]["value"] == 3 - assert len(by_metric["hermes.model_call.count"]) == 1 - model_counter = by_metric["hermes.model_call.count"][0] + assert len(by_metric["hermes.model_route.count"]) == 1 + model_counter = by_metric["hermes.model_route.count"][0] assert model_counter["dimensions"] == { "model": model_canary, "provider": "custom", @@ -468,7 +468,7 @@ def test_real_binding_drives_lifecycle_aggregation_export_and_snapshot( json.loads(package.read_text(encoding="utf-8")) for package in packages ] for package in package_payloads: - assert package["schema_version"] == "hermes.shared_metrics.v1" + assert package["schema_version"] == "hermes.shared_metrics.v2" for metric in package["metrics"]: key = (metric["name"], tuple(sorted(metric["dimensions"].items()))) package_values[key] = package_values.get(key, 0) + metric["value"] diff --git a/tests/plugins/test_nemo_relay_plugin.py b/tests/plugins/test_nemo_relay_plugin.py index 6081f0660f..249fe62909 100644 --- a/tests/plugins/test_nemo_relay_plugin.py +++ b/tests/plugins/test_nemo_relay_plugin.py @@ -466,7 +466,7 @@ def test_shared_metrics_and_rich_plugin_share_one_core_session( ) assert len(packages) == 1 package = json.loads(packages[0].read_text(encoding="utf-8")) - assert package["metrics"][0]["name"] == "hermes.model_call.count" + assert package["metrics"][0]["name"] == "hermes.model_route.count" assert package["metrics"][0]["value"] == 1 assert (tmp_path / "atif" / "hermes-atif-s1.json").exists()