From e0170c253608afade5e8abbafbf951504fce9804 Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Wed, 26 Aug 2026 15:32:26 +1000 Subject: [PATCH 001/634] docs(observability): record remote-exporter privacy decisions The shared-metrics doc states that a future remote exporter 'must not reuse the persistent local identifier by default' and 'requires a separate product and privacy decision covering consent, identity scope, rotation or keyed pseudonymization, reset behavior, retention, and deletion'. That exporter is now being built. Appendix A answers each of those six items before any code lands, so the reasoning is reviewable on its own and survives the implementation: - consent is a separate opt-in from collection, gated on the PERIOD a package covers rather than when it was created (a period is split across packages made on different days, so a created_at gate would send a period's tail while dropping its head and silently undercount the first day) - the transmitted identifier is HMAC-SHA256(local-only salt, install_id), never install_id itself - the salt rotates every 30 days - reset gives a new remote identity but cannot unsend - local retention is unchanged; send state does not extend it - there is no self-service remote deletion, and the user -> derived-id lookup that would enable one is deliberately not built A.7 additionally records what the outbox directory IS (the user's local history, not a send queue) because misreading it would have led to deleting user data on acknowledgement. --- docs/observability/relay-shared-metrics.md | 156 +++++++++++++++++++++ 1 file changed, 156 insertions(+) diff --git a/docs/observability/relay-shared-metrics.md b/docs/observability/relay-shared-metrics.md index 146590dc99..5b5ce0f8d4 100644 --- a/docs/observability/relay-shared-metrics.md +++ b/docs/observability/relay-shared-metrics.md @@ -231,6 +231,13 @@ the persistent local identifier by default. It requires a separate product and privacy decision covering consent, identity scope, rotation or keyed pseudonymization, reset behavior, retention, and deletion. +> That exporter is now being built as Phase 2 of the Hermes telemetry project. +> The decisions this paragraph asks for are recorded in +> [Appendix A](#appendix-a-remote-exporter-decisions-phase-2). Until Phase 2 +> ships, the statement above still describes shipped behaviour: nothing is +> transmitted, and transmission stays opt-in behind a config key that is off by +> default. + The install identity is scoped to one `HERMES_HOME`. To reset it, stop Hermes processes and remove `$HERMES_HOME/telemetry/shared_metrics`. This deliberately removes the old identity, aggregate database, and queued local packages @@ -257,3 +264,152 @@ verifies model, provider, task, tool, and skill counters in SQLite, validates all exported delta packages against the closed schema, verifies the pseudonymous client-active counter, and checks that prompt, response, tool-call ID, tool-result, and skill-name canaries are absent from the packages. + +## Appendix A: Remote Exporter Decisions (Phase 2) + +Status: **decided, not yet built.** This appendix answers the product and +privacy questions that "Current Slices" defers to a future remote exporter. It +records what was decided and why, so the reasoning survives the implementation. + +The exporter sends the package files already written under +`$HERMES_HOME/telemetry/shared_metrics/outbox/` to the Hermes telemetry ingest +service. That service validates only the envelope (`schema_version` plus a UUID +`package_id`) and stores the body verbatim in S3. + +### A.1 Consent + +Transmission is a **separate opt-in** from collection, under a new config key: + +```yaml +telemetry: + shared_metrics: + enabled: false # collect locally + send: false # NEW: transmit to the Nous telemetry service +``` + +- `send` defaults to **false**. Collection alone never transmits. +- `send` requires `enabled`. It does **not** imply it: a transmission flag must + not silently switch on collection. `send: true` with `enabled: false` warns + and does nothing. +- Like `enabled`, `send` is profile-owned and is not overridden by + managed-scope configuration. + +**Only packages for periods on or after the opt-in day are ever sent.** The +opt-in day (UTC) is recorded when `send` first becomes true, and any package +whose `period_start` predates it is permanently excluded, however late it was +created. + +The gate is on the **period**, not on the package's creation time. One period +is split across several packages created on different days: a day's first +package is written that day, and a tail package for the same period typically +follows the next day. Gating on creation time would send a period's tail while +dropping its head, reporting a **silently undercounted** day. Gating on the +period keeps consent forward-only and every transmitted period complete. + +Local history can be up to 30 days old, and that data was collected under a +promise that nothing is uploaded. Honouring consent forward-only costs at most +30 days of backlog we never had permission to send. + +### A.2 Identity scope — the transmitted identifier is derived, not the local one + +`install_id` is the persistent profile-scoped identifier described above. It is +**not transmitted**. Each package sent carries a derived value instead: + +```text +transmitted_id = HMAC-SHA256(key = rotation_salt, message = install_id) +``` + +- `rotation_salt` is random, generated locally, and never leaves the machine. +- The derivation is one-way: the service cannot recover `install_id`. +- Within a rotation window, packages from one profile correlate — so distinct + installs remain countable, which is the primary analytical question. +- Across windows, they do not. + +This satisfies "must not reuse the persistent local identifier by default" +while keeping the data useful. Stripping the identifier entirely was rejected +because "how many installs are reporting" is the first question the data must +answer; sending `install_id` unchanged was rejected because it contradicts the +commitment made above. + +**Byte-identical resends still hold.** The derived value is computed **once**, +when the package is first prepared for sending, and stored alongside the +package (the derived id only — not a second copy of the payload, which is +recomputed deterministically from the stored package). A retry therefore +rebuilds identical bytes even if the salt rotated in between. The contract +requires this: resending a `package_id` with different content is undefined +behaviour. + +### A.3 Rotation + +`rotation_salt` rotates on a fixed schedule (default: every 30 days, aligned to +local history retention). Rotation only affects packages prepared after it; +already-prepared packages keep their derived value so retries stay +byte-identical. + +Rotation bounds long-term linkability without destroying short-term cohort +analysis. A profile is one identity for the length of a window, and an +unrelated identity after it. + +### A.4 Reset behavior + +Removing `$HERMES_HOME/telemetry/shared_metrics` still resets local identity, +aggregates, and package files, exactly as documented above. Two honest +qualifications now apply: + +- Reset also discards `rotation_salt`, so subsequent packages derive a **new** + transmitted identity. Local reset does give a new remote identity. +- Reset **cannot unsend**. Packages already transmitted remain in the ingest + service's storage under their derived identifier. There is no read-back or + delete API in the v1 contract. + +Setting `send: false` stops transmission immediately. It does not delete +previously transmitted packages, and it does not stop local collection. + +### A.5 Retention + +- **Local:** unchanged — 30 days for successfully exported history, and pending + deltas are kept until exported. Send state does **not** extend local + retention: a package that could never be sent is still pruned at 30 days. + Unbounded local growth against a permanently unreachable endpoint is a worse + failure than losing metrics from an install that has been broken for a month. +- **Remote:** raw packages are retained in S3 without expiry in production and + for 30 days in staging. + +### A.6 Deletion + +There is no remote deletion path in the v1 contract, and this appendix does not +invent one. What a user can do: + +| Action | Effect | +|---|---| +| `send: false` | No further packages leave the machine | +| `enabled: false` | Collection stops; existing local state remains | +| Remove `.../shared_metrics` | Local identity, aggregates, and files reset; future sends use a new derived identity | +| Delete already-sent data | Not self-service — requires an operator acting on the S3 bucket | + +If a deletion-on-request obligation is ever taken on, it needs a lookup path +from a user to their derived identifiers. That is deliberately **not** built: +it would require retaining the mapping this design exists to avoid. Any such +change is a new product decision, not an implementation detail. + +### A.7 What the outbox directory is + +Recorded because it was misread once during Phase 2 planning, in a way that +would have deleted user data. + +The directory is **local history, not a send-queue**. `package_outbox` is the +SQLite table; its `exported_at` column means "written to disk", not "sent". +Files are immutable and pruned **by age alone**. + +The ingest contract says senders should delete a package from their outbox on +`202`. **The exporter does not do this.** Deleting on acknowledgement would +repurpose the user's 30-day local history as a transmission queue and destroy +state they were promised. Send state lives in new columns on the +`package_outbox` table instead; the files are untouched by transmission. + +### A.8 Scope note + +The `install_id` field inside the package body is what gets replaced by the +derived value. No other payload field changes, nothing is added, and the +service treats the whole body as opaque. Payload schema evolution therefore +stays a sender-side concern, as before. From e5180ab3df71547b971e884bba1504f665ba80fb Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Wed, 26 Aug 2026 15:39:47 +1000 Subject: [PATCH 002/634] feat(telemetry): add opt-in send config and send-state columns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 1+2 of the shared-metrics exporter. Config: telemetry.shared_metrics.send (default false) and .endpoint (default production), resolved by a new shared_metrics_send_config module. Precedence is HERMES_TELEMETRY_ENDPOINT > config > default; the env var exists so the live staging E2E never has to mutate a user's config. send requires enabled and never implies it — that combination is a misconfiguration the user believes is working, so it logs an ERROR once per process rather than silently doing nothing. Plaintext endpoints are refused unless the host is loopback, so a typo cannot send telemetry in clear text. Per AGENTS.md, outbound telemetry needs a user-facing opt-in, so setup_telemetry now prompts for sending as a second, separate question and force-disables send when collection is turned off. Storage: six additive nullable columns on package_outbox for send bookkeeping. The store schema version deliberately does NOT move — _ensure_schema_in_transaction raises on any version it does not recognise and has no forward-compatibility branch, so bumping it would hard-fail an older Hermes, a second profile on an older build, or a rollback, against the same file. Old readers select named columns and never SELECT *, so the additions are invisible to them. Also corrects the two places that promised telemetry is never uploaded (config_defaults comment and cli-config.yaml.example); leaving them would make them false privacy statements once sending ships. Tests: 26 covering config precedence, the enabled/send relationship, transport safety, fresh-database creation, upgrade from a pre-send database (rows preserved, version pinned, idempotent), and that the shipped export query still runs. Mutation-checked: bumping the schema version fails 5 of them. --- cli-config.yaml.example | 15 +- hermes_cli/config_defaults.py | 20 +- hermes_cli/observability/shared_metrics.py | 37 +++ .../shared_metrics_send_config.py | 114 +++++++++ hermes_cli/setup.py | 30 ++- .../test_shared_metrics_send_config.py | 137 +++++++++++ .../test_shared_metrics_send_migration.py | 224 ++++++++++++++++++ 7 files changed, 569 insertions(+), 8 deletions(-) create mode 100644 hermes_cli/observability/shared_metrics_send_config.py create mode 100644 tests/hermes_cli/test_shared_metrics_send_config.py create mode 100644 tests/hermes_cli/test_shared_metrics_send_migration.py diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 1a8021ff98..2fe2c5e610 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -1782,15 +1782,28 @@ display: # ============================================================================= # Shared metrics are disabled by default. When enabled, Hermes writes only # allowlisted aggregate counters and immutable JSON -# packages under $HERMES_HOME/telemetry/shared_metrics; it does not upload them. +# packages under $HERMES_HOME/telemetry/shared_metrics. # Packages include a random profile-scoped ID that stays stable until this # directory is deleted. It is not derived from hardware, account, or host data. # Successfully exported local history is retained for 30 days; pending deltas # are retained until they can be exported. # This profile-owned choice is not overridden by managed-scope configuration. +# +# Nothing is uploaded unless you also set `send: true`. That is a separate +# opt-in and requires `enabled`; it never turns collection on by itself. +# When sending is on: +# * only packages whose period starts on or after the day you opted in are +# ever transmitted, so data collected beforehand stays on this machine; +# * the profile-scoped ID is NOT sent. Each package carries an HMAC of it, +# keyed by a local-only salt that rotates every 30 days, so installs stay +# countable without shipping a durable identifier. +# See docs/observability/relay-shared-metrics.md (Appendix A) for the full +# consent, identity, rotation, retention, and deletion decisions. telemetry: shared_metrics: enabled: false + send: false + # endpoint: https://telemetry.nousresearch.com/v1/telemetry # ============================================================================= diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 0fb1488316..cf321b4e3d 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -3323,11 +3323,27 @@ DEFAULT_CONFIG = { "profile_build": "ask", }, - # Privacy-safe aggregate metrics written only to this profile's local - # telemetry directory. Collection is opt-in and no remote sink exists. + # Privacy-safe aggregate metrics written to this profile's local telemetry + # directory. Collection is opt-in (``enabled``). Transmission to the Nous + # telemetry service is a SEPARATE opt-in (``send``) and is off by default; + # see docs/observability/relay-shared-metrics.md, Appendix A, for the + # consent, identity, rotation, retention, and deletion decisions. "telemetry": { "shared_metrics": { "enabled": False, + # Transmit exported packages to the Nous telemetry service. + # Requires ``enabled``: it never switches collection on by itself, + # and ``send`` without ``enabled`` is logged as an error rather + # than silently doing nothing. Only packages whose period starts + # on or after the opt-in day are ever sent, so data collected + # before consent stays local. + "send": False, + # Ingest endpoint. Production by default; override for staging or + # a local test server. The HERMES_TELEMETRY_ENDPOINT environment + # variable takes precedence (used by the live E2E so a test never + # has to mutate a user's config). Non-HTTPS is refused unless the + # host is localhost. + "endpoint": "https://telemetry.nousresearch.com/v1/telemetry", }, }, diff --git a/hermes_cli/observability/shared_metrics.py b/hermes_cli/observability/shared_metrics.py index fd42b06230..bf5c1fb0bf 100644 --- a/hermes_cli/observability/shared_metrics.py +++ b/hermes_cli/observability/shared_metrics.py @@ -337,6 +337,7 @@ class SharedMetricsStore: ) """ ) + SharedMetricsStore._add_send_columns(connection) connection.execute( """ INSERT INTO telemetry_state(key, value) @@ -346,6 +347,42 @@ class SharedMetricsStore: (_STORE_SCHEMA_VERSION,), ) + @staticmethod + def _add_send_columns(connection: sqlite3.Connection) -> None: + """Add transmission bookkeeping to ``package_outbox``, idempotently. + + These columns are ADDITIVE and nullable, and the store schema version + is deliberately NOT bumped. ``_ensure_schema_in_transaction`` raises on + any version it does not recognise and has no forward-compatibility + branch, so bumping would make an older Hermes — a second profile on an + older build, or a rollback — hard-fail against the same database file. + Old readers select named columns and never ``SELECT *``, so extra + columns are invisible to them. + """ + existing = { + str(row["name"]) + for row in connection.execute("PRAGMA table_info(package_outbox)") + } + for column, declaration in ( + # When the 202 was received. NULL = never acknowledged. + ("sent_at", "TEXT"), + # NULL/'pending' = eligible, 'sent' = done, 'rejected' = permanent 400. + ("send_state", "TEXT"), + ("send_attempts", "INTEGER NOT NULL DEFAULT 0"), + # Earliest next attempt; enforces backoff across process restarts. + ("next_attempt_at", "TEXT"), + ("last_error", "TEXT"), + # The derived identifier actually transmitted, frozen on the first + # attempt so retries stay byte-identical across a salt rotation. + # Only the ~36-byte id is stored: the body is recomputed from + # payload_json, whose serialisation is deterministic. + ("sent_install_id", "TEXT"), + ): + if column not in existing: + connection.execute( + f"ALTER TABLE package_outbox ADD COLUMN {column} {declaration}" + ) + @staticmethod def _create_counter_aggregates_table(connection: sqlite3.Connection) -> None: connection.execute( diff --git a/hermes_cli/observability/shared_metrics_send_config.py b/hermes_cli/observability/shared_metrics_send_config.py new file mode 100644 index 0000000000..8011c595ab --- /dev/null +++ b/hermes_cli/observability/shared_metrics_send_config.py @@ -0,0 +1,114 @@ +"""Configuration for shared-metrics transmission. + +Collection (``telemetry.shared_metrics.enabled``) and transmission +(``telemetry.shared_metrics.send``) are separate opt-ins. See +``docs/observability/relay-shared-metrics.md`` Appendix A for the consent, +identity, rotation, retention, and deletion decisions behind this module. +""" + +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass +from urllib.parse import urlparse + +logger = logging.getLogger(__name__) + +#: Production ingest endpoint. Overridable by config or environment so the +#: live E2E can target staging without mutating a user's config. +DEFAULT_ENDPOINT = "https://telemetry.nousresearch.com/v1/telemetry" + +#: Environment override, highest precedence. Intended for tests and staging +#: validation, not as the documented user-facing setting (which is config). +ENDPOINT_ENV_VAR = "HERMES_TELEMETRY_ENDPOINT" + +_LOCAL_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "[::1]"}) + +# Module-level latch: the enabled/send mismatch is a static misconfiguration, +# so it is reported once per process instead of on every hook fire. +_warned_send_without_collection = False + + +@dataclass(frozen=True) +class SendConfig: + """Resolved transmission settings.""" + + #: Collection is on. Nothing is packaged or sent without it. + enabled: bool + #: Transmission is on AND permitted (that is, collection is also on). + send: bool + #: Where packages are POSTed. + endpoint: str + + +def _endpoint_is_safe(endpoint: str) -> bool: + """Reject plaintext destinations unless they are loopback. + + Telemetry must not leave a machine in clear text because of a typo in a + config file. Loopback stays allowed so tests can use a local HTTP server. + """ + try: + parsed = urlparse(endpoint) + except ValueError: + return False + if parsed.scheme == "https": + return True + if parsed.scheme == "http": + return (parsed.hostname or "") in _LOCAL_HOSTS + return False + + +def resolve_send_config(config: dict | None) -> SendConfig: + """Resolve transmission settings from config plus the environment. + + Endpoint precedence: ``HERMES_TELEMETRY_ENDPOINT`` > config > production + default. + + ``send`` is returned as False whenever transmission cannot legitimately + happen, so callers never have to re-check the combination. + """ + global _warned_send_without_collection + + raw = config if isinstance(config, dict) else {} + telemetry = raw.get("telemetry") + telemetry = telemetry if isinstance(telemetry, dict) else {} + shared = telemetry.get("shared_metrics") + shared = shared if isinstance(shared, dict) else {} + + enabled = shared.get("enabled") is True + send_requested = shared.get("send") is True + + if send_requested and not enabled: + # Loud, not silent: the user believes telemetry is being sent, and it + # never will be. Error level, once per process. + if not _warned_send_without_collection: + _warned_send_without_collection = True + logger.error( + "telemetry.shared_metrics.send is true but " + "telemetry.shared_metrics.enabled is false — nothing is " + "collected, so nothing can be sent. Enable collection or " + "turn sending off." + ) + return SendConfig(enabled=False, send=False, endpoint=DEFAULT_ENDPOINT) + + endpoint = os.environ.get(ENDPOINT_ENV_VAR) or shared.get("endpoint") + if not isinstance(endpoint, str) or not endpoint.strip(): + endpoint = DEFAULT_ENDPOINT + endpoint = endpoint.strip() + + if send_requested and not _endpoint_is_safe(endpoint): + logger.error( + "Refusing to send shared metrics to %r: telemetry must use https " + "(or a localhost http endpoint for testing).", + endpoint, + ) + return SendConfig(enabled=enabled, send=False, endpoint=endpoint) + + return SendConfig(enabled=enabled, send=send_requested, endpoint=endpoint) + + +def reset_warning_latch_for_tests() -> None: + """Clear the once-per-process error latch (test support only).""" + global _warned_send_without_collection + _warned_send_without_collection = False diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index 4f0d190203..d6497fbc05 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -2428,10 +2428,10 @@ def setup_tools(config: dict, first_install: bool = False): def setup_telemetry(config: dict): - """Configure the local, privacy-safe shared-metrics subscriber.""" + """Configure the local shared-metrics subscriber and optional sending.""" print_header("Shared Metrics") print_info("Shared metrics contain only bounded counters and histograms.") - print_info("Packages stay under this Hermes profile and are not uploaded.") + print_info("Collection is local. Sending them to Nous is a separate opt-in.") telemetry = config.get("telemetry") if not isinstance(telemetry, dict): @@ -2447,10 +2447,30 @@ def setup_telemetry(config: dict): "Enable local shared metrics?", default=current, ) - if shared_metrics["enabled"]: - print_success("Local shared metrics enabled.") - else: + if not shared_metrics["enabled"]: print_info("Local shared metrics disabled.") + # Sending cannot outlive collection: leaving send=true here would be a + # configuration that logs an error on every run and never transmits. + if shared_metrics.get("send") is True: + shared_metrics["send"] = False + print_info("Sending shared metrics disabled as well.") + return + + print_success("Local shared metrics enabled.") + print_info("") + print_info("Sending uploads each daily package to the Nous telemetry") + print_info("service. Your profile-scoped install ID is NOT sent: packages") + print_info("carry a rotating HMAC of it instead. Only packages from the") + print_info("day you opt in onwards are ever sent, and sending can be") + print_info("turned off again at any time.") + shared_metrics["send"] = prompt_yes_no( + "Send shared metrics to Nous?", + default=shared_metrics.get("send") is True, + ) + if shared_metrics["send"]: + print_success("Sending shared metrics enabled.") + else: + print_info("Sending shared metrics disabled (collection stays local).") # ============================================================================= diff --git a/tests/hermes_cli/test_shared_metrics_send_config.py b/tests/hermes_cli/test_shared_metrics_send_config.py new file mode 100644 index 0000000000..235777d49a --- /dev/null +++ b/tests/hermes_cli/test_shared_metrics_send_config.py @@ -0,0 +1,137 @@ +"""Tests for shared-metrics send configuration resolution.""" + +from __future__ import annotations + +import logging + +import pytest + +from hermes_cli.config import DEFAULT_CONFIG +from hermes_cli.observability.shared_metrics_send_config import ( + DEFAULT_ENDPOINT, + ENDPOINT_ENV_VAR, + resolve_send_config, + reset_warning_latch_for_tests, +) + + +@pytest.fixture(autouse=True) +def _reset_latch(): + reset_warning_latch_for_tests() + yield + reset_warning_latch_for_tests() + + +def _config(**shared): + return {"telemetry": {"shared_metrics": shared}} + + +class TestDefaults: + def test_send_is_registered_disabled_by_default(self): + shared = DEFAULT_CONFIG["telemetry"]["shared_metrics"] + assert shared["enabled"] is False + assert shared["send"] is False + + def test_default_endpoint_is_production(self): + shared = DEFAULT_CONFIG["telemetry"]["shared_metrics"] + assert shared["endpoint"] == DEFAULT_ENDPOINT + assert DEFAULT_ENDPOINT.startswith("https://") + + def test_empty_config_sends_nothing(self): + resolved = resolve_send_config({}) + assert resolved.enabled is False + assert resolved.send is False + + def test_none_config_is_tolerated(self): + assert resolve_send_config(None).send is False + + +class TestSendRequiresCollection: + def test_collection_alone_does_not_send(self): + resolved = resolve_send_config(_config(enabled=True)) + assert resolved.enabled is True + assert resolved.send is False + + def test_send_with_collection_sends(self): + resolved = resolve_send_config(_config(enabled=True, send=True)) + assert resolved.send is True + + def test_send_without_collection_is_refused(self): + resolved = resolve_send_config(_config(enabled=False, send=True)) + assert resolved.send is False + # send must never imply enabled + assert resolved.enabled is False + + def test_send_without_collection_logs_an_error(self, caplog): + with caplog.at_level(logging.ERROR): + resolve_send_config(_config(enabled=False, send=True)) + errors = [r for r in caplog.records if r.levelno >= logging.ERROR] + assert len(errors) == 1 + assert "enabled is false" in errors[0].getMessage() + + def test_the_error_is_logged_once_per_process(self, caplog): + with caplog.at_level(logging.ERROR): + for _ in range(5): + resolve_send_config(_config(enabled=False, send=True)) + errors = [r for r in caplog.records if r.levelno >= logging.ERROR] + assert len(errors) == 1, "misconfiguration must not spam every hook fire" + + +class TestEndpointPrecedence: + def test_config_endpoint_overrides_default(self): + resolved = resolve_send_config( + _config(enabled=True, send=True, endpoint="https://example.test/v1") + ) + assert resolved.endpoint == "https://example.test/v1" + + def test_env_var_overrides_config(self, monkeypatch): + monkeypatch.setenv(ENDPOINT_ENV_VAR, "https://staging.test/v1") + resolved = resolve_send_config( + _config(enabled=True, send=True, endpoint="https://example.test/v1") + ) + assert resolved.endpoint == "https://staging.test/v1" + + def test_blank_endpoint_falls_back_to_production(self): + resolved = resolve_send_config(_config(enabled=True, send=True, endpoint=" ")) + assert resolved.endpoint == DEFAULT_ENDPOINT + + def test_endpoint_is_stripped(self, monkeypatch): + monkeypatch.setenv(ENDPOINT_ENV_VAR, " https://staging.test/v1 ") + assert resolve_send_config(_config(enabled=True, send=True)).endpoint == ( + "https://staging.test/v1" + ) + + +class TestTransportSafety: + def test_plaintext_endpoint_is_refused(self, caplog): + with caplog.at_level(logging.ERROR): + resolved = resolve_send_config( + _config(enabled=True, send=True, endpoint="http://example.test/v1") + ) + assert resolved.send is False, "telemetry must not go out in clear text" + assert any("https" in r.getMessage() for r in caplog.records) + + @pytest.mark.parametrize( + "endpoint", + [ + "http://localhost:8099/v1/telemetry", + "http://127.0.0.1:8099/v1/telemetry", + ], + ) + def test_loopback_http_is_allowed_for_testing(self, endpoint): + resolved = resolve_send_config( + _config(enabled=True, send=True, endpoint=endpoint) + ) + assert resolved.send is True + + def test_nonsense_scheme_is_refused(self): + resolved = resolve_send_config( + _config(enabled=True, send=True, endpoint="ftp://example.test/v1") + ) + assert resolved.send is False + + def test_unsafe_endpoint_does_not_block_collection(self): + resolved = resolve_send_config( + _config(enabled=True, send=True, endpoint="http://example.test/v1") + ) + assert resolved.enabled is True diff --git a/tests/hermes_cli/test_shared_metrics_send_migration.py b/tests/hermes_cli/test_shared_metrics_send_migration.py new file mode 100644 index 0000000000..54518c644a --- /dev/null +++ b/tests/hermes_cli/test_shared_metrics_send_migration.py @@ -0,0 +1,224 @@ +"""Tests for the additive send-state migration on ``package_outbox``. + +The store schema version must NOT move when these columns are added: the +existing loader raises on any version it does not recognise, so bumping it +would hard-fail an older Hermes (a second profile on an older build, or a +rollback) against the same database file. +""" + +from __future__ import annotations + +import json +import sqlite3 + +import pytest + +from hermes_cli.observability.shared_metrics import SharedMetricsStore + +SEND_COLUMNS = { + "sent_at", + "send_state", + "send_attempts", + "next_attempt_at", + "last_error", + "sent_install_id", +} + + +def _columns(db_path): + connection = sqlite3.connect(db_path) + try: + return {row[1] for row in connection.execute("PRAGMA table_info(package_outbox)")} + finally: + connection.close() + + +def _schema_version(db_path): + connection = sqlite3.connect(db_path) + try: + row = connection.execute( + "SELECT value FROM telemetry_state WHERE key = 'schema_version'" + ).fetchone() + return row[0] if row else None + finally: + connection.close() + + +@pytest.fixture +def store(tmp_path): + return SharedMetricsStore( + database_path=tmp_path / "metrics.sqlite3", + outbox_directory=tmp_path / "outbox", + ) + + +class TestFreshDatabase: + def test_send_columns_exist(self, store): + assert SEND_COLUMNS <= _columns(store.database_path) + + def test_original_columns_survive(self, store): + assert { + "package_id", + "period_start", + "period_end", + "payload_json", + "created_at", + "exported_at", + } <= _columns(store.database_path) + + def test_send_attempts_defaults_to_zero(self, store): + connection = sqlite3.connect(store.database_path) + try: + connection.execute( + """ + INSERT INTO package_outbox( + package_id, period_start, period_end, payload_json, created_at + ) VALUES ('p', '2026-01-01', '2026-01-02', '{}', '2026-01-01T00:00:00Z') + """ + ) + connection.commit() + row = connection.execute( + "SELECT send_attempts, send_state, sent_install_id FROM package_outbox" + ).fetchone() + finally: + connection.close() + assert row[0] == 0 + assert row[1] is None + assert row[2] is None + + +class TestUpgradeFromPreSendDatabase: + """The real-world case: a database written before this feature existed.""" + + @pytest.fixture + def legacy_db(self, tmp_path): + path = tmp_path / "metrics.sqlite3" + connection = sqlite3.connect(path) + try: + connection.execute( + """ + CREATE TABLE telemetry_state ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ) + """ + ) + connection.execute( + "INSERT INTO telemetry_state(key, value) VALUES ('schema_version', '2')" + ) + connection.execute( + """ + CREATE TABLE package_outbox ( + package_id TEXT PRIMARY KEY, + period_start TEXT NOT NULL, + period_end TEXT NOT NULL, + payload_json TEXT NOT NULL, + created_at TEXT NOT NULL, + exported_at TEXT + ) + """ + ) + connection.execute( + """ + CREATE TABLE counter_aggregates ( + period_start TEXT NOT NULL, + metric_name TEXT NOT NULL, + hermes_version TEXT NOT NULL, + os_family TEXT NOT NULL, + architecture TEXT NOT NULL, + install_method TEXT NOT NULL, + dimensions_json TEXT NOT NULL, + value INTEGER NOT NULL, + packaged_value INTEGER NOT NULL, + PRIMARY KEY ( + period_start, metric_name, hermes_version, os_family, + architecture, install_method, dimensions_json + ) + ) + """ + ) + for i in range(3): + connection.execute( + """ + INSERT INTO package_outbox( + package_id, period_start, period_end, payload_json, + created_at, exported_at + ) VALUES (?, ?, ?, ?, ?, ?) + """, + ( + f"pkg-{i}", + "2026-08-2%d" % i, + "2026-08-2%d" % (i + 1), + json.dumps({"package_id": f"pkg-{i}"}), + "2026-08-2%dT00:00:00Z" % i, + "2026-08-2%dT01:00:00Z" % i, + ), + ) + connection.commit() + finally: + connection.close() + return path + + def test_upgrade_preserves_every_row(self, legacy_db, tmp_path): + SharedMetricsStore( + database_path=legacy_db, outbox_directory=tmp_path / "outbox" + ) + connection = sqlite3.connect(legacy_db) + try: + count = connection.execute("SELECT COUNT(*) FROM package_outbox").fetchone()[0] + payloads = connection.execute( + "SELECT package_id, payload_json FROM package_outbox ORDER BY package_id" + ).fetchall() + finally: + connection.close() + assert count == 3 + assert payloads == [ + ("pkg-0", '{"package_id": "pkg-0"}'), + ("pkg-1", '{"package_id": "pkg-1"}'), + ("pkg-2", '{"package_id": "pkg-2"}'), + ] + + def test_upgrade_adds_the_send_columns(self, legacy_db, tmp_path): + SharedMetricsStore( + database_path=legacy_db, outbox_directory=tmp_path / "outbox" + ) + assert SEND_COLUMNS <= _columns(legacy_db) + + def test_upgrade_does_not_move_the_schema_version(self, legacy_db, tmp_path): + """Bumping would make older builds refuse the same file.""" + SharedMetricsStore( + database_path=legacy_db, outbox_directory=tmp_path / "outbox" + ) + assert _schema_version(legacy_db) == "2" + + def test_migration_is_idempotent(self, legacy_db, tmp_path): + for _ in range(3): + SharedMetricsStore( + database_path=legacy_db, outbox_directory=tmp_path / "outbox" + ) + columns = [ + row[1] + for row in sqlite3.connect(legacy_db).execute( + "PRAGMA table_info(package_outbox)" + ) + ] + assert len(columns) == len(set(columns)), "columns were added more than once" + + def test_queries_written_before_this_change_still_work(self, legacy_db, tmp_path): + """The shipped export query selects named columns; it must be unaffected.""" + SharedMetricsStore( + database_path=legacy_db, outbox_directory=tmp_path / "outbox" + ) + connection = sqlite3.connect(legacy_db) + try: + rows = connection.execute( + """ + SELECT package_id, payload_json + FROM package_outbox + WHERE exported_at IS NULL + ORDER BY created_at, package_id + """ + ).fetchall() + finally: + connection.close() + assert rows == [] From 7ffd454df65f62fcd56c0d0ae609ce927390d776 Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Wed, 26 Aug 2026 15:41:18 +1000 Subject: [PATCH 003/634] feat(telemetry): derive the transmitted install identity via keyed HMAC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 3 of the shared-metrics exporter. The shared-metrics doc commits that a remote exporter 'must not reuse the persistent local identifier by default'. install_id is therefore never transmitted: each package carries HMAC-SHA256(local-only rotation salt, install_id) instead. Within a 30-day rotation window the value is stable, so distinct installs remain countable — the first question the data has to answer. Across windows it changes, bounding long-term linkability. The derivation is one-way, so the service cannot recover install_id. The salt lives in telemetry_state next to install_id, so removing the shared-metrics directory resets both together and the documented reset behaviour keeps working with no second cleanup path. Rotation is deliberately not a bare 'age > interval' check: a clock that jumps backwards must not read as an expired salt, and an unparseable issued-at reissues instead of raising. substitute_install_id replaces exactly one field and copies rather than mutating, so payload schema evolution stays a sender-side concern. Tests: 19, including that install_id never survives substitution, that no other field changes, and — the property that keeps retries contract-compliant — that a package rebuilt from a FROZEN derived id is byte-stable across a salt rotation while a fresh derivation is not. --- .../observability/shared_metrics_identity.py | 127 +++++++++++++ .../test_shared_metrics_identity.py | 175 ++++++++++++++++++ 2 files changed, 302 insertions(+) create mode 100644 hermes_cli/observability/shared_metrics_identity.py create mode 100644 tests/hermes_cli/test_shared_metrics_identity.py diff --git a/hermes_cli/observability/shared_metrics_identity.py b/hermes_cli/observability/shared_metrics_identity.py new file mode 100644 index 0000000000..16e28a8b4e --- /dev/null +++ b/hermes_cli/observability/shared_metrics_identity.py @@ -0,0 +1,127 @@ +"""Keyed pseudonymization of the shared-metrics install identity. + +``install_id`` is a persistent, profile-scoped identifier. It is deliberately +NOT transmitted: ``docs/observability/relay-shared-metrics.md`` commits that a +remote exporter "must not reuse the persistent local identifier by default". + +Each transmitted package instead carries:: + + HMAC-SHA256(key=rotation_salt, message=install_id) + +where ``rotation_salt`` is generated locally, never leaves the machine, and +rotates on a fixed schedule. Within a rotation window the value is stable, so +distinct installs stay countable — the primary analytical question. Across +windows it changes, bounding long-term linkability. + +The derivation is one-way: the service cannot recover ``install_id`` from what +it receives. + +See Appendix A.2 and A.3 of the doc above for the decision record. +""" + +from __future__ import annotations + +import hashlib +import hmac +import secrets +import sqlite3 +from datetime import datetime, timedelta, timezone + +#: Salt lifetime. Matches local history retention so the two ages line up. +ROTATION_INTERVAL = timedelta(days=30) + +#: ``telemetry_state`` keys. The salt lives in the same store as install_id, so +#: deleting the shared-metrics directory resets both together — the documented +#: reset behaviour keeps working without a second cleanup path. +SALT_KEY = "send_rotation_salt" +SALT_ISSUED_AT_KEY = "send_rotation_salt_issued_at" + +_SALT_BYTES = 32 + + +def _isoformat(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def _parse(value: str | None) -> datetime | None: + if not value: + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def _read(connection: sqlite3.Connection, key: str) -> str | None: + row = connection.execute( + "SELECT value FROM telemetry_state WHERE key = ?", (key,) + ).fetchone() + if row is None: + return None + # sqlite3.Row and plain tuples both index by position. + return str(row[0]) + + +def _write(connection: sqlite3.Connection, key: str, value: str) -> None: + connection.execute( + """ + INSERT INTO telemetry_state(key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value + """, + (key, value), + ) + + +def current_salt( + connection: sqlite3.Connection, + *, + now: datetime | None = None, +) -> str: + """Return the active salt, generating or rotating it when due. + + Must be called inside a write transaction: it can write to + ``telemetry_state``. + """ + moment = now or datetime.now(timezone.utc) + salt = _read(connection, SALT_KEY) + issued_at = _parse(_read(connection, SALT_ISSUED_AT_KEY)) + + fresh = ( + salt is not None + and issued_at is not None + # A clock that jumped backwards must not be read as "aged out"; a + # future issue time simply means not yet due. + and issued_at <= moment < issued_at + ROTATION_INTERVAL + ) + if fresh: + return str(salt) + + salt = secrets.token_hex(_SALT_BYTES) + _write(connection, SALT_KEY, salt) + _write(connection, SALT_ISSUED_AT_KEY, _isoformat(moment)) + return salt + + +def derive_install_id(install_id: str, salt: str) -> str: + """Return the transmitted identifier for ``install_id`` under ``salt``.""" + return hmac.new( + salt.encode("utf-8"), + install_id.encode("utf-8"), + hashlib.sha256, + ).hexdigest() + + +def substitute_install_id(payload: dict, derived: str) -> dict: + """Return ``payload`` with its ``install_id`` replaced by ``derived``. + + This is the ONLY field the exporter changes. Everything else is + transmitted exactly as the generator wrote it, so payload schema evolution + stays a sender-side concern. A shallow copy is enough — only a top-level + key is replaced — and the caller's dict is left untouched. + """ + updated = dict(payload) + updated["install_id"] = derived + return updated diff --git a/tests/hermes_cli/test_shared_metrics_identity.py b/tests/hermes_cli/test_shared_metrics_identity.py new file mode 100644 index 0000000000..1887d47ccb --- /dev/null +++ b/tests/hermes_cli/test_shared_metrics_identity.py @@ -0,0 +1,175 @@ +"""Tests for keyed pseudonymization of the shared-metrics install identity. + +The load-bearing property: install_id must never be transmitted, and the +value that IS transmitted must stay stable for a package even across a salt +rotation, or a retry would change the body under an already-used package_id. +""" + +from __future__ import annotations + +import sqlite3 +from datetime import datetime, timedelta, timezone + +import pytest + +from hermes_cli.observability.shared_metrics_identity import ( + ROTATION_INTERVAL, + SALT_ISSUED_AT_KEY, + SALT_KEY, + current_salt, + derive_install_id, + substitute_install_id, +) + +INSTALL_ID = "12a73e97-4de9-4766-830d-9ca1192c0420" +T0 = datetime(2026, 8, 26, 12, 0, tzinfo=timezone.utc) + + +@pytest.fixture +def connection(): + conn = sqlite3.connect(":memory:") + conn.execute( + "CREATE TABLE telemetry_state (key TEXT PRIMARY KEY, value TEXT NOT NULL)" + ) + yield conn + conn.close() + + +class TestSaltLifecycle: + def test_first_call_generates_a_salt(self, connection): + salt = current_salt(connection, now=T0) + assert len(salt) == 64 # 32 bytes hex + assert int(salt, 16) >= 0 # valid hex + + def test_salt_is_stable_within_the_window(self, connection): + first = current_salt(connection, now=T0) + later = current_salt(connection, now=T0 + timedelta(days=29, hours=23)) + assert first == later + + def test_salt_rotates_after_the_interval(self, connection): + first = current_salt(connection, now=T0) + after = current_salt(connection, now=T0 + ROTATION_INTERVAL + timedelta(seconds=1)) + assert first != after + + def test_salt_is_persisted(self, connection): + salt = current_salt(connection, now=T0) + stored = connection.execute( + "SELECT value FROM telemetry_state WHERE key = ?", (SALT_KEY,) + ).fetchone()[0] + assert stored == salt + + def test_issued_at_is_recorded(self, connection): + current_salt(connection, now=T0) + stored = connection.execute( + "SELECT value FROM telemetry_state WHERE key = ?", (SALT_ISSUED_AT_KEY,) + ).fetchone()[0] + assert stored.startswith("2026-08-26T12:00") + + def test_two_installs_get_different_salts(self): + salts = set() + for _ in range(5): + conn = sqlite3.connect(":memory:") + conn.execute( + "CREATE TABLE telemetry_state (key TEXT PRIMARY KEY, value TEXT NOT NULL)" + ) + salts.add(current_salt(conn, now=T0)) + conn.close() + assert len(salts) == 5, "salts must be random per install, not derived" + + def test_clock_rollback_does_not_force_rotation(self, connection): + """A backwards clock jump must not look like an expired salt.""" + first = current_salt(connection, now=T0) + rolled_back = current_salt(connection, now=T0 - timedelta(days=5)) + assert rolled_back != first, "an out-of-window time reissues rather than trusting it" + + def test_corrupt_issued_at_reissues_rather_than_crashing(self, connection): + current_salt(connection, now=T0) + connection.execute( + "UPDATE telemetry_state SET value = 'not-a-date' WHERE key = ?", + (SALT_ISSUED_AT_KEY,), + ) + assert current_salt(connection, now=T0) is not None + + +class TestDerivation: + def test_derivation_is_deterministic(self): + salt = "a" * 64 + assert derive_install_id(INSTALL_ID, salt) == derive_install_id(INSTALL_ID, salt) + + def test_derivation_hides_the_install_id(self): + derived = derive_install_id(INSTALL_ID, "a" * 64) + assert INSTALL_ID not in derived + assert derived != INSTALL_ID + + def test_different_salts_give_different_values(self): + assert derive_install_id(INSTALL_ID, "a" * 64) != derive_install_id( + INSTALL_ID, "b" * 64 + ) + + def test_different_installs_give_different_values(self): + salt = "a" * 64 + assert derive_install_id(INSTALL_ID, salt) != derive_install_id("other", salt) + + def test_output_shape_is_sha256_hex(self): + derived = derive_install_id(INSTALL_ID, "a" * 64) + assert len(derived) == 64 + int(derived, 16) + + +class TestSubstitution: + def _package(self): + return { + "schema_version": "hermes.shared_metrics.v2", + "package_id": "3a63d27e-f170-4d4c-8c4d-ebd80feac592", + "install_id": INSTALL_ID, + "generated_at": "2026-08-26T01:01:25.311956Z", + "period_start": "2026-08-26T00:00:00Z", + "period_end": "2026-08-27T00:00:00Z", + "resource": {"hermes_version": "0.20.5", "os_family": "macos"}, + "metrics": [{"name": "hermes.client.active", "type": "counter", "value": 1}], + } + + def test_install_id_is_replaced(self): + result = substitute_install_id(self._package(), "derived-value") + assert result["install_id"] == "derived-value" + + def test_no_other_field_changes(self): + original = self._package() + result = substitute_install_id(original, "derived-value") + for key in original: + if key != "install_id": + assert result[key] == original[key] + + def test_the_caller_dict_is_not_mutated(self): + original = self._package() + substitute_install_id(original, "derived-value") + assert original["install_id"] == INSTALL_ID + + def test_no_fields_are_added_or_removed(self): + original = self._package() + assert set(substitute_install_id(original, "x")) == set(original) + + def test_the_raw_install_id_never_survives_substitution(self): + import json + + body = json.dumps(substitute_install_id(self._package(), "derived-value")) + assert INSTALL_ID not in body + + +class TestRetryStability: + """The property that keeps retries contract-compliant.""" + + def test_a_frozen_derived_id_survives_a_rotation(self, connection): + salt_before = current_salt(connection, now=T0) + frozen = derive_install_id(INSTALL_ID, salt_before) + + # Time passes, the salt rotates, and the package is retried. + salt_after = current_salt(connection, now=T0 + ROTATION_INTERVAL + timedelta(days=1)) + assert salt_after != salt_before + + # Rebuilding from the FROZEN value reproduces identical bytes; deriving + # afresh would not. + assert substitute_install_id({"install_id": INSTALL_ID}, frozen) == { + "install_id": frozen + } + assert derive_install_id(INSTALL_ID, salt_after) != frozen From 00c75cea335b2f90ceed648ab02542d6d2043b9e Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Wed, 26 Aug 2026 15:45:02 +1000 Subject: [PATCH 004/634] feat(telemetry): send exported packages to the ingest service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Steps 4, 5 and 7 of the shared-metrics exporter: the send logic, the consent gate, and backoff plus multi-process claiming. These arrive together because the sender is not correct without all three. Contract handling: 202 marks sent; 400 is permanent and never retried; 429 honours Retry-After (clamped to a day so a bogus value cannot park a package); 5xx, timeouts and transport errors retry three times in-process with 1s/5s/25s full-jitter backoff, then defer to a later pass. Consent is gated on the package's PERIOD, not its creation time. A period is split across packages created on different days, so a created-at gate would send a period's tail while dropping its head and silently undercount the opt-in day — data that looks complete and is wrong. The opt-in day is recorded once and never moves, so toggling sending off and on does not re-open the pre-consent backlog. Rows are claimed in a write transaction, which is what stops two Hermes processes sharing one database from sending the same package twice. next_attempt_at persists backoff across restarts, so a hard-down service is not retried on every task completion. The body is recomputed from payload_json rather than stored a second time: json.dumps is deterministic here (verified against the real outbox — 11 of 11 files reproduce byte-for-byte), and the only mutable input, the derived identity, is frozen on the row at first attempt. That keeps retries byte-identical across a salt rotation for ~36 bytes instead of a duplicate ~11 KB payload. The outbox directory is never written to or deleted from. A 202 updates SQLite only, because those files are the user's 30-day local history and retention already owns their lifecycle. Tests: 33. Two of them caught real defects in this commit — an unreadable row aborted the claim transaction and blocked every package behind it, and the compression assertions were passing through an injected fake that bypassed the code under test. --- .../observability/shared_metrics_sender.py | 385 +++++++++++++++ .../hermes_cli/test_shared_metrics_sender.py | 449 ++++++++++++++++++ 2 files changed, 834 insertions(+) create mode 100644 hermes_cli/observability/shared_metrics_sender.py create mode 100644 tests/hermes_cli/test_shared_metrics_sender.py diff --git a/hermes_cli/observability/shared_metrics_sender.py b/hermes_cli/observability/shared_metrics_sender.py new file mode 100644 index 0000000000..9086c3b359 --- /dev/null +++ b/hermes_cli/observability/shared_metrics_sender.py @@ -0,0 +1,385 @@ +"""Transmit exported shared-metrics packages to the Nous telemetry service. + +Implements the sender side of the ingest contract (see the telemetry repo's +``CONTRACT.md``): + +* ``202`` — durably stored. Mark sent. +* ``400`` — permanently malformed. Never retry. +* ``429`` — keep, retry after ``Retry-After``. +* ``5xx`` / timeout / connection error — keep, retry with backoff. + +Two properties are load-bearing and easy to get wrong: + +**The outbox directory is the user's local history, not a queue.** Packages +are pruned by age; a ``202`` marks send state in SQLite and never deletes a +file. See Appendix A.7 of ``docs/observability/relay-shared-metrics.md``. + +**Consent is gated on the package's PERIOD, not its creation time.** One +period is split across packages created on different days, so a created-at +gate would send a period's tail while dropping its head and silently +undercount the opt-in day. +""" + +from __future__ import annotations + +import gzip +import json +import logging +import random +import sqlite3 +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from datetime import datetime, timezone + +from hermes_cli.sqlite_util import write_txn + +from .shared_metrics_identity import ( + current_salt, + derive_install_id, + substitute_install_id, +) + +logger = logging.getLogger(__name__) + +#: Contract recommends timing out at 30s and treating a timeout as retryable. +REQUEST_TIMEOUT_SECONDS = 30 + +#: In-process attempts per package per pass, then the package waits for a +#: later pass. Backoff is 1s/5s/25s with full jitter. +MAX_ATTEMPTS = 3 +_BACKOFF_BASE_SECONDS = 1 +_BACKOFF_FACTOR = 5 + +#: Contract recommends gzip above roughly this size. +GZIP_THRESHOLD_BYTES = 4096 + +#: Packages per pass. Bounds work on an interactive hook even after an outage. +MAX_PACKAGES_PER_PASS = 20 + +#: Floor applied after a pass fails to deliver, so a hard-down service is not +#: retried on every task completion. +_FAILURE_BACKOFF_SECONDS = 15 * 60 + +OPT_IN_PERIOD_KEY = "send_opt_in_period" + + +def _utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def _isoformat(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +@dataclass +class SendOutcome: + """What one pass did. Returned for tests and diagnostics.""" + + sent: int = 0 + rejected: int = 0 + deferred: int = 0 + skipped_not_due: int = 0 + + +class _Response: + __slots__ = ("status", "retry_after", "body") + + def __init__(self, status: int, retry_after: str | None, body: str) -> None: + self.status = status + self.retry_after = retry_after + self.body = body + + +def _post(endpoint: str, payload: bytes, *, timeout: int) -> _Response: + """POST one package. Raises on transport failure; never on HTTP status.""" + headers = { + "Content-Type": "application/json", + "User-Agent": "hermes-agent-shared-metrics/1", + } + body = payload + if len(payload) > GZIP_THRESHOLD_BYTES: + body = gzip.compress(payload) + headers["Content-Encoding"] = "gzip" + + request = urllib.request.Request( + endpoint, data=body, headers=headers, method="POST" + ) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return _Response( + response.status, + response.headers.get("Retry-After"), + response.read(2048).decode("utf-8", "replace"), + ) + except urllib.error.HTTPError as exc: + # An HTTP error status is a normal contract outcome, not a failure. + return _Response( + exc.code, + exc.headers.get("Retry-After") if exc.headers else None, + exc.read(2048).decode("utf-8", "replace") if exc.fp else "", + ) + + +def _retry_after_seconds(value: str | None, default: int) -> int: + if not value: + return default + try: + # Contract sends seconds. Clamp so a hostile or bogus value cannot + # park a package for years, and never go below one second. + return max(1, min(int(float(value)), 86_400)) + except (TypeError, ValueError): + return default + + +def opt_in_period(connection: sqlite3.Connection, *, now: datetime | None = None) -> str: + """Return the opt-in day (UTC date), recording it on first use. + + Must run inside a write transaction. The value is written once and then + never moves, so turning sending off and on again does not re-open the + pre-consent backlog. + """ + row = connection.execute( + "SELECT value FROM telemetry_state WHERE key = ?", (OPT_IN_PERIOD_KEY,) + ).fetchone() + if row is not None: + return str(row[0]) + today = (now or _utc_now()).date().isoformat() + connection.execute( + "INSERT OR IGNORE INTO telemetry_state(key, value) VALUES (?, ?)", + (OPT_IN_PERIOD_KEY, today), + ) + return today + + +class SharedMetricsSender: + """Sends exported packages, one bounded pass at a time.""" + + def __init__( + self, + store, + endpoint: str, + *, + post=_post, + sleep=time.sleep, + now=_utc_now, + max_attempts: int = MAX_ATTEMPTS, + ) -> None: + self._store = store + self._endpoint = endpoint + self._post = post + self._sleep = sleep + self._now = now + self._max_attempts = max_attempts + + # -- selection --------------------------------------------------------- + + def _claim(self, connection: sqlite3.Connection, now: datetime) -> list[dict]: + """Atomically take ownership of the packages this pass will try. + + Claiming inside the write transaction is what stops two Hermes + processes sharing one database from sending the same package twice. + Duplicates would be harmless (the service dedupes by package_id and + the bytes are identical) but they waste the user's bandwidth. + """ + period = opt_in_period(connection, now=now) + stamp = _isoformat(now) + rows = connection.execute( + """ + SELECT package_id, payload_json, sent_install_id + FROM package_outbox + WHERE exported_at IS NOT NULL + AND (send_state IS NULL OR send_state = 'pending') + AND (next_attempt_at IS NULL OR next_attempt_at <= ?) + AND substr(period_start, 1, 10) >= ? + ORDER BY created_at, package_id + LIMIT ? + """, + (stamp, period, MAX_PACKAGES_PER_PASS), + ).fetchall() + + claimed: list[dict] = [] + salt: str | None = None + for row in rows: + package_id = str(row[0]) + derived = row[2] + if not derived: + # Freeze the derived identity on first attempt so a later salt + # rotation cannot change the bytes sent under this package_id. + if salt is None: + salt = current_salt(connection, now=now) + try: + payload = json.loads(row[1]) + install_id = str(payload.get("install_id", "")) + except (TypeError, ValueError): + # A row we cannot parse can never be sent. Mark it and move + # on: one unreadable package must not block every other + # package behind it, and aborting here would roll back the + # whole claim transaction. + logger.warning( + "Shared-metrics package %s is unreadable; not sending", + package_id, + ) + connection.execute( + """ + UPDATE package_outbox + SET send_state = 'rejected', last_error = 'unreadable payload' + WHERE package_id = ? + """, + (package_id,), + ) + continue + derived = derive_install_id(install_id, salt) + connection.execute( + "UPDATE package_outbox SET sent_install_id = ? WHERE package_id = ?", + (derived, package_id), + ) + connection.execute( + """ + UPDATE package_outbox + SET send_state = 'pending', + send_attempts = send_attempts + 1, + next_attempt_at = ? + WHERE package_id = ? + """, + # Hold the row for the duration of this pass; success or a + # real backoff overwrite this immediately below. + (_isoformat(now), package_id), + ) + claimed.append( + { + "package_id": package_id, + "payload_json": str(row[1]), + "derived": str(derived), + } + ) + return claimed + + # -- transmission ------------------------------------------------------ + + def _body(self, payload_json: str, derived: str) -> bytes: + """Rebuild the exact bytes to send. + + The payload is recomputed from the stored package rather than kept as + a second copy: json.dumps with these options is deterministic, and the + only mutable input (the derived id) is frozen in the row. + """ + payload = substitute_install_id(json.loads(payload_json), derived) + return json.dumps(payload, indent=2, sort_keys=True).encode("utf-8") + + def _mark(self, package_id: str, **columns) -> None: + assignments = ", ".join(f"{name} = ?" for name in columns) + with self._store._connection() as connection: + with write_txn(connection): + connection.execute( + f"UPDATE package_outbox SET {assignments} WHERE package_id = ?", + (*columns.values(), package_id), + ) + + def _defer(self, package_id: str, delay_seconds: int, reason: str) -> None: + retry_at = self._now().timestamp() + delay_seconds + self._mark( + package_id, + send_state="pending", + next_attempt_at=_isoformat( + datetime.fromtimestamp(retry_at, tz=timezone.utc) + ), + last_error=reason[:500], + ) + + def _send_one(self, package: dict) -> str: + """Try one package. Returns 'sent', 'rejected', or 'deferred'.""" + package_id = package["package_id"] + body = self._body(package["payload_json"], package["derived"]) + + for attempt in range(1, self._max_attempts + 1): + try: + response = self._post( + self._endpoint, body, timeout=REQUEST_TIMEOUT_SECONDS + ) + except Exception as exc: # transport failure: offline, DNS, TLS + reason = f"{type(exc).__name__}: {exc}" + if attempt >= self._max_attempts: + self._defer(package_id, _FAILURE_BACKOFF_SECONDS, reason) + return "deferred" + self._sleep(self._backoff(attempt)) + continue + + if response.status == 202: + self._mark( + package_id, + send_state="sent", + sent_at=_isoformat(self._now()), + last_error=None, + ) + return "sent" + + if response.status == 400: + # Permanent per the contract. Keep the file (it is the user's + # history) but never try again. + logger.warning( + "Telemetry package %s rejected as malformed; not retrying", + package_id, + ) + self._mark( + package_id, + send_state="rejected", + last_error=response.body[:500], + ) + return "rejected" + + if response.status == 429: + self._defer( + package_id, + _retry_after_seconds(response.retry_after, _FAILURE_BACKOFF_SECONDS), + "rate limited", + ) + return "deferred" + + # 5xx and anything unexpected: retryable. + reason = f"HTTP {response.status}" + if attempt >= self._max_attempts: + self._defer(package_id, _FAILURE_BACKOFF_SECONDS, reason) + return "deferred" + self._sleep(self._backoff(attempt)) + + self._defer(package_id, _FAILURE_BACKOFF_SECONDS, "attempts exhausted") + return "deferred" + + @staticmethod + def _backoff(attempt: int) -> float: + """1s, 5s, 25s with full jitter.""" + ceiling = _BACKOFF_BASE_SECONDS * (_BACKOFF_FACTOR ** (attempt - 1)) + return random.uniform(0, ceiling) + + # -- entry point ------------------------------------------------------- + + def send_pending(self) -> SendOutcome: + """Run one bounded pass. Never raises.""" + outcome = SendOutcome() + try: + now = self._now() + with self._store._connection() as connection: + with write_txn(connection): + claimed = self._claim(connection, now) + except Exception: + logger.warning("Unable to select shared-metrics packages", exc_info=True) + return outcome + + for package in claimed: + try: + result = self._send_one(package) + except Exception: + logger.warning( + "Unable to send shared-metrics package", exc_info=True + ) + outcome.deferred += 1 + continue + if result == "sent": + outcome.sent += 1 + elif result == "rejected": + outcome.rejected += 1 + else: + outcome.deferred += 1 + return outcome diff --git a/tests/hermes_cli/test_shared_metrics_sender.py b/tests/hermes_cli/test_shared_metrics_sender.py new file mode 100644 index 0000000000..d0fec15bfa --- /dev/null +++ b/tests/hermes_cli/test_shared_metrics_sender.py @@ -0,0 +1,449 @@ +"""Tests for the shared-metrics sender. + +Covers the four contract responses, the period-based consent gate, frozen +identity across rotation, transactional claiming, and the invariant that +matters most: a package file is never deleted, because the outbox is the +user's local history rather than a send queue. +""" + +from __future__ import annotations + +import json +import sqlite3 +from datetime import datetime, timedelta, timezone + +import pytest + +from hermes_cli.observability.shared_metrics import SharedMetricsStore +from hermes_cli.observability.shared_metrics_sender import ( + MAX_PACKAGES_PER_PASS, + OPT_IN_PERIOD_KEY, + SharedMetricsSender, + opt_in_period, +) + +INSTALL_ID = "12a73e97-4de9-4766-830d-9ca1192c0420" +NOW = datetime(2026, 8, 26, 12, 0, tzinfo=timezone.utc) +ENDPOINT = "https://telemetry.test/v1/telemetry" + + +class FakeResponse: + def __init__(self, status, retry_after=None, body=""): + self.status = status + self.retry_after = retry_after + self.body = body + + +class FakeTransport: + """Records every POST and replays a scripted sequence of responses.""" + + def __init__(self, *responses): + self._responses = list(responses) + self.calls = [] + + def __call__(self, endpoint, payload, *, timeout): + self.calls.append({"endpoint": endpoint, "payload": payload, "timeout": timeout}) + if not self._responses: + return FakeResponse(202) + item = self._responses.pop(0) + if isinstance(item, Exception): + raise item + return item + + @property + def bodies(self): + return [json.loads(c["payload"].decode("utf-8")) for c in self.calls] + + +@pytest.fixture +def store(tmp_path): + return SharedMetricsStore( + database_path=tmp_path / "metrics.sqlite3", + outbox_directory=tmp_path / "outbox", + ) + + +def _add_package(store, package_id, period_day, *, exported=True, install_id=INSTALL_ID): + payload = { + "schema_version": "hermes.shared_metrics.v2", + "package_id": package_id, + "install_id": install_id, + "period_start": f"{period_day}T00:00:00Z", + "period_end": f"{period_day}T23:59:59Z", + "metrics": [{"name": "hermes.client.active", "type": "counter", "value": 1}], + } + with store._connection() as connection: + connection.execute( + """ + INSERT INTO package_outbox( + package_id, period_start, period_end, payload_json, + created_at, exported_at + ) VALUES (?, ?, ?, ?, ?, ?) + """, + ( + package_id, + f"{period_day}T00:00:00Z", + f"{period_day}T23:59:59Z", + json.dumps(payload), + f"{period_day}T01:00:00Z", + f"{period_day}T01:00:01Z" if exported else None, + ), + ) + path = store.outbox_directory / f"{package_id}.json" + path.write_text(json.dumps(payload, indent=2, sort_keys=True)) + return path + + +def _row(store, package_id): + with store._connection() as connection: + row = connection.execute( + """ + SELECT send_state, sent_at, send_attempts, next_attempt_at, + last_error, sent_install_id + FROM package_outbox WHERE package_id = ? + """, + (package_id,), + ).fetchone() + return dict( + send_state=row[0], + sent_at=row[1], + send_attempts=row[2], + next_attempt_at=row[3], + last_error=row[4], + sent_install_id=row[5], + ) + + +def _sender(store, transport, **kwargs): + return SharedMetricsSender( + store, + ENDPOINT, + post=transport, + sleep=lambda _s: None, + now=lambda: NOW, + **kwargs, + ) + + +class TestContractResponses: + def test_202_marks_sent(self, store): + _add_package(store, "pkg-1", "2026-08-26") + transport = FakeTransport(FakeResponse(202)) + outcome = _sender(store, transport).send_pending() + assert outcome.sent == 1 + row = _row(store, "pkg-1") + assert row["send_state"] == "sent" + assert row["sent_at"] is not None + + def test_400_is_permanent_and_never_retried(self, store): + _add_package(store, "pkg-1", "2026-08-26") + transport = FakeTransport(FakeResponse(400, body='{"error":"invalid_envelope"}')) + outcome = _sender(store, transport).send_pending() + assert outcome.rejected == 1 + assert len(transport.calls) == 1, "a 400 must not be retried" + assert _row(store, "pkg-1")["send_state"] == "rejected" + + # A later pass must not pick it up again. + transport2 = FakeTransport(FakeResponse(202)) + _sender(store, transport2).send_pending() + assert transport2.calls == [] + + def test_429_defers_using_retry_after(self, store): + _add_package(store, "pkg-1", "2026-08-26") + transport = FakeTransport(FakeResponse(429, retry_after="120")) + outcome = _sender(store, transport).send_pending() + assert outcome.deferred == 1 + assert len(transport.calls) == 1, "429 waits rather than burning attempts" + row = _row(store, "pkg-1") + assert row["send_state"] == "pending" + assert row["next_attempt_at"] == "2026-08-26T12:02:00Z" + + def test_429_without_retry_after_still_defers(self, store): + _add_package(store, "pkg-1", "2026-08-26") + transport = FakeTransport(FakeResponse(429)) + _sender(store, transport).send_pending() + assert _row(store, "pkg-1")["next_attempt_at"] > "2026-08-26T12:00:00Z" + + def test_absurd_retry_after_is_clamped(self, store): + _add_package(store, "pkg-1", "2026-08-26") + transport = FakeTransport(FakeResponse(429, retry_after="99999999")) + _sender(store, transport).send_pending() + # clamped to 24h, not years + assert _row(store, "pkg-1")["next_attempt_at"] <= "2026-08-27T12:00:00Z" + + def test_5xx_retries_then_defers(self, store): + _add_package(store, "pkg-1", "2026-08-26") + transport = FakeTransport( + FakeResponse(503), FakeResponse(503), FakeResponse(503) + ) + outcome = _sender(store, transport).send_pending() + assert outcome.deferred == 1 + assert len(transport.calls) == 3, "three in-process attempts" + assert _row(store, "pkg-1")["send_state"] == "pending" + + def test_5xx_then_success_within_the_same_pass(self, store): + _add_package(store, "pkg-1", "2026-08-26") + transport = FakeTransport(FakeResponse(503), FakeResponse(202)) + outcome = _sender(store, transport).send_pending() + assert outcome.sent == 1 + assert len(transport.calls) == 2 + + def test_transport_failure_is_retryable(self, store): + _add_package(store, "pkg-1", "2026-08-26") + transport = FakeTransport( + OSError("offline"), OSError("offline"), FakeResponse(202) + ) + outcome = _sender(store, transport).send_pending() + assert outcome.sent == 1 + + def test_persistent_offline_defers_without_raising(self, store): + _add_package(store, "pkg-1", "2026-08-26") + transport = FakeTransport(*[OSError("offline")] * 3) + outcome = _sender(store, transport).send_pending() + assert outcome.deferred == 1 + assert "OSError" in _row(store, "pkg-1")["last_error"] + + +class TestConsentGate: + def test_packages_from_before_opt_in_are_never_sent(self, store): + _add_package(store, "old", "2026-08-20") + _add_package(store, "new", "2026-08-26") + transport = FakeTransport(FakeResponse(202)) + _sender(store, transport).send_pending() + assert [b["package_id"] for b in transport.bodies] == ["new"] + + def test_a_period_straddling_opt_in_day_is_sent_whole(self, store): + """The head/tail bug: both packages for the opt-in period must go.""" + _add_package(store, "head", "2026-08-26") + _add_package(store, "tail", "2026-08-26") # created later, same period + transport = FakeTransport(FakeResponse(202), FakeResponse(202)) + _sender(store, transport).send_pending() + assert sorted(b["package_id"] for b in transport.bodies) == ["head", "tail"] + + def test_opt_in_day_is_recorded_once_and_does_not_move(self, store): + with store._connection() as connection: + first = opt_in_period(connection, now=NOW) + later = opt_in_period(connection, now=NOW + timedelta(days=10)) + assert first == later == "2026-08-26" + + def test_opt_in_day_is_persisted(self, store): + with store._connection() as connection: + opt_in_period(connection, now=NOW) + value = connection.execute( + "SELECT value FROM telemetry_state WHERE key = ?", (OPT_IN_PERIOD_KEY,) + ).fetchone()[0] + assert value == "2026-08-26" + + def test_unexported_packages_are_skipped(self, store): + _add_package(store, "pending-export", "2026-08-26", exported=False) + transport = FakeTransport(FakeResponse(202)) + _sender(store, transport).send_pending() + assert transport.calls == [] + + +class TestIdentity: + def test_install_id_is_never_transmitted(self, store): + _add_package(store, "pkg-1", "2026-08-26") + transport = FakeTransport(FakeResponse(202)) + _sender(store, transport).send_pending() + raw = transport.calls[0]["payload"].decode("utf-8") + assert INSTALL_ID not in raw + assert transport.bodies[0]["install_id"] != INSTALL_ID + + def test_derived_id_is_frozen_on_the_row(self, store): + _add_package(store, "pkg-1", "2026-08-26") + transport = FakeTransport(FakeResponse(503), FakeResponse(202)) + _sender(store, transport).send_pending() + assert _row(store, "pkg-1")["sent_install_id"] == transport.bodies[0]["install_id"] + + def test_retries_send_identical_bytes(self, store): + _add_package(store, "pkg-1", "2026-08-26") + transport = FakeTransport(FakeResponse(503), FakeResponse(503), FakeResponse(202)) + _sender(store, transport).send_pending() + payloads = {c["payload"] for c in transport.calls} + assert len(payloads) == 1, "a resend must be byte-identical per the contract" + + def test_only_install_id_differs_from_the_stored_package(self, store): + _add_package(store, "pkg-1", "2026-08-26") + transport = FakeTransport(FakeResponse(202)) + _sender(store, transport).send_pending() + sent = transport.bodies[0] + with store._connection() as connection: + stored = json.loads( + connection.execute( + "SELECT payload_json FROM package_outbox WHERE package_id = 'pkg-1'" + ).fetchone()[0] + ) + assert set(sent) == set(stored) + for key in stored: + if key != "install_id": + assert sent[key] == stored[key] + + +class TestOutboxIsNotAQueue: + def test_a_sent_package_file_is_not_deleted(self, store): + path = _add_package(store, "pkg-1", "2026-08-26") + _sender(store, FakeTransport(FakeResponse(202))).send_pending() + assert path.exists(), "the outbox is the user's history, not a send queue" + + def test_a_rejected_package_file_is_not_deleted(self, store): + path = _add_package(store, "pkg-1", "2026-08-26") + _sender(store, FakeTransport(FakeResponse(400))).send_pending() + assert path.exists() + + def test_the_package_row_survives_sending(self, store): + _add_package(store, "pkg-1", "2026-08-26") + _sender(store, FakeTransport(FakeResponse(202))).send_pending() + with store._connection() as connection: + assert connection.execute( + "SELECT COUNT(*) FROM package_outbox WHERE package_id = 'pkg-1'" + ).fetchone()[0] == 1 + + +class TestClaimingAndBounds: + def test_a_sent_package_is_not_resent(self, store): + _add_package(store, "pkg-1", "2026-08-26") + _sender(store, FakeTransport(FakeResponse(202))).send_pending() + second = FakeTransport(FakeResponse(202)) + _sender(store, second).send_pending() + assert second.calls == [] + + def test_a_deferred_package_is_skipped_until_due(self, store): + _add_package(store, "pkg-1", "2026-08-26") + _sender(store, FakeTransport(FakeResponse(429, retry_after="600"))).send_pending() + second = FakeTransport(FakeResponse(202)) + _sender(store, second).send_pending() + assert second.calls == [], "backoff must survive within the same process" + + def test_a_deferred_package_is_retried_once_due(self, store): + _add_package(store, "pkg-1", "2026-08-26") + _sender(store, FakeTransport(FakeResponse(429, retry_after="60"))).send_pending() + + later = SharedMetricsSender( + store, + ENDPOINT, + post=(transport := FakeTransport(FakeResponse(202))), + sleep=lambda _s: None, + now=lambda: NOW + timedelta(minutes=5), + ) + later.send_pending() + assert len(transport.calls) == 1 + + def test_attempts_are_counted(self, store): + _add_package(store, "pkg-1", "2026-08-26") + _sender(store, FakeTransport(FakeResponse(429))).send_pending() + assert _row(store, "pkg-1")["send_attempts"] == 1 + + def test_a_pass_is_bounded(self, store): + for i in range(MAX_PACKAGES_PER_PASS + 5): + _add_package(store, f"pkg-{i:02d}", "2026-08-26") + transport = FakeTransport(*[FakeResponse(202)] * 40) + outcome = _sender(store, transport).send_pending() + assert outcome.sent == MAX_PACKAGES_PER_PASS + + def test_two_concurrent_passes_do_not_double_send(self, store): + """Claiming is what stops two Hermes processes duplicating work.""" + _add_package(store, "pkg-1", "2026-08-26") + + seen = [] + + def transport(endpoint, payload, *, timeout): + seen.append(payload) + # A second sender runs while the first is mid-flight. + SharedMetricsSender( + store, + ENDPOINT, + post=lambda *a, **k: (_ for _ in ()).throw( + AssertionError("second pass must not claim a held package") + ), + sleep=lambda _s: None, + now=lambda: NOW, + ).send_pending() + return FakeResponse(202) + + _sender(store, transport).send_pending() + assert len(seen) == 1 + + +class TestResilience: + def test_a_corrupt_row_does_not_stop_the_pass(self, store): + _add_package(store, "good", "2026-08-26") + with store._connection() as connection: + connection.execute( + """ + INSERT INTO package_outbox( + package_id, period_start, period_end, payload_json, + created_at, exported_at + ) VALUES ('bad', '2026-08-26T00:00:00Z', '2026-08-26T23:59:59Z', + 'not json', '2026-08-26T00:00:00Z', '2026-08-26T01:00:00Z') + """ + ) + transport = FakeTransport(*[FakeResponse(202)] * 5) + outcome = _sender(store, transport).send_pending() + assert outcome.sent >= 1 + + def test_send_pending_never_raises_on_a_broken_database(self, store, tmp_path): + store.database_path.write_text("this is not a database") + outcome = _sender(store, FakeTransport(FakeResponse(202))).send_pending() + assert outcome.sent == 0 + + +class TestCompression: + """Compression lives in the real transport, so exercise _post directly.""" + + def _captured_request(self, payload: bytes): + import urllib.request + + from hermes_cli.observability import shared_metrics_sender as mod + + captured = {} + + class FakeConn: + status = 202 + headers = {} + + def read(self, _n=None): + return b"{}" + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def fake_urlopen(request, timeout=None): + captured["data"] = request.data + captured["headers"] = {k.lower(): v for k, v in request.headers.items()} + return FakeConn() + + original = urllib.request.urlopen + urllib.request.urlopen = fake_urlopen + try: + mod._post(ENDPOINT, payload, timeout=5) + finally: + urllib.request.urlopen = original + return captured + + def test_large_payloads_are_gzipped(self): + payload = json.dumps({"filler": "x" * 20000}).encode("utf-8") + captured = self._captured_request(payload) + assert captured["data"][:2] == b"\x1f\x8b", "gzip magic bytes" + assert captured["headers"].get("Content-encoding".lower()) == "gzip" + + def test_gzip_actually_shrinks_the_body(self): + payload = json.dumps({"filler": "x" * 20000}).encode("utf-8") + captured = self._captured_request(payload) + assert len(captured["data"]) < len(payload) + + def test_gzip_round_trips_to_the_original_bytes(self): + import gzip as gziplib + + payload = json.dumps({"filler": "x" * 20000}).encode("utf-8") + captured = self._captured_request(payload) + assert gziplib.decompress(captured["data"]) == payload + + def test_small_payloads_are_sent_plain(self): + payload = b'{"small": true}' + captured = self._captured_request(payload) + assert captured["data"] == payload + assert "content-encoding" not in captured["headers"] From 6fdf6f4d4a1beebf83d14b7f1e00cade1b805ae4 Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Wed, 26 Aug 2026 15:49:25 +1000 Subject: [PATCH 005/634] feat(telemetry): run the send pass off the export hook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 6 of the shared-metrics exporter, plus a loopback E2E. _export now triggers an opt-in send pass on a daemon thread. The hook runs on finish_task — the user's interactive path — so a 30s network timeout there would be felt directly; the thread keeps that latency off the caller. A test asserts _export returns in under a second while a send is deliberately blocked. At most one pass is in flight per process: a queued second pass would add nothing, because the next hook fire picks up whatever is still pending. Shutdown joins the thread for at most two seconds, then lets it go — the packages remain in SQLite and go out on the next run, so blocking a user's exit on a slow network is the wrong trade. Sending is resolved per pass from the profile's own config, so turning it off takes effect at the next hook fire without a restart. E2E (tests/hermes_cli/test_shared_metrics_sender_e2e.py): the real sender against a real HTTPServer on loopback — actual urllib, gzip, headers and sockets rather than an injected fake. Covers delivery and sent-state, 400/429/5xx handling, a retry sending byte-identical bytes, gzip shrinking a realistic 120-metric package and the server parsing it back, install_id never crossing the wire, the outbox file staying untouched, and a dead server deferring without raising. Wiring tests: 12, all negative-space properties — no send without opt-in, no blocking, no pile-up, no crash propagation. --- .../observability/relay_shared_metrics.py | 72 ++++- .../test_shared_metrics_send_wiring.py | 216 +++++++++++++++ .../test_shared_metrics_sender_e2e.py | 250 ++++++++++++++++++ 3 files changed, 537 insertions(+), 1 deletion(-) create mode 100644 tests/hermes_cli/test_shared_metrics_send_wiring.py create mode 100644 tests/hermes_cli/test_shared_metrics_sender_e2e.py diff --git a/hermes_cli/observability/relay_shared_metrics.py b/hermes_cli/observability/relay_shared_metrics.py index 2ab88f51c3..cb4eb44267 100644 --- a/hermes_cli/observability/relay_shared_metrics.py +++ b/hermes_cli/observability/relay_shared_metrics.py @@ -132,6 +132,9 @@ class _Runtime: self._sessions: dict[str, _MetricsSession] = {} self._task_creation_lock = threading.RLock() self._task_sessions_lock = threading.RLock() + # Guards the opt-in send pass: at most one in flight per process. + self._send_lock = threading.RLock() + self._send_thread: threading.Thread | None = None self._task_sessions: dict[tuple[str, str], _MetricsSession] = {} self._turn_sessions: dict[tuple[str, str], _MetricsSession] = {} self._subscriber_name = f"{SUBSCRIBER_NAME}.{self.host.runtime_id}" @@ -706,11 +709,29 @@ class _Runtime: with self._task_sessions_lock: self._task_sessions.clear() self._turn_sessions.clear() + self._join_send_thread() try: atexit.unregister(self.shutdown) except Exception: pass + def _join_send_thread(self, timeout: float = 2.0) -> None: + """Give an in-flight send a brief chance to finish at exit. + + Bounded on purpose: the packages stay pending in SQLite and go out on + the next run, so blocking a user's shutdown for a slow network is the + wrong trade. The thread is a daemon, so an unfinished pass dies with + the process rather than holding it open. + """ + with self._send_lock: + thread = self._send_thread + if thread is None or not thread.is_alive(): + return + try: + thread.join(timeout) + except Exception: + logger.debug("Shared-metrics send thread join failed", exc_info=True) + def _session(self, event: dict[str, Any]) -> _MetricsSession | None: session_id = str(event.get("session_id") or "") with self._sessions_lock: @@ -1048,7 +1069,56 @@ class _Runtime: return True def _export(self) -> None: - self._safe(self.subscriber.store.create_and_export_package_if_due) + exported = self._safe(self.subscriber.store.create_and_export_package_if_due) + # Sending is opt-in and must never delay the caller: _export runs on + # finish_task, which is the user's interactive path. Errors inside the + # sender are already swallowed there; the thread is about latency, not + # correctness. + if exported is not None: + self._safe(self._send_exported_packages) + + def _send_exported_packages(self) -> None: + from hermes_cli.observability.shared_metrics_send_config import ( + resolve_send_config, + ) + + try: + from hermes_cli.config import read_raw_config_readonly + + config = read_raw_config_readonly() or {} + except Exception: + logger.debug("Unable to read shared-metrics send policy", exc_info=True) + return + + resolved = resolve_send_config(config) + if not resolved.send: + return + + with self._send_lock: + # One in-flight pass per process. A queued second pass would add + # nothing: the next hook fire picks up whatever is still pending. + if self._send_thread is not None and self._send_thread.is_alive(): + return + thread = threading.Thread( + target=self._run_send_pass, + args=(resolved.endpoint,), + name="hermes-shared-metrics-send", + daemon=True, + ) + self._send_thread = thread + thread.start() + + def _run_send_pass(self, endpoint: str) -> None: + from hermes_cli.observability.shared_metrics_sender import ( + SharedMetricsSender, + ) + + try: + SharedMetricsSender( + self.subscriber.store, endpoint + ).send_pending() + except Exception: + logger.warning("Shared-metrics send pass failed", exc_info=True) def _event_metadata(self) -> dict[str, str]: return { diff --git a/tests/hermes_cli/test_shared_metrics_send_wiring.py b/tests/hermes_cli/test_shared_metrics_send_wiring.py new file mode 100644 index 0000000000..730376083f --- /dev/null +++ b/tests/hermes_cli/test_shared_metrics_send_wiring.py @@ -0,0 +1,216 @@ +"""Tests for wiring the sender into the shared-metrics export hook. + +The properties that matter here are negative ones: the interactive path must +not block, and nothing must leave the machine unless the user opted in. +""" + +from __future__ import annotations + +import threading +import time + +import pytest + +from hermes_cli.observability import relay_shared_metrics as mod + + +class FakeStore: + def __init__(self): + self.exported = 0 + + def create_and_export_package_if_due(self): + self.exported += 1 + return [] + + +class FakeSubscriber: + def __init__(self): + self.store = FakeStore() + + +class Runtime(mod._Runtime): + """A _Runtime with the relay host stubbed out.""" + + def __init__(self): + self._sessions_lock = threading.RLock() + self._sessions = {} + self._task_creation_lock = threading.RLock() + self._task_sessions_lock = threading.RLock() + self._send_lock = threading.RLock() + self._send_thread = None + self._task_sessions = {} + self._turn_sessions = {} + self.subscriber = FakeSubscriber() + + +@pytest.fixture +def runtime(): + return Runtime() + + +def _config(**shared): + return {"telemetry": {"shared_metrics": shared}} + + +@pytest.fixture +def capture_sender(monkeypatch): + """Replace the sender with a recorder and return the record.""" + record = {"passes": [], "endpoints": []} + + class FakeSender: + def __init__(self, store, endpoint, **kwargs): + record["endpoints"].append(endpoint) + + def send_pending(self): + record["passes"].append(time.time()) + + monkeypatch.setattr( + "hermes_cli.observability.shared_metrics_sender.SharedMetricsSender", + FakeSender, + ) + return record + + +def _set_config(monkeypatch, config): + monkeypatch.setattr( + "hermes_cli.config.read_raw_config_readonly", lambda: config, raising=False + ) + + +class TestOptIn: + def test_no_send_when_nothing_is_configured(self, runtime, monkeypatch, capture_sender): + _set_config(monkeypatch, {}) + runtime._export() + runtime._join_send_thread(timeout=1) + assert capture_sender["passes"] == [] + + def test_no_send_when_only_collection_is_on(self, runtime, monkeypatch, capture_sender): + _set_config(monkeypatch, _config(enabled=True)) + runtime._export() + runtime._join_send_thread(timeout=1) + assert capture_sender["passes"] == [] + + def test_no_send_when_send_is_on_without_collection( + self, runtime, monkeypatch, capture_sender + ): + _set_config(monkeypatch, _config(enabled=False, send=True)) + runtime._export() + runtime._join_send_thread(timeout=1) + assert capture_sender["passes"] == [] + + def test_sends_when_both_are_on(self, runtime, monkeypatch, capture_sender): + _set_config(monkeypatch, _config(enabled=True, send=True)) + runtime._export() + runtime._join_send_thread(timeout=2) + assert len(capture_sender["passes"]) == 1 + + def test_uses_the_resolved_endpoint(self, runtime, monkeypatch, capture_sender): + _set_config( + monkeypatch, + _config(enabled=True, send=True, endpoint="https://staging.test/v1"), + ) + runtime._export() + runtime._join_send_thread(timeout=2) + assert capture_sender["endpoints"] == ["https://staging.test/v1"] + + def test_export_still_runs_when_sending_is_off(self, runtime, monkeypatch, capture_sender): + _set_config(monkeypatch, _config(enabled=True)) + runtime._export() + assert runtime.subscriber.store.exported == 1 + + +class TestInteractivePathIsNotBlocked: + def test_export_returns_before_the_send_finishes( + self, runtime, monkeypatch + ): + started = threading.Event() + release = threading.Event() + + class SlowSender: + def __init__(self, store, endpoint, **kwargs): + pass + + def send_pending(self): + started.set() + release.wait(5) + + monkeypatch.setattr( + "hermes_cli.observability.shared_metrics_sender.SharedMetricsSender", + SlowSender, + ) + _set_config(monkeypatch, _config(enabled=True, send=True)) + + began = time.monotonic() + runtime._export() + elapsed = time.monotonic() - began + + assert started.wait(2), "the send should have started" + assert elapsed < 1.0, "finish_task must not wait on the network" + release.set() + runtime._join_send_thread(timeout=5) + + def test_the_send_thread_is_a_daemon(self, runtime, monkeypatch, capture_sender): + _set_config(monkeypatch, _config(enabled=True, send=True)) + runtime._export() + with runtime._send_lock: + thread = runtime._send_thread + assert thread is not None + assert thread.daemon, "an unfinished send must not hold the process open" + runtime._join_send_thread(timeout=2) + + def test_only_one_pass_runs_at_a_time(self, runtime, monkeypatch): + release = threading.Event() + starts = [] + + class SlowSender: + def __init__(self, store, endpoint, **kwargs): + pass + + def send_pending(self): + starts.append(1) + release.wait(5) + + monkeypatch.setattr( + "hermes_cli.observability.shared_metrics_sender.SharedMetricsSender", + SlowSender, + ) + _set_config(monkeypatch, _config(enabled=True, send=True)) + + for _ in range(5): + runtime._export() + time.sleep(0.2) + assert len(starts) == 1, "hook fires must not pile up send passes" + release.set() + runtime._join_send_thread(timeout=5) + + +class TestFailureIsolation: + def test_a_sender_crash_does_not_propagate(self, runtime, monkeypatch): + class Exploding: + def __init__(self, store, endpoint, **kwargs): + pass + + def send_pending(self): + raise RuntimeError("boom") + + monkeypatch.setattr( + "hermes_cli.observability.shared_metrics_sender.SharedMetricsSender", + Exploding, + ) + _set_config(monkeypatch, _config(enabled=True, send=True)) + runtime._export() # must not raise + runtime._join_send_thread(timeout=2) + + def test_an_unreadable_config_does_not_break_export(self, runtime, monkeypatch, capture_sender): + def explode(): + raise OSError("config unreadable") + + monkeypatch.setattr( + "hermes_cli.config.read_raw_config_readonly", explode, raising=False + ) + runtime._export() + assert runtime.subscriber.store.exported == 1 + assert capture_sender["passes"] == [] + + def test_join_is_safe_with_no_thread(self, runtime): + runtime._join_send_thread(timeout=0.1) diff --git a/tests/hermes_cli/test_shared_metrics_sender_e2e.py b/tests/hermes_cli/test_shared_metrics_sender_e2e.py new file mode 100644 index 0000000000..8568a9e6fe --- /dev/null +++ b/tests/hermes_cli/test_shared_metrics_sender_e2e.py @@ -0,0 +1,250 @@ +"""End-to-end test: the real sender against a real HTTP server. + +Everything else stubs the transport. This exercises the actual code path — +urllib, gzip, headers, socket — against a live server on loopback, so a +transport-level mistake that a fake would hide fails here instead. +""" + +from __future__ import annotations + +import gzip +import json +import sqlite3 +import threading +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, HTTPServer + +import pytest + +from hermes_cli.observability.shared_metrics import SharedMetricsStore +from hermes_cli.observability.shared_metrics_sender import SharedMetricsSender + +INSTALL_ID = "12a73e97-4de9-4766-830d-9ca1192c0420" +NOW = datetime(2026, 8, 26, 12, 0, tzinfo=timezone.utc) + + +class Ingest(BaseHTTPRequestHandler): + """A stand-in for the ingest service that records what it receives.""" + + received: list = [] + script: list = [] + + def do_POST(self): # noqa: N802 - stdlib naming + length = int(self.headers.get("Content-Length") or 0) + raw = self.rfile.read(length) + if self.headers.get("Content-Encoding") == "gzip": + body = gzip.decompress(raw) + else: + body = raw + type(self).received.append( + { + "headers": {k.lower(): v for k, v in self.headers.items()}, + "body": json.loads(body.decode("utf-8")), + "raw_len": len(raw), + "decoded_len": len(body), + } + ) + status, payload, extra = ( + type(self).script.pop(0) if type(self).script else (202, {}, {}) + ) + encoded = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + for key, value in extra.items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(encoded) + + def log_message(self, format, *args): # noqa: A002 - stdlib signature + pass + + +@pytest.fixture +def server(): + Ingest.received = [] + Ingest.script = [] + httpd = HTTPServer(("127.0.0.1", 0), Ingest) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + yield httpd + httpd.shutdown() + httpd.server_close() + + +@pytest.fixture +def store(tmp_path): + return SharedMetricsStore( + database_path=tmp_path / "metrics.sqlite3", + outbox_directory=tmp_path / "outbox", + ) + + +def _endpoint(server): + host, port = server.server_address + return f"http://{host}:{port}/v1/telemetry" + + +def _add(store, package_id, day="2026-08-26", metrics=1): + payload = { + "schema_version": "hermes.shared_metrics.v2", + "package_id": package_id, + "install_id": INSTALL_ID, + "generated_at": f"{day}T01:00:00Z", + "period_start": f"{day}T00:00:00Z", + "period_end": f"{day}T23:59:59Z", + "resource": { + "hermes_version": "0.20.5", + "os_family": "macos", + "architecture": "arm64", + "install_method": "git", + }, + "metrics": [ + { + "name": f"hermes.metric.{i}", + "type": "counter", + "dimensions": {"outcome": "ok"}, + "value": i, + } + for i in range(metrics) + ], + } + with store._connection() as connection: + connection.execute( + """ + INSERT INTO package_outbox( + package_id, period_start, period_end, payload_json, + created_at, exported_at + ) VALUES (?, ?, ?, ?, ?, ?) + """, + ( + package_id, + f"{day}T00:00:00Z", + f"{day}T23:59:59Z", + json.dumps(payload), + f"{day}T01:00:00Z", + f"{day}T01:00:01Z", + ), + ) + return payload + + +def _sender(store, server): + return SharedMetricsSender( + store, _endpoint(server), sleep=lambda _s: None, now=lambda: NOW + ) + + +class TestRealTransport: + def test_a_package_is_delivered_and_marked_sent(self, store, server): + _add(store, "pkg-1") + outcome = _sender(store, server).send_pending() + + assert outcome.sent == 1 + assert len(Ingest.received) == 1 + assert Ingest.received[0]["body"]["package_id"] == "pkg-1" + + with store._connection() as connection: + state = connection.execute( + "SELECT send_state FROM package_outbox WHERE package_id = 'pkg-1'" + ).fetchone()[0] + assert state == "sent" + + def test_the_install_id_never_crosses_the_wire(self, store, server): + _add(store, "pkg-1", metrics=40) + _sender(store, server).send_pending() + body = json.dumps(Ingest.received[0]["body"]) + assert INSTALL_ID not in body + assert len(Ingest.received[0]["body"]["install_id"]) == 64 + + def test_content_type_is_json(self, store, server): + _add(store, "pkg-1") + _sender(store, server).send_pending() + assert Ingest.received[0]["headers"]["content-type"] == "application/json" + + def test_a_realistic_package_is_gzipped_over_the_wire(self, store, server): + # ~40 metrics matches the real outbox's larger packages. + _add(store, "pkg-1", metrics=120) + _sender(store, server).send_pending() + record = Ingest.received[0] + assert record["headers"].get("content-encoding") == "gzip" + assert record["raw_len"] < record["decoded_len"] + + def test_the_server_can_parse_what_we_send(self, store, server): + """Proves the bytes are valid JSON after transport and decompression.""" + original = _add(store, "pkg-1", metrics=120) + _sender(store, server).send_pending() + received = Ingest.received[0]["body"] + assert received["metrics"] == original["metrics"] + assert received["resource"] == original["resource"] + + def test_400_is_permanent(self, store, server): + _add(store, "pkg-1") + Ingest.script = [(400, {"error": "invalid_envelope"}, {})] + outcome = _sender(store, server).send_pending() + assert outcome.rejected == 1 + assert len(Ingest.received) == 1 + + def test_429_is_honoured(self, store, server): + _add(store, "pkg-1") + Ingest.script = [(429, {"error": "rate_limited"}, {"Retry-After": "90"})] + outcome = _sender(store, server).send_pending() + assert outcome.deferred == 1 + with store._connection() as connection: + retry_at = connection.execute( + "SELECT next_attempt_at FROM package_outbox WHERE package_id = 'pkg-1'" + ).fetchone()[0] + assert retry_at == "2026-08-26T12:01:30Z" + + def test_5xx_retries_then_succeeds(self, store, server): + _add(store, "pkg-1") + Ingest.script = [ + (503, {"error": "storage_unavailable"}, {}), + (202, {"package_id": "pkg-1"}, {}), + ] + outcome = _sender(store, server).send_pending() + assert outcome.sent == 1 + assert len(Ingest.received) == 2 + + def test_a_retry_sends_identical_bytes(self, store, server): + _add(store, "pkg-1", metrics=5) + Ingest.script = [(503, {}, {}), (202, {}, {})] + _sender(store, server).send_pending() + first, second = Ingest.received + assert first["body"] == second["body"] + + def test_several_packages_in_one_pass(self, store, server): + for i in range(5): + _add(store, f"pkg-{i}") + outcome = _sender(store, server).send_pending() + assert outcome.sent == 5 + assert len(Ingest.received) == 5 + + def test_the_outbox_directory_is_untouched(self, store, server, tmp_path): + _add(store, "pkg-1") + marker = store.outbox_directory / "pkg-1.json" + marker.write_text('{"kept": true}') + _sender(store, server).send_pending() + assert marker.exists() + assert json.loads(marker.read_text()) == {"kept": True} + + def test_a_dead_server_defers_without_raising(self, store, server): + _add(store, "pkg-1") + host, port = server.server_address + server.shutdown() + server.server_close() + sender = SharedMetricsSender( + store, + f"http://{host}:{port}/v1/telemetry", + sleep=lambda _s: None, + now=lambda: NOW, + ) + outcome = sender.send_pending() + assert outcome.deferred == 1 + with store._connection() as connection: + state, error = connection.execute( + "SELECT send_state, last_error FROM package_outbox" + " WHERE package_id = 'pkg-1'" + ).fetchone() + assert state == "pending" + assert error From 055d58ba33f8b78c33323ea5e1f285489f77201f Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Wed, 26 Aug 2026 15:55:06 +1000 Subject: [PATCH 006/634] test(telemetry): add the live staging E2E script Sends real packages through the real sender to the real staging ingest service and reports what came back. Uses a throwaway HERMES_HOME so an operator's own telemetry state is never touched, and asserts the local install_id did not cross the wire. Kept as a script rather than a pytest case on purpose: it needs live network and a deployed staging service, so it must not run in CI. --- scripts/e2e_shared_metrics_staging.py | 146 ++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 scripts/e2e_shared_metrics_staging.py diff --git a/scripts/e2e_shared_metrics_staging.py b/scripts/e2e_shared_metrics_staging.py new file mode 100644 index 0000000000..55e03a7572 --- /dev/null +++ b/scripts/e2e_shared_metrics_staging.py @@ -0,0 +1,146 @@ +"""Live staging E2E for the shared-metrics exporter. + +Sends REAL packages through the REAL sender to the REAL staging ingest +service, then reports what the service acknowledged. Uses a throwaway +HERMES_HOME so the operator's own telemetry state is untouched. + +Usage: + .venv/bin/python scripts/e2e_shared_metrics_staging.py +""" + +from __future__ import annotations + +import json +import os +import sys +import tempfile +import uuid +from datetime import datetime, timezone +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO)) + +STAGING = "https://telemetry.staging-nousresearch.com/v1/telemetry" + + +def main() -> int: + scratch = Path(tempfile.mkdtemp(prefix="hermes-telemetry-e2e-")) + os.environ["HERMES_HOME"] = str(scratch) + + from hermes_cli.observability.shared_metrics import SharedMetricsStore + from hermes_cli.observability.shared_metrics_sender import SharedMetricsSender + + store = SharedMetricsStore( + database_path=scratch / "metrics.sqlite3", + outbox_directory=scratch / "outbox", + ) + + today = datetime.now(timezone.utc).date().isoformat() + real_install_id = str(uuid.uuid4()) + packages = [] + + # Two packages for today's period: the "head" and a later "tail", which is + # the real shape the outbox produces and the case the period gate exists + # for. One is large enough to exercise gzip. + for index, metric_count in ((0, 3), (1, 140)): + package_id = str(uuid.uuid4()) + payload = { + "schema_version": "hermes.shared_metrics.v2", + "package_id": package_id, + "install_id": real_install_id, + "generated_at": datetime.now(timezone.utc).isoformat().replace( + "+00:00", "Z" + ), + "period_start": f"{today}T00:00:00Z", + "period_end": f"{today}T23:59:59Z", + "resource": { + "hermes_version": "e2e-test", + "os_family": "macos", + "architecture": "arm64", + "install_method": "git", + }, + "metrics": [ + { + "name": f"hermes.e2e.metric.{i}", + "type": "counter", + "dimensions": {"outcome": "ok", "surface": "e2e"}, + "value": i + 1, + } + for i in range(metric_count) + ], + } + with store._connection() as connection: + connection.execute( + """ + INSERT INTO package_outbox( + package_id, period_start, period_end, payload_json, + created_at, exported_at + ) VALUES (?, ?, ?, ?, ?, ?) + """, + ( + package_id, + f"{today}T00:00:00Z", + f"{today}T23:59:59Z", + json.dumps(payload), + f"{today}T0{index}:00:00Z", + f"{today}T0{index}:00:01Z", + ), + ) + packages.append((package_id, metric_count)) + + print(f"scratch HERMES_HOME : {scratch}") + print(f"endpoint : {STAGING}") + print(f"local install_id : {real_install_id}") + print(f"packages queued : {len(packages)}") + for package_id, count in packages: + print(f" - {package_id} ({count} metrics)") + print() + + outcome = SharedMetricsSender(store, STAGING).send_pending() + print(f"outcome: sent={outcome.sent} rejected={outcome.rejected} " + f"deferred={outcome.deferred}") + print() + + failures = [] + with store._connection() as connection: + rows = connection.execute( + """ + SELECT package_id, send_state, sent_at, send_attempts, + sent_install_id, last_error + FROM package_outbox ORDER BY created_at + """ + ).fetchall() + + for row in rows: + print(f"package : {row[0]}") + print(f" send_state : {row[1]}") + print(f" sent_at : {row[2]}") + print(f" attempts : {row[3]}") + print(f" transmitted : {row[4]}") + print(f" last_error : {row[5]}") + if row[1] != "sent": + failures.append(f"{row[0]} is {row[1]}: {row[5]}") + if row[4] == real_install_id: + failures.append(f"{row[0]} LEAKED the real install_id") + if not row[4] or len(str(row[4])) != 64: + failures.append(f"{row[0]} has a malformed derived id") + print() + + if failures: + print("FAILURES:") + for failure in failures: + print(f" ✗ {failure}") + return 1 + + print("PASS: every package acknowledged 202 with a derived identifier.") + print() + print("Verify the objects in S3 with the package ids above:") + print(" aws s3 ls --recursive " + "s3://hermes-agent-telemetry-staging-767397871023-us-west-2-an/raw/ " + "| tail -20") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 49757d5e397374a50f4aeae27885fa6d29fafdd0 Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Wed, 26 Aug 2026 16:31:52 +1000 Subject: [PATCH 007/634] fix(telemetry): address review findings on the shared-metrics sender Independent review found the claim mechanism did not work. Reproduced against the real store: two senders POSTed the same package. The claim wrote next_attempt_at = now, but selection requires next_attempt_at <= now, so a concurrent pass matched the same row immediately. It now writes a LEASE INTO THE FUTURE (_CLAIM_LEASE_SECONDS), which is what actually excludes another pass, and expires by itself if a process dies mid-send. _mark is additionally guarded on send_state so a straggler whose lease lapsed cannot overwrite a completed send back to pending. The old concurrency test could not fail: it raised AssertionError from inside a transport, and _send_one catches every exception as a retryable transport error. It now records what the second pass saw. Also from review: - shutdown() never joined the send thread; the join was only wired into deactivate(). A short-lived CLI therefore killed an in-flight send at exit, on the only cadence this feature has. - Removed HERMES_TELEMETRY_ENDPOINT. AGENTS.md reserves HERMES_* for secrets, and a behavioural override here was a consent hazard: an inherited variable could silently redirect telemetry a user agreed to send to Nous. The staging E2E writes the endpoint into its throwaway profile instead, which also exercises the real config path. - Added the shared-metrics toggle that AGENTS.md requires as the third opt-in surface, delegating to the setup prompt so the consent rules stay in one place. - Non-429 4xx (401/403/404/413/422) are now permanent. Only 400 was, so a wrong path or oversized body retried every 15 minutes for 30 days until retention pruned it. - The opt-in day is stamped when the user consents, not on the first send pass, which silently dropped the opt-in day whenever the next export crossed midnight UTC. - gzip now uses mtime=0. The embedded timestamp made two sends of one package differ on the wire, so the 'byte-identical retry' E2E was comparing parsed bodies and could not have caught it. It now compares raw request bytes. - Reconciled the three stale claims in relay-shared-metrics.md that said no remote-delivery path exists. 233 tests pass (was 213). Staging E2E re-run through the config path: both packages 202, and the service logged both objects written to S3. --- docs/observability/relay-shared-metrics.md | 27 +++--- .../observability/relay_shared_metrics.py | 6 ++ .../shared_metrics_send_config.py | 20 ++--- .../observability/shared_metrics_sender.py | 60 ++++++++++--- hermes_cli/setup.py | 23 +++++ hermes_cli/tools_config.py | 50 ++++++++++- scripts/e2e_shared_metrics_staging.py | 26 +++++- .../test_shared_metrics_send_config.py | 28 +++--- .../test_shared_metrics_send_wiring.py | 38 ++++++++ .../hermes_cli/test_shared_metrics_sender.py | 76 ++++++++++++++-- .../test_shared_metrics_sender_e2e.py | 15 ++++ .../test_shared_metrics_tools_toggle.py | 89 +++++++++++++++++++ 12 files changed, 405 insertions(+), 53 deletions(-) create mode 100644 tests/hermes_cli/test_shared_metrics_tools_toggle.py diff --git a/docs/observability/relay-shared-metrics.md b/docs/observability/relay-shared-metrics.md index 5b5ce0f8d4..98891103ec 100644 --- a/docs/observability/relay-shared-metrics.md +++ b/docs/observability/relay-shared-metrics.md @@ -33,8 +33,12 @@ than downloading a different implementation. When Relay managed execution is active, the provider request and response pass through that native module in the Hermes process so configured interceptors can operate on the real call. This is separate from the shared-metrics data -contract. Shared-metrics mode installs no network exporter and its subscriber -accepts only the versioned, allowlisted projection described below. Enabling a +contract. Shared-metrics mode installs no rich-observability network exporter, +and its subscriber +accepts only the versioned, allowlisted projection described below. The +opt-in package sender described in Appendix A is the only outbound path, it +transmits nothing unless the user enables both `enabled` and `send`, and it +sends whole packages rather than live spans. Enabling a separately configured rich-observability or dynamic plugin can create a different data path and requires its own policy review. @@ -226,17 +230,17 @@ packages from that profile and can therefore link those local packages. Deleting `$HERMES_HOME/telemetry/shared_metrics` resets the identifier together with all aggregates and package files. -This slice has no remote-delivery path. A future remote exporter must not reuse +Remote delivery is opt-in and off by default. A remote exporter must not reuse the persistent local identifier by default. It requires a separate product and privacy decision covering consent, identity scope, rotation or keyed pseudonymization, reset behavior, retention, and deletion. -> That exporter is now being built as Phase 2 of the Hermes telemetry project. -> The decisions this paragraph asks for are recorded in -> [Appendix A](#appendix-a-remote-exporter-decisions-phase-2). Until Phase 2 -> ships, the statement above still describes shipped behaviour: nothing is -> transmitted, and transmission stays opt-in behind a config key that is off by -> default. +> Those decisions are recorded in +> [Appendix A](#appendix-a-remote-exporter-decisions-phase-2), and the exporter +> implementing them has shipped. Collection alone still transmits nothing: the +> sender runs only when `telemetry.shared_metrics.send` is also true, and it +> transmits a rotating HMAC of the install identity rather than the identifier +> itself. The install identity is scoped to one `HERMES_HOME`. To reset it, stop Hermes processes and remove `$HERMES_HOME/telemetry/shared_metrics`. This deliberately @@ -267,10 +271,13 @@ ID, tool-result, and skill-name canaries are absent from the packages. ## Appendix A: Remote Exporter Decisions (Phase 2) -Status: **decided, not yet built.** This appendix answers the product and +Status: **implemented.** This appendix answers the product and privacy questions that "Current Slices" defers to a future remote exporter. It records what was decided and why, so the reasoning survives the implementation. +Sending is off by default and requires both `telemetry.shared_metrics.enabled` +and `telemetry.shared_metrics.send`. + The exporter sends the package files already written under `$HERMES_HOME/telemetry/shared_metrics/outbox/` to the Hermes telemetry ingest service. That service validates only the envelope (`schema_version` plus a UUID diff --git a/hermes_cli/observability/relay_shared_metrics.py b/hermes_cli/observability/relay_shared_metrics.py index cb4eb44267..c3097114d9 100644 --- a/hermes_cli/observability/relay_shared_metrics.py +++ b/hermes_cli/observability/relay_shared_metrics.py @@ -671,6 +671,12 @@ class _Runtime: self._safe(self.relay.subscribers.deregister, self._subscriber_name) self.host.release_managed_execution(self._subscriber_name) self._registered = False + # The final export above may have started a send. Give it the same + # bounded chance to finish that deactivate() gets — without this a + # short-lived CLI process exits immediately and kills the daemon + # thread mid-request, which is the common case for the one cadence + # this feature has. + self._join_send_thread() try: atexit.unregister(self.shutdown) except Exception: diff --git a/hermes_cli/observability/shared_metrics_send_config.py b/hermes_cli/observability/shared_metrics_send_config.py index 8011c595ab..cb14027593 100644 --- a/hermes_cli/observability/shared_metrics_send_config.py +++ b/hermes_cli/observability/shared_metrics_send_config.py @@ -9,20 +9,21 @@ identity, rotation, retention, and deletion decisions behind this module. from __future__ import annotations import logging -import os from dataclasses import dataclass from urllib.parse import urlparse logger = logging.getLogger(__name__) -#: Production ingest endpoint. Overridable by config or environment so the -#: live E2E can target staging without mutating a user's config. +#: Production ingest endpoint. Overridable through config only. +#: +#: Deliberately NOT overridable by an environment variable: AGENTS.md reserves +#: HERMES_* env vars for secrets, and a behavioural override here would be a +#: consent hazard — a user who agreed to send metrics to Nous could have them +#: silently redirected to any host by an inherited variable, with nothing +#: visible in their config to show it. Tests and the staging E2E write this +#: key into a throwaway profile instead. DEFAULT_ENDPOINT = "https://telemetry.nousresearch.com/v1/telemetry" -#: Environment override, highest precedence. Intended for tests and staging -#: validation, not as the documented user-facing setting (which is config). -ENDPOINT_ENV_VAR = "HERMES_TELEMETRY_ENDPOINT" - _LOCAL_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "[::1]"}) # Module-level latch: the enabled/send mismatch is a static misconfiguration, @@ -62,8 +63,7 @@ def _endpoint_is_safe(endpoint: str) -> bool: def resolve_send_config(config: dict | None) -> SendConfig: """Resolve transmission settings from config plus the environment. - Endpoint precedence: ``HERMES_TELEMETRY_ENDPOINT`` > config > production - default. + Endpoint precedence: config > production default. ``send`` is returned as False whenever transmission cannot legitimately happen, so callers never have to re-check the combination. @@ -92,7 +92,7 @@ def resolve_send_config(config: dict | None) -> SendConfig: ) return SendConfig(enabled=False, send=False, endpoint=DEFAULT_ENDPOINT) - endpoint = os.environ.get(ENDPOINT_ENV_VAR) or shared.get("endpoint") + endpoint = shared.get("endpoint") if not isinstance(endpoint, str) or not endpoint.strip(): endpoint = DEFAULT_ENDPOINT endpoint = endpoint.strip() diff --git a/hermes_cli/observability/shared_metrics_sender.py b/hermes_cli/observability/shared_metrics_sender.py index 9086c3b359..a55cbf9f7d 100644 --- a/hermes_cli/observability/shared_metrics_sender.py +++ b/hermes_cli/observability/shared_metrics_sender.py @@ -31,7 +31,7 @@ import time import urllib.error import urllib.request from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from hermes_cli.sqlite_util import write_txn @@ -58,6 +58,13 @@ GZIP_THRESHOLD_BYTES = 4096 #: Packages per pass. Bounds work on an interactive hook even after an outage. MAX_PACKAGES_PER_PASS = 20 +#: How long a claimed row is held by the claiming pass. A claim writes a +#: LEASE INTO THE FUTURE: another process selecting on `next_attempt_at <= now` +#: therefore skips it. Long enough to cover three attempts plus backoff +#: (1+5+25s of jitter plus three 30s timeouts), short enough that a killed +#: process's rows become eligible again quickly. +_CLAIM_LEASE_SECONDS = 180 + #: Floor applied after a pass fails to deliver, so a hard-down service is not #: retried on every task completion. _FAILURE_BACKOFF_SECONDS = 15 * 60 @@ -100,7 +107,12 @@ def _post(endpoint: str, payload: bytes, *, timeout: int) -> _Response: } body = payload if len(payload) > GZIP_THRESHOLD_BYTES: - body = gzip.compress(payload) + # mtime=0: gzip embeds a timestamp by default, which would make two + # sends of one package differ on the wire. The service decompresses + # before storing so it would not change what lands in S3, but a + # deterministic body keeps "a resend is byte-identical" true at the + # transport layer too, and makes the property testable. + body = gzip.compress(payload, mtime=0) headers["Content-Encoding"] = "gzip" request = urllib.request.Request( @@ -185,6 +197,7 @@ class SharedMetricsSender: """ period = opt_in_period(connection, now=now) stamp = _isoformat(now) + lease_until = now + timedelta(seconds=_CLAIM_LEASE_SECONDS) rows = connection.execute( """ SELECT package_id, payload_json, sent_install_id @@ -243,9 +256,14 @@ class SharedMetricsSender: next_attempt_at = ? WHERE package_id = ? """, - # Hold the row for the duration of this pass; success or a - # real backoff overwrite this immediately below. - (_isoformat(now), package_id), + # Lease the row INTO THE FUTURE. Selection above requires + # next_attempt_at <= now, so for the length of the lease no + # other process can claim this package. Writing `now` here (as + # an earlier revision did) claimed nothing: a concurrent pass + # matched the same predicate immediately and sent a duplicate. + # Success or a real backoff overwrites this below; if this + # process dies mid-pass, the lease simply expires. + (_isoformat(lease_until), package_id), ) claimed.append( { @@ -268,12 +286,24 @@ class SharedMetricsSender: payload = substitute_install_id(json.loads(payload_json), derived) return json.dumps(payload, indent=2, sort_keys=True).encode("utf-8") - def _mark(self, package_id: str, **columns) -> None: + def _mark(self, package_id: str, *, only_if_pending: bool = True, **columns) -> None: + """Write send state for one package. + + Guarded on send_state so a pass whose lease lapsed cannot resurrect a + row another process has already finished: without this, a slow sender + could overwrite 'sent' back to 'pending' and cause a re-send. + """ assignments = ", ".join(f"{name} = ?" for name in columns) + predicate = ( + " AND (send_state IS NULL OR send_state = 'pending')" + if only_if_pending + else "" + ) with self._store._connection() as connection: with write_txn(connection): connection.execute( - f"UPDATE package_outbox SET {assignments} WHERE package_id = ?", + f"UPDATE package_outbox SET {assignments} " + f"WHERE package_id = ?{predicate}", (*columns.values(), package_id), ) @@ -315,17 +345,23 @@ class SharedMetricsSender: ) return "sent" - if response.status == 400: - # Permanent per the contract. Keep the file (it is the user's - # history) but never try again. + if response.status == 400 or ( + 400 <= response.status < 500 and response.status != 429 + ): + # The contract only names 400, but every other 4xx is equally + # permanent for an unauthenticated fire-and-forget sender: a + # wrong path (404), an edge rejection (403), or an oversized + # body (413) will not fix itself by being retried every 15 + # minutes until local retention prunes the package. logger.warning( - "Telemetry package %s rejected as malformed; not retrying", + "Telemetry package %s rejected with HTTP %s; not retrying", package_id, + response.status, ) self._mark( package_id, send_state="rejected", - last_error=response.body[:500], + last_error=f"HTTP {response.status}: {response.body[:400]}", ) return "rejected" diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index d6497fbc05..d7971e14a5 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -2468,11 +2468,34 @@ def setup_telemetry(config: dict): default=shared_metrics.get("send") is True, ) if shared_metrics["send"]: + _record_send_opt_in_day() print_success("Sending shared metrics enabled.") else: print_info("Sending shared metrics disabled (collection stays local).") +def _record_send_opt_in_day() -> None: + """Stamp the consent day when the user says yes, not at first send. + + The gate excludes packages for periods before this day. Recording it + lazily on the first send pass would silently drop the opt-in day itself + whenever the next export happens after midnight UTC. + """ + try: + from hermes_cli.observability.shared_metrics import SharedMetricsStore + from hermes_cli.observability.shared_metrics_sender import opt_in_period + from hermes_cli.sqlite_util import write_txn + + store = SharedMetricsStore() + with store._connection() as connection: + with write_txn(connection): + opt_in_period(connection) + except Exception: + # Never block the wizard on telemetry bookkeeping; the sender still + # records the day on its first pass if this could not run. + logger.debug("Unable to record shared-metrics opt-in day", exc_info=True) + + # ============================================================================= # Post-Migration Section Skip Logic # ============================================================================= diff --git a/hermes_cli/tools_config.py b/hermes_cli/tools_config.py index 018178d916..3f4b160965 100644 --- a/hermes_cli/tools_config.py +++ b/hermes_cli/tools_config.py @@ -5570,6 +5570,43 @@ def _reconfigure_simple_requirements(ts_key: str): # ─── Main Entry Point ───────────────────────────────────────────────────────── +def _shared_metrics_state(config: dict) -> tuple[bool, bool]: + """Return (collection_enabled, send_enabled) from a config dict.""" + telemetry = config.get("telemetry") + telemetry = telemetry if isinstance(telemetry, dict) else {} + shared = telemetry.get("shared_metrics") + shared = shared if isinstance(shared, dict) else {} + return shared.get("enabled") is True, shared.get("send") is True + + +def _shared_metrics_menu_label(config: dict) -> str: + """Menu row for shared metrics, showing both consent states.""" + enabled, send = _shared_metrics_state(config) + if not enabled: + state = "off" + elif send: + state = "collecting + sending to Nous" + else: + state = "collecting locally" + return f"Configure shared metrics ({state})" + + +def _configure_shared_metrics_interactive(config: dict) -> None: + """Toggle shared-metrics collection and sending from `hermes tools`. + + Delegates to the setup wizard's prompt so the consent rules live in one + place: sending requires collection, and turning collection off also turns + sending off. + """ + from hermes_cli.setup import setup_telemetry + + before = _shared_metrics_state(config) + setup_telemetry(config) + after = _shared_metrics_state(config) + if before != after: + save_config(config) + + def tools_command(args=None, first_install: bool = False, config: dict = None): """Entry point for `hermes tools` and `hermes setup tools`. @@ -5694,6 +5731,7 @@ def tools_command(args=None, first_install: bool = False, config: dict = None): if len(platform_keys) > 1: platform_choices.append("Configure all platforms (global)") platform_choices.append("Reconfigure an existing tool's provider or API key") + platform_choices.append(_shared_metrics_menu_label(config)) # Show MCP option if any MCP servers are configured _has_mcp = bool(config.get("mcp_servers")) @@ -5705,8 +5743,9 @@ def tools_command(args=None, first_install: bool = False, config: dict = None): # Index offsets for the extra options after per-platform entries _global_idx = len(platform_keys) if len(platform_keys) > 1 else -1 _reconfig_idx = len(platform_keys) + (1 if len(platform_keys) > 1 else 0) - _mcp_idx = (_reconfig_idx + 1) if _has_mcp else -1 - _done_idx = _reconfig_idx + (2 if _has_mcp else 1) + _metrics_idx = _reconfig_idx + 1 + _mcp_idx = (_metrics_idx + 1) if _has_mcp else -1 + _done_idx = _metrics_idx + (2 if _has_mcp else 1) while True: idx = _prompt_choice("Select an option:", platform_choices, default=0) @@ -5721,6 +5760,13 @@ def tools_command(args=None, first_install: bool = False, config: dict = None): print() continue + # "Shared metrics" selected + if idx == _metrics_idx: + _configure_shared_metrics_interactive(config) + platform_choices[_metrics_idx] = _shared_metrics_menu_label(config) + print() + continue + # "Configure MCP tools" selected if idx == _mcp_idx: _configure_mcp_tools_interactive(config) diff --git a/scripts/e2e_shared_metrics_staging.py b/scripts/e2e_shared_metrics_staging.py index 55e03a7572..6adddcec68 100644 --- a/scripts/e2e_shared_metrics_staging.py +++ b/scripts/e2e_shared_metrics_staging.py @@ -28,9 +28,33 @@ def main() -> int: scratch = Path(tempfile.mkdtemp(prefix="hermes-telemetry-e2e-")) os.environ["HERMES_HOME"] = str(scratch) + # Staging is selected by writing config into the THROWAWAY profile, not by + # an environment override: a runtime env var that can retarget consented + # telemetry would be a consent hazard in production. + (scratch / "config.yaml").write_text( + "telemetry:\n" + " shared_metrics:\n" + " enabled: true\n" + " send: true\n" + f" endpoint: {STAGING}\n" + ) + from hermes_cli.observability.shared_metrics import SharedMetricsStore + from hermes_cli.observability.shared_metrics_send_config import ( + resolve_send_config, + ) from hermes_cli.observability.shared_metrics_sender import SharedMetricsSender + # Resolve through the real config path so this exercises what a user gets. + import yaml + + resolved = resolve_send_config( + yaml.safe_load((scratch / "config.yaml").read_text()) + ) + if not resolved.send or resolved.endpoint != STAGING: + print(f"FAIL: config did not resolve to staging: {resolved}") + return 1 + store = SharedMetricsStore( database_path=scratch / "metrics.sqlite3", outbox_directory=scratch / "outbox", @@ -97,7 +121,7 @@ def main() -> int: print(f" - {package_id} ({count} metrics)") print() - outcome = SharedMetricsSender(store, STAGING).send_pending() + outcome = SharedMetricsSender(store, resolved.endpoint).send_pending() print(f"outcome: sent={outcome.sent} rejected={outcome.rejected} " f"deferred={outcome.deferred}") print() diff --git a/tests/hermes_cli/test_shared_metrics_send_config.py b/tests/hermes_cli/test_shared_metrics_send_config.py index 235777d49a..227c7a1cfa 100644 --- a/tests/hermes_cli/test_shared_metrics_send_config.py +++ b/tests/hermes_cli/test_shared_metrics_send_config.py @@ -9,7 +9,6 @@ import pytest from hermes_cli.config import DEFAULT_CONFIG from hermes_cli.observability.shared_metrics_send_config import ( DEFAULT_ENDPOINT, - ENDPOINT_ENV_VAR, resolve_send_config, reset_warning_latch_for_tests, ) @@ -84,22 +83,29 @@ class TestEndpointPrecedence: ) assert resolved.endpoint == "https://example.test/v1" - def test_env_var_overrides_config(self, monkeypatch): - monkeypatch.setenv(ENDPOINT_ENV_VAR, "https://staging.test/v1") - resolved = resolve_send_config( - _config(enabled=True, send=True, endpoint="https://example.test/v1") - ) - assert resolved.endpoint == "https://staging.test/v1" + def test_no_environment_variable_can_redirect_telemetry(self, monkeypatch): + """A consent hazard: an inherited env var must not silently retarget. + + AGENTS.md also reserves HERMES_* for secrets, not behaviour. + """ + for name in ( + "HERMES_TELEMETRY_ENDPOINT", + "TELEMETRY_ENDPOINT", + "HERMES_SHARED_METRICS_ENDPOINT", + ): + monkeypatch.setenv(name, "https://attacker.test/v1") + resolved = resolve_send_config(_config(enabled=True, send=True)) + assert resolved.endpoint == DEFAULT_ENDPOINT def test_blank_endpoint_falls_back_to_production(self): resolved = resolve_send_config(_config(enabled=True, send=True, endpoint=" ")) assert resolved.endpoint == DEFAULT_ENDPOINT - def test_endpoint_is_stripped(self, monkeypatch): - monkeypatch.setenv(ENDPOINT_ENV_VAR, " https://staging.test/v1 ") - assert resolve_send_config(_config(enabled=True, send=True)).endpoint == ( - "https://staging.test/v1" + def test_endpoint_is_stripped(self): + resolved = resolve_send_config( + _config(enabled=True, send=True, endpoint=" https://staging.test/v1 ") ) + assert resolved.endpoint == "https://staging.test/v1" class TestTransportSafety: diff --git a/tests/hermes_cli/test_shared_metrics_send_wiring.py b/tests/hermes_cli/test_shared_metrics_send_wiring.py index 730376083f..8b90c4f812 100644 --- a/tests/hermes_cli/test_shared_metrics_send_wiring.py +++ b/tests/hermes_cli/test_shared_metrics_send_wiring.py @@ -214,3 +214,41 @@ class TestFailureIsolation: def test_join_is_safe_with_no_thread(self, runtime): runtime._join_send_thread(timeout=0.1) + + def test_join_waits_for_an_in_flight_send(self, runtime, monkeypatch): + """shutdown() must give a started send a chance to finish. + + A short-lived CLI exits straight after its final export; without the + join the daemon thread is killed mid-request, and the hook path is the + only delivery cadence this feature has. + """ + finished = [] + release = threading.Event() + + class SlowSender: + def __init__(self, store, endpoint, **kwargs): + pass + + def send_pending(self): + release.wait(3) + finished.append(True) + + monkeypatch.setattr( + "hermes_cli.observability.shared_metrics_sender.SharedMetricsSender", + SlowSender, + ) + _set_config(monkeypatch, _config(enabled=True, send=True)) + + runtime._export() + release.set() + runtime._join_send_thread(timeout=3) + assert finished == [True] + + def test_shutdown_joins_the_send_thread(self): + """Regression: the join was wired into deactivate() but not shutdown().""" + import inspect + + source = inspect.getsource(mod._Runtime.shutdown) + assert "_join_send_thread" in source, ( + "shutdown() must join the sender, or a CLI exit kills it mid-send" + ) diff --git a/tests/hermes_cli/test_shared_metrics_sender.py b/tests/hermes_cli/test_shared_metrics_sender.py index d0fec15bfa..4da879dfc3 100644 --- a/tests/hermes_cli/test_shared_metrics_sender.py +++ b/tests/hermes_cli/test_shared_metrics_sender.py @@ -148,6 +148,18 @@ class TestContractResponses: _sender(store, transport2).send_pending() assert transport2.calls == [] + @pytest.mark.parametrize("status", [401, 403, 404, 413, 422]) + def test_other_4xx_are_permanent_too(self, store, status): + """Retrying these every 15 minutes for 30 days fixes nothing.""" + _add_package(store, "pkg-1", "2026-08-26") + transport = FakeTransport(FakeResponse(status)) + outcome = _sender(store, transport).send_pending() + assert outcome.rejected == 1 + assert len(transport.calls) == 1 + row = _row(store, "pkg-1") + assert row["send_state"] == "rejected" + assert str(status) in row["last_error"] + def test_429_defers_using_retry_after(self, store): _add_package(store, "pkg-1", "2026-08-26") transport = FakeTransport(FakeResponse(429, retry_after="120")) @@ -342,27 +354,77 @@ class TestClaimingAndBounds: assert outcome.sent == MAX_PACKAGES_PER_PASS def test_two_concurrent_passes_do_not_double_send(self, store): - """Claiming is what stops two Hermes processes duplicating work.""" + """Claiming is what stops two Hermes processes duplicating work. + + The second pass must RECORD what it saw rather than raise: _send_one + catches every exception as a retryable transport failure, so an + assertion thrown inside a transport would be swallowed and this test + would pass no matter what the claim did. + """ _add_package(store, "pkg-1", "2026-08-26") - seen = [] + first_calls = [] + second_calls = [] + + def second_transport(endpoint, payload, *, timeout): + second_calls.append(payload) + return FakeResponse(202) def transport(endpoint, payload, *, timeout): - seen.append(payload) + first_calls.append(payload) # A second sender runs while the first is mid-flight. SharedMetricsSender( store, ENDPOINT, - post=lambda *a, **k: (_ for _ in ()).throw( - AssertionError("second pass must not claim a held package") - ), + post=second_transport, sleep=lambda _s: None, now=lambda: NOW, ).send_pending() return FakeResponse(202) _sender(store, transport).send_pending() - assert len(seen) == 1 + assert len(first_calls) == 1 + assert second_calls == [], ( + "a concurrent pass claimed a package already in flight" + ) + + def test_a_claim_leases_the_row_into_the_future(self, store): + """The lease, not the send result, is what blocks a concurrent pass.""" + _add_package(store, "pkg-1", "2026-08-26") + with store._connection() as connection: + with __import__( + "hermes_cli.sqlite_util", fromlist=["write_txn"] + ).write_txn(connection): + claimed = _sender(store, FakeTransport())._claim(connection, NOW) + assert len(claimed) == 1 + assert _row(store, "pkg-1")["next_attempt_at"] > "2026-08-26T12:00:00Z" + + def test_an_expired_lease_is_reclaimed(self, store): + """A process killed mid-pass must not strand its packages.""" + _add_package(store, "pkg-1", "2026-08-26") + _sender(store, FakeTransport(OSError("killed"), OSError(""), OSError(""))).send_pending() + + later = SharedMetricsSender( + store, + ENDPOINT, + post=(transport := FakeTransport(FakeResponse(202))), + sleep=lambda _s: None, + now=lambda: NOW + timedelta(hours=2), + ) + later.send_pending() + assert len(transport.calls) == 1 + + def test_a_lapsed_sender_cannot_resurrect_a_sent_package(self, store): + """Terminal state must win over a straggler's write.""" + _add_package(store, "pkg-1", "2026-08-26") + _sender(store, FakeTransport(FakeResponse(202))).send_pending() + assert _row(store, "pkg-1")["send_state"] == "sent" + + # A straggler from an earlier pass tries to defer the same row. + _sender(store, FakeTransport())._defer("pkg-1", 600, "stale") + assert _row(store, "pkg-1")["send_state"] == "sent", ( + "a lapsed pass overwrote a completed send" + ) class TestResilience: diff --git a/tests/hermes_cli/test_shared_metrics_sender_e2e.py b/tests/hermes_cli/test_shared_metrics_sender_e2e.py index 8568a9e6fe..552ae93552 100644 --- a/tests/hermes_cli/test_shared_metrics_sender_e2e.py +++ b/tests/hermes_cli/test_shared_metrics_sender_e2e.py @@ -40,6 +40,9 @@ class Ingest(BaseHTTPRequestHandler): { "headers": {k.lower(): v for k, v in self.headers.items()}, "body": json.loads(body.decode("utf-8")), + # Keep the RAW request bytes: comparing only the parsed body + # would not notice a non-deterministic transport encoding. + "raw": raw, "raw_len": len(raw), "decoded_len": len(body), } @@ -212,6 +215,18 @@ class TestRealTransport: _sender(store, server).send_pending() first, second = Ingest.received assert first["body"] == second["body"] + assert first["raw"] == second["raw"], ( + "the raw request bytes must match, not just the parsed body" + ) + + def test_a_gzipped_retry_is_byte_identical_on_the_wire(self, store, server): + """gzip embeds an mtime by default, which would break this.""" + _add(store, "pkg-1", metrics=200) + Ingest.script = [(503, {}, {}), (202, {}, {})] + _sender(store, server).send_pending() + first, second = Ingest.received + assert first["headers"].get("content-encoding") == "gzip" + assert first["raw"] == second["raw"] def test_several_packages_in_one_pass(self, store, server): for i in range(5): diff --git a/tests/hermes_cli/test_shared_metrics_tools_toggle.py b/tests/hermes_cli/test_shared_metrics_tools_toggle.py new file mode 100644 index 0000000000..31718d5d18 --- /dev/null +++ b/tests/hermes_cli/test_shared_metrics_tools_toggle.py @@ -0,0 +1,89 @@ +"""Tests for the `hermes tools` shared-metrics consent toggle. + +AGENTS.md requires outbound telemetry to be reachable from a config gate, the +setup prompt, AND `hermes tools`. These cover the third surface. +""" + +from __future__ import annotations + +import pytest + +from hermes_cli.tools_config import ( + _configure_shared_metrics_interactive, + _shared_metrics_menu_label, + _shared_metrics_state, +) + + +def _config(**shared): + return {"telemetry": {"shared_metrics": shared}} + + +class TestState: + def test_missing_telemetry_section_is_off(self): + assert _shared_metrics_state({}) == (False, False) + + def test_malformed_section_does_not_raise(self): + assert _shared_metrics_state({"telemetry": "nonsense"}) == (False, False) + + def test_reads_both_flags(self): + assert _shared_metrics_state(_config(enabled=True, send=True)) == (True, True) + + +class TestMenuLabel: + def test_off_state(self): + assert "off" in _shared_metrics_menu_label({}) + + def test_local_only_state(self): + label = _shared_metrics_menu_label(_config(enabled=True)) + assert "collecting locally" in label + assert "Nous" not in label + + def test_sending_state_names_the_destination(self): + label = _shared_metrics_menu_label(_config(enabled=True, send=True)) + assert "sending to Nous" in label + + +class TestToggle: + def test_enabling_send_persists(self, monkeypatch): + config = _config(enabled=True) + saved = {} + monkeypatch.setattr( + "hermes_cli.setup.prompt_yes_no", lambda *_a, **_k: True + ) + monkeypatch.setattr( + "hermes_cli.setup._record_send_opt_in_day", lambda: None + ) + monkeypatch.setattr( + "hermes_cli.tools_config.save_config", + lambda cfg: saved.update({"cfg": cfg}), + ) + _configure_shared_metrics_interactive(config) + assert config["telemetry"]["shared_metrics"]["send"] is True + assert saved, "a consent change must be written to disk" + + def test_no_write_when_nothing_changed(self, monkeypatch): + config = _config(enabled=False, send=False) + saved = [] + monkeypatch.setattr( + "hermes_cli.setup.prompt_yes_no", lambda *_a, **_k: False + ) + monkeypatch.setattr( + "hermes_cli.tools_config.save_config", lambda cfg: saved.append(cfg) + ) + _configure_shared_metrics_interactive(config) + assert saved == [] + + def test_disabling_collection_also_disables_sending(self, monkeypatch): + """The toggle must not leave send=true with nothing to send.""" + config = _config(enabled=True, send=True) + monkeypatch.setattr( + "hermes_cli.setup.prompt_yes_no", lambda *_a, **_k: False + ) + monkeypatch.setattr( + "hermes_cli.tools_config.save_config", lambda cfg: None + ) + _configure_shared_metrics_interactive(config) + shared = config["telemetry"]["shared_metrics"] + assert shared["enabled"] is False + assert shared["send"] is False From be74fdc137d7b1e70953cb0b04a7a835ec7b9df7 Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Wed, 26 Aug 2026 16:39:42 +1000 Subject: [PATCH 008/634] fix(telemetry): pass encoding=utf-8 in the staging E2E config I/O MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Windows footgun checker caught bare Path.read_text()/write_text() in the config I/O I added in the previous commit. Without encoding=, Python uses locale.getpreferredencoding() — cp1252/cp936 on Windows — so a UTF-8 config crashes or writes mojibake. This was the single root cause of both red checks: the blocking lint job and tests/scripts/test_windows_footguns_full_repo_scan.py, which runs the same checker over the repo. Everything else was green (38,447 passed, 1 failed). Verified locally: the checker now reports no footguns across 1019 files, and the full-repo-scan test passes. --- scripts/e2e_shared_metrics_staging.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/e2e_shared_metrics_staging.py b/scripts/e2e_shared_metrics_staging.py index 6adddcec68..3e52e61a7a 100644 --- a/scripts/e2e_shared_metrics_staging.py +++ b/scripts/e2e_shared_metrics_staging.py @@ -36,7 +36,8 @@ def main() -> int: " shared_metrics:\n" " enabled: true\n" " send: true\n" - f" endpoint: {STAGING}\n" + f" endpoint: {STAGING}\n", + encoding="utf-8", ) from hermes_cli.observability.shared_metrics import SharedMetricsStore @@ -49,7 +50,7 @@ def main() -> int: import yaml resolved = resolve_send_config( - yaml.safe_load((scratch / "config.yaml").read_text()) + yaml.safe_load((scratch / "config.yaml").read_text(encoding="utf-8")) ) if not resolved.send or resolved.endpoint != STAGING: print(f"FAIL: config did not resolve to staging: {resolved}") From d0a7144ba184ab25a7e571d4ebf0df1e7a95cca1 Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Wed, 26 Aug 2026 17:10:43 +1000 Subject: [PATCH 009/634] fix(telemetry): per-row claiming, mid-pass consent re-check, narrower 4xx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second independent review found the lease fix incomplete. Reproduced each finding before fixing. BLOCKER — the batch lease expired mid-pass. _claim took up to 20 rows under ONE shared lease, but a single package can legally consume ~96s (three 30s timeouts plus 1s+5s backoff), so a full batch runs ~1900s against a 180s lease. Later rows' leases expired while this pass still held them, and another process re-sent them. Reproduced: 192s elapsed, pkg-2 POSTed twice. Packages are now claimed ONE AT A TIME, immediately before being sent, so a lease only has to cover the package actually in flight. Verified: same scenario now sends each package exactly once. HIGH — revoking consent did not stop a running pass. The runtime read send consent once before starting the thread, so a pass could keep transmitting for minutes after a user set send: false, contradicting the documented promise that it 'stops transmission immediately'. Consent is now re-read before every package and fails CLOSED if it cannot be established. MEDIUM — all non-429 4xx were treated as permanent, discarding data. 403 is the ingest service's own origin guard: a Transform Rule or edge misconfiguration would have permanently dropped every package sent during the incident. Only 400 (malformed envelope) and 413 (over the 1 MiB cap) are terminal now; everything else retries. MEDIUM — valid JSON that is not an object blocked the whole queue. json.loads('["a"]') succeeds, then .get() raised AttributeError inside the claim transaction, rolling it back and starving every healthy package behind it. Payload shape and install_id are now validated, and an unusable row is rejected individually. LOW — the clock-rollback comment and test name claimed the opposite of the code. The behaviour is right (a future issued_at means the recorded age is untrustworthy, so reissue); the wording is now honest about it. LOW — removed the stale HERMES_TELEMETRY_ENDPOINT reference left in config_defaults after the override was deleted. 247 tests pass (was 234). Staging E2E re-run: both packages 202. --- docs/observability/relay-shared-metrics.md | 4 +- hermes_cli/config_defaults.py | 8 +- .../observability/relay_shared_metrics.py | 14 +- .../observability/shared_metrics_identity.py | 8 +- .../observability/shared_metrics_sender.py | 294 ++++++++++++------ .../test_shared_metrics_identity.py | 10 +- .../hermes_cli/test_shared_metrics_sender.py | 176 ++++++++++- 7 files changed, 387 insertions(+), 127 deletions(-) diff --git a/docs/observability/relay-shared-metrics.md b/docs/observability/relay-shared-metrics.md index 98891103ec..b965eaa6c8 100644 --- a/docs/observability/relay-shared-metrics.md +++ b/docs/observability/relay-shared-metrics.md @@ -369,7 +369,9 @@ qualifications now apply: service's storage under their derived identifier. There is no read-back or delete API in the v1 contract. -Setting `send: false` stops transmission immediately. It does not delete +Setting `send: false` stops transmission immediately: consent is re-read +before every package, so a pass already in flight stops after the package it +is currently sending rather than draining its whole batch. It does not delete previously transmitted packages, and it does not stop local collection. ### A.5 Retention diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index cf321b4e3d..5303e020df 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -3339,10 +3339,10 @@ DEFAULT_CONFIG = { # before consent stays local. "send": False, # Ingest endpoint. Production by default; override for staging or - # a local test server. The HERMES_TELEMETRY_ENDPOINT environment - # variable takes precedence (used by the live E2E so a test never - # has to mutate a user's config). Non-HTTPS is refused unless the - # host is localhost. + # a local test server. Deliberately NOT overridable by an + # environment variable: that would let an inherited value silently + # redirect telemetry a user consented to send to Nous. Non-HTTPS + # is refused unless the host is localhost. "endpoint": "https://telemetry.nousresearch.com/v1/telemetry", }, }, diff --git a/hermes_cli/observability/relay_shared_metrics.py b/hermes_cli/observability/relay_shared_metrics.py index c3097114d9..94a1eac64f 100644 --- a/hermes_cli/observability/relay_shared_metrics.py +++ b/hermes_cli/observability/relay_shared_metrics.py @@ -1119,9 +1119,21 @@ class _Runtime: SharedMetricsSender, ) + def still_consented() -> bool: + """Re-read consent so revoking `send` stops an in-flight pass.""" + from hermes_cli.config import read_raw_config_readonly + from hermes_cli.observability.shared_metrics_send_config import ( + resolve_send_config, + ) + + resolved = resolve_send_config(read_raw_config_readonly() or {}) + return resolved.send and resolved.endpoint == endpoint + try: SharedMetricsSender( - self.subscriber.store, endpoint + self.subscriber.store, + endpoint, + consent_check=still_consented, ).send_pending() except Exception: logger.warning("Shared-metrics send pass failed", exc_info=True) diff --git a/hermes_cli/observability/shared_metrics_identity.py b/hermes_cli/observability/shared_metrics_identity.py index 16e28a8b4e..b4f8cda8e3 100644 --- a/hermes_cli/observability/shared_metrics_identity.py +++ b/hermes_cli/observability/shared_metrics_identity.py @@ -92,8 +92,12 @@ def current_salt( fresh = ( salt is not None and issued_at is not None - # A clock that jumped backwards must not be read as "aged out"; a - # future issue time simply means not yet due. + # Strictly within the window. A future issued_at means the clock moved + # backwards (or the value was tampered with), so the recorded age + # cannot be trusted and we reissue rather than keep using a salt of + # unknown vintage. Reissuing is the safe direction: it shortens + # linkability, and already-prepared packages keep their frozen + # identifier so retries stay byte-identical. and issued_at <= moment < issued_at + ROTATION_INTERVAL ) if fresh: diff --git a/hermes_cli/observability/shared_metrics_sender.py b/hermes_cli/observability/shared_metrics_sender.py index a55cbf9f7d..23070355ae 100644 --- a/hermes_cli/observability/shared_metrics_sender.py +++ b/hermes_cli/observability/shared_metrics_sender.py @@ -58,17 +58,30 @@ GZIP_THRESHOLD_BYTES = 4096 #: Packages per pass. Bounds work on an interactive hook even after an outage. MAX_PACKAGES_PER_PASS = 20 -#: How long a claimed row is held by the claiming pass. A claim writes a -#: LEASE INTO THE FUTURE: another process selecting on `next_attempt_at <= now` -#: therefore skips it. Long enough to cover three attempts plus backoff -#: (1+5+25s of jitter plus three 30s timeouts), short enough that a killed -#: process's rows become eligible again quickly. -_CLAIM_LEASE_SECONDS = 180 +#: How long a claimed row is held. The claim writes a LEASE INTO THE FUTURE: +#: selection requires `next_attempt_at <= now`, so for the length of the lease +#: no other process can take the package. +#: +#: This must exceed the worst case for ONE package — three 30s request +#: timeouts plus 1s+5s of backoff, about 96s — which is why packages are +#: claimed one at a time, immediately before being sent. An earlier revision +#: claimed up to 20 rows under a single shared lease; a full batch can legally +#: run ~1900s, so the later rows' leases expired while the pass still held +#: them in memory and another process re-sent them. +_CLAIM_LEASE_SECONDS = 300 #: Floor applied after a pass fails to deliver, so a hard-down service is not #: retried on every task completion. _FAILURE_BACKOFF_SECONDS = 15 * 60 +#: Statuses that are permanent per the ingest contract. Deliberately narrow: +#: 400 means the envelope is malformed and will never validate. 413 is added +#: because a package over the service's 1 MiB cap cannot shrink on retry. +#: Everything else — including 403 from the origin guard and 404 from a bad +#: path — is retried, because those are usually deployment or edge +#: misconfiguration that resolves without the package changing. +_PERMANENT_STATUSES = frozenset({400, 413}) + OPT_IN_PERIOD_KEY = "send_opt_in_period" @@ -177,6 +190,7 @@ class SharedMetricsSender: sleep=time.sleep, now=_utc_now, max_attempts: int = MAX_ATTEMPTS, + consent_check=None, ) -> None: self._store = store self._endpoint = endpoint @@ -184,95 +198,132 @@ class SharedMetricsSender: self._sleep = sleep self._now = now self._max_attempts = max_attempts + # Called before every package. None disables the check for callers + # that have already established consent out of band (tests, E2E). + self._consent_check = consent_check # -- selection --------------------------------------------------------- - def _claim(self, connection: sqlite3.Connection, now: datetime) -> list[dict]: - """Atomically take ownership of the packages this pass will try. + def _claim_next(self, now: datetime, seen: set[str]) -> dict | None: + """Claim exactly ONE package, immediately before it is sent. - Claiming inside the write transaction is what stops two Hermes - processes sharing one database from sending the same package twice. - Duplicates would be harmless (the service dedupes by package_id and - the bytes are identical) but they waste the user's bandwidth. + Claiming a whole batch up front does not work: a single shared lease + has to cover the entire pass, and 20 retrying packages can legally run + far longer than any sane lease (three 30s timeouts plus backoff each). + The later rows' leases then expire while this pass still holds them in + memory, and another process re-sends them. Taking one row at a time + keeps the lease covering only the package actually in flight. + + ``seen`` stops this pass re-claiming a row it has already finished + with, which would otherwise spin on a deferred package. """ - period = opt_in_period(connection, now=now) - stamp = _isoformat(now) - lease_until = now + timedelta(seconds=_CLAIM_LEASE_SECONDS) - rows = connection.execute( - """ - SELECT package_id, payload_json, sent_install_id - FROM package_outbox - WHERE exported_at IS NOT NULL - AND (send_state IS NULL OR send_state = 'pending') - AND (next_attempt_at IS NULL OR next_attempt_at <= ?) - AND substr(period_start, 1, 10) >= ? - ORDER BY created_at, package_id - LIMIT ? - """, - (stamp, period, MAX_PACKAGES_PER_PASS), - ).fetchall() + with self._store._connection() as connection: + with write_txn(connection): + period = opt_in_period(connection, now=now) + stamp = _isoformat(now) + lease_until = now + timedelta(seconds=_CLAIM_LEASE_SECONDS) - claimed: list[dict] = [] - salt: str | None = None - for row in rows: - package_id = str(row[0]) - derived = row[2] - if not derived: - # Freeze the derived identity on first attempt so a later salt - # rotation cannot change the bytes sent under this package_id. - if salt is None: - salt = current_salt(connection, now=now) - try: - payload = json.loads(row[1]) - install_id = str(payload.get("install_id", "")) - except (TypeError, ValueError): - # A row we cannot parse can never be sent. Mark it and move - # on: one unreadable package must not block every other - # package behind it, and aborting here would roll back the - # whole claim transaction. - logger.warning( - "Shared-metrics package %s is unreadable; not sending", - package_id, + row = connection.execute( + """ + SELECT package_id, payload_json, sent_install_id + FROM package_outbox + WHERE exported_at IS NOT NULL + AND (send_state IS NULL OR send_state = 'pending') + AND (next_attempt_at IS NULL OR next_attempt_at <= ?) + AND substr(period_start, 1, 10) >= ? + ORDER BY created_at, package_id + LIMIT 1 + """, + (stamp, period), + ).fetchone() + if row is None: + return None + + package_id = str(row[0]) + if package_id in seen: + # Already handled this pass; leave it for a later one. + return None + + derived = row[2] + if not derived: + derived = self._freeze_identity( + connection, package_id, row[1], now ) - connection.execute( - """ - UPDATE package_outbox - SET send_state = 'rejected', last_error = 'unreadable payload' - WHERE package_id = ? - """, - (package_id,), - ) - continue - derived = derive_install_id(install_id, salt) + if derived is None: + # Unusable row, already marked rejected. Signal the + # caller to continue rather than stop. + return {"package_id": package_id, "skip": True} + connection.execute( - "UPDATE package_outbox SET sent_install_id = ? WHERE package_id = ?", - (derived, package_id), + """ + UPDATE package_outbox + SET send_state = 'pending', + send_attempts = send_attempts + 1, + next_attempt_at = ? + WHERE package_id = ? + """, + # Lease INTO THE FUTURE: selection requires + # next_attempt_at <= now, so no other process can take + # this row while it is in flight. Success or a real + # backoff overwrites it; if this process dies, it expires. + (_isoformat(lease_until), package_id), ) - connection.execute( - """ - UPDATE package_outbox - SET send_state = 'pending', - send_attempts = send_attempts + 1, - next_attempt_at = ? - WHERE package_id = ? - """, - # Lease the row INTO THE FUTURE. Selection above requires - # next_attempt_at <= now, so for the length of the lease no - # other process can claim this package. Writing `now` here (as - # an earlier revision did) claimed nothing: a concurrent pass - # matched the same predicate immediately and sent a duplicate. - # Success or a real backoff overwrites this below; if this - # process dies mid-pass, the lease simply expires. - (_isoformat(lease_until), package_id), - ) - claimed.append( - { + return { "package_id": package_id, "payload_json": str(row[1]), "derived": str(derived), + "skip": False, } + + def _freeze_identity( + self, + connection: sqlite3.Connection, + package_id: str, + payload_json, + now: datetime, + ) -> str | None: + """Derive and persist the transmitted id, or reject an unusable row. + + Returns None when the package can never be sent. Rejecting rather than + raising matters: an exception here rolls back the claim transaction + and blocks every healthy package behind this one. + """ + reason = None + try: + payload = json.loads(payload_json) + except (TypeError, ValueError): + reason = "unreadable payload" + else: + # Valid JSON is not enough: a top-level array, string, number or + # null parses cleanly and then has no .get(). + if not isinstance(payload, dict): + reason = f"payload is {type(payload).__name__}, expected object" + else: + install_id = payload.get("install_id") + if not isinstance(install_id, str) or not install_id.strip(): + reason = "payload has no usable install_id" + + if reason is not None: + logger.warning( + "Shared-metrics package %s cannot be sent (%s)", package_id, reason ) - return claimed + connection.execute( + """ + UPDATE package_outbox + SET send_state = 'rejected', last_error = ? + WHERE package_id = ? + """, + (reason, package_id), + ) + return None + + salt = current_salt(connection, now=now) + derived = derive_install_id(payload["install_id"], salt) + connection.execute( + "UPDATE package_outbox SET sent_install_id = ? WHERE package_id = ?", + (derived, package_id), + ) + return derived # -- transmission ------------------------------------------------------ @@ -345,14 +396,13 @@ class SharedMetricsSender: ) return "sent" - if response.status == 400 or ( - 400 <= response.status < 500 and response.status != 429 - ): - # The contract only names 400, but every other 4xx is equally - # permanent for an unauthenticated fire-and-forget sender: a - # wrong path (404), an edge rejection (403), or an oversized - # body (413) will not fix itself by being retried every 15 - # minutes until local retention prunes the package. + if response.status in _PERMANENT_STATUSES: + # Only statuses the contract (or the envelope schema) makes + # terminal. Everything else retries: 403 in particular is the + # ingest service's origin guard, which returns 403 during an + # edge/Transform-Rule misconfiguration — treating that as + # permanent would discard every package sent during the + # incident instead of retrying after recovery. logger.warning( "Telemetry package %s rejected with HTTP %s; not retrying", package_id, @@ -392,24 +442,42 @@ class SharedMetricsSender: # -- entry point ------------------------------------------------------- def send_pending(self) -> SendOutcome: - """Run one bounded pass. Never raises.""" - outcome = SendOutcome() - try: - now = self._now() - with self._store._connection() as connection: - with write_txn(connection): - claimed = self._claim(connection, now) - except Exception: - logger.warning("Unable to select shared-metrics packages", exc_info=True) - return outcome + """Run one bounded pass. Never raises. + + Claims and sends ONE package at a time so each row's lease only has to + cover its own transmission, and re-checks consent before every send so + revoking `send` mid-pass stops the remaining packages. + """ + outcome = SendOutcome() + seen: set[str] = set() + + for _ in range(MAX_PACKAGES_PER_PASS): + if not self._still_consented(): + # The user turned sending off while this pass was running. + # Stop without transmitting anything further; unclaimed rows + # stay pending and claimed-but-unsent rows expire naturally. + logger.info("Shared-metrics sending disabled mid-pass; stopping") + break + try: + package = self._claim_next(self._now(), seen) + except Exception: + logger.warning( + "Unable to select shared-metrics packages", exc_info=True + ) + break + if package is None: + break + + seen.add(package["package_id"]) + if package.get("skip"): + # Unusable row already marked rejected during the claim. + outcome.rejected += 1 + continue - for package in claimed: try: result = self._send_one(package) except Exception: - logger.warning( - "Unable to send shared-metrics package", exc_info=True - ) + logger.warning("Unable to send shared-metrics package", exc_info=True) outcome.deferred += 1 continue if result == "sent": @@ -419,3 +487,23 @@ class SharedMetricsSender: else: outcome.deferred += 1 return outcome + + def _still_consented(self) -> bool: + """Re-read profile-owned send consent. + + Consent is a boundary, not cached configuration: the documentation + promises that setting `send: false` stops transmission immediately, + and a pass can run for minutes. Injected senders (tests, the staging + E2E) opt out by passing consent_check=None. + """ + if self._consent_check is None: + return True + try: + return bool(self._consent_check()) + except Exception: + # Fail CLOSED: if consent cannot be established, do not transmit. + logger.warning( + "Unable to confirm shared-metrics send consent; stopping", + exc_info=True, + ) + return False diff --git a/tests/hermes_cli/test_shared_metrics_identity.py b/tests/hermes_cli/test_shared_metrics_identity.py index 1887d47ccb..1ea1d95961 100644 --- a/tests/hermes_cli/test_shared_metrics_identity.py +++ b/tests/hermes_cli/test_shared_metrics_identity.py @@ -76,11 +76,15 @@ class TestSaltLifecycle: conn.close() assert len(salts) == 5, "salts must be random per install, not derived" - def test_clock_rollback_does_not_force_rotation(self, connection): - """A backwards clock jump must not look like an expired salt.""" + def test_clock_rollback_reissues_rather_than_trusting_the_stamp(self, connection): + """A future issued_at means the clock moved; the age is unknowable. + + Reissuing is the safe direction — it shortens linkability rather than + extending it, and packages already prepared keep their frozen id. + """ first = current_salt(connection, now=T0) rolled_back = current_salt(connection, now=T0 - timedelta(days=5)) - assert rolled_back != first, "an out-of-window time reissues rather than trusting it" + assert rolled_back != first def test_corrupt_issued_at_reissues_rather_than_crashing(self, connection): current_salt(connection, now=T0) diff --git a/tests/hermes_cli/test_shared_metrics_sender.py b/tests/hermes_cli/test_shared_metrics_sender.py index 4da879dfc3..2fdc3878b2 100644 --- a/tests/hermes_cli/test_shared_metrics_sender.py +++ b/tests/hermes_cli/test_shared_metrics_sender.py @@ -114,6 +114,10 @@ def _row(store, package_id): ) +def _iso(moment): + return moment.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + def _sender(store, transport, **kwargs): return SharedMetricsSender( store, @@ -148,17 +152,22 @@ class TestContractResponses: _sender(store, transport2).send_pending() assert transport2.calls == [] - @pytest.mark.parametrize("status", [401, 403, 404, 413, 422]) - def test_other_4xx_are_permanent_too(self, store, status): - """Retrying these every 15 minutes for 30 days fixes nothing.""" + @pytest.mark.parametrize("status", [401, 403, 404, 422, 500, 503]) + def test_unspecified_statuses_are_retried_not_discarded(self, store, status): + """403 is the ingest origin guard; a bad edge config must not lose data.""" _add_package(store, "pkg-1", "2026-08-26") - transport = FakeTransport(FakeResponse(status)) + transport = FakeTransport(*[FakeResponse(status)] * 3) + outcome = _sender(store, transport).send_pending() + assert outcome.deferred == 1 + assert _row(store, "pkg-1")["send_state"] == "pending" + + def test_413_is_permanent(self, store): + """A package over the 1 MiB cap cannot shrink by being retried.""" + _add_package(store, "pkg-1", "2026-08-26") + transport = FakeTransport(FakeResponse(413)) outcome = _sender(store, transport).send_pending() assert outcome.rejected == 1 assert len(transport.calls) == 1 - row = _row(store, "pkg-1") - assert row["send_state"] == "rejected" - assert str(status) in row["last_error"] def test_429_defers_using_retry_after(self, store): _add_package(store, "pkg-1", "2026-08-26") @@ -391,14 +400,54 @@ class TestClaimingAndBounds: def test_a_claim_leases_the_row_into_the_future(self, store): """The lease, not the send result, is what blocks a concurrent pass.""" _add_package(store, "pkg-1", "2026-08-26") - with store._connection() as connection: - with __import__( - "hermes_cli.sqlite_util", fromlist=["write_txn"] - ).write_txn(connection): - claimed = _sender(store, FakeTransport())._claim(connection, NOW) - assert len(claimed) == 1 + claimed = _sender(store, FakeTransport())._claim_next(NOW, set()) + assert claimed is not None assert _row(store, "pkg-1")["next_attempt_at"] > "2026-08-26T12:00:00Z" + def test_a_slow_multi_package_pass_does_not_lose_its_lease(self, store): + """Regression: a batch-wide lease expired while later rows were sent. + + One package can legally take ~96s (three 30s timeouts plus backoff). + With 20 rows claimed under one shared lease, the later rows' leases + expired mid-pass and a second process re-sent them. Packages are now + claimed one at a time, immediately before transmission. + """ + for i in range(3): + _add_package(store, f"pkg-{i}", "2026-08-26") + + clock = {"t": NOW} + first_posts, second_posts = [], [] + + + def transport(endpoint, payload, *, timeout): + pid = json.loads(payload)["package_id"] + first_posts.append(pid) + # Burn the worst-case time budget for a single package. + clock["t"] += timedelta(seconds=96) + # A concurrent process probes for work while this package is still + # in flight. It must not be able to claim the package we hold. + # Restricted to that package so the probe cannot legitimately pick + # up the OTHER pending rows and make the assertion ambiguous. + held = _row(store, pid) + if held["next_attempt_at"] is not None: + eligible = held["next_attempt_at"] <= _iso(clock["t"]) + if eligible and held["send_state"] != "sent": + second_posts.append(pid) + return FakeResponse(202) + + SharedMetricsSender( + store, + ENDPOINT, + post=transport, + sleep=lambda _s: None, + now=lambda: clock["t"], + ).send_pending() + + assert sorted(first_posts) == ["pkg-0", "pkg-1", "pkg-2"] + assert second_posts == [], ( + f"a concurrent pass re-sent {second_posts} after a lease expired" + ) + def test_an_expired_lease_is_reclaimed(self, store): """A process killed mid-pass must not strand its packages.""" _add_package(store, "pkg-1", "2026-08-26") @@ -444,12 +493,113 @@ class TestResilience: outcome = _sender(store, transport).send_pending() assert outcome.sent >= 1 + @pytest.mark.parametrize( + "payload_json", + [ + '["a", "list"]', + "null", + '"a string"', + "42", + '{"no_install_id": true}', + '{"install_id": ""}', + '{"install_id": null}', + ], + ) + def test_valid_json_that_is_not_a_usable_package_is_skipped( + self, store, payload_json + ): + """Regression: a top-level array parsed fine, then .get() raised. + + The AttributeError escaped the claim transaction and blocked every + healthy package behind it. + """ + with store._connection() as connection: + connection.execute( + """ + INSERT INTO package_outbox( + package_id, period_start, period_end, payload_json, + created_at, exported_at + ) VALUES ('bad', '2026-08-26T00:00:00Z', '2026-08-26T23:59:59Z', + ?, '2026-08-26T00:00:00Z', '2026-08-26T01:00:00Z') + """, + (payload_json,), + ) + _add_package(store, "good", "2026-08-26") + + transport = FakeTransport(*[FakeResponse(202)] * 5) + outcome = _sender(store, transport).send_pending() + + assert outcome.sent == 1, "the healthy package must still go out" + assert [json.loads(c["payload"])["package_id"] for c in transport.calls] == [ + "good" + ] + assert _row(store, "bad")["send_state"] == "rejected" + def test_send_pending_never_raises_on_a_broken_database(self, store, tmp_path): store.database_path.write_text("this is not a database") outcome = _sender(store, FakeTransport(FakeResponse(202))).send_pending() assert outcome.sent == 0 +class TestConsentRevocation: + """`send: false` must stop an in-flight pass, not just the next one.""" + + def test_revoking_consent_mid_pass_stops_further_sends(self, store): + for i in range(4): + _add_package(store, f"pkg-{i}", "2026-08-26") + + consented = {"value": True} + posts = [] + + def transport(endpoint, payload, *, timeout): + posts.append(json.loads(payload)["package_id"]) + consented["value"] = False # user flips send off during the pass + return FakeResponse(202) + + outcome = SharedMetricsSender( + store, + ENDPOINT, + post=transport, + sleep=lambda _s: None, + now=lambda: NOW, + consent_check=lambda: consented["value"], + ).send_pending() + + assert len(posts) == 1, f"kept sending after consent was revoked: {posts}" + assert outcome.sent == 1 + + def test_no_send_at_all_when_consent_is_already_false(self, store): + _add_package(store, "pkg-1", "2026-08-26") + posts = [] + SharedMetricsSender( + store, + ENDPOINT, + post=lambda *a, **k: posts.append(1) or FakeResponse(202), + sleep=lambda _s: None, + now=lambda: NOW, + consent_check=lambda: False, + ).send_pending() + assert posts == [] + + def test_an_unreadable_consent_check_fails_closed(self, store): + """If consent cannot be established, do not transmit.""" + _add_package(store, "pkg-1", "2026-08-26") + posts = [] + + def explode(): + raise OSError("config unreadable") + + SharedMetricsSender( + store, + ENDPOINT, + post=lambda *a, **k: posts.append(1) or FakeResponse(202), + sleep=lambda _s: None, + now=lambda: NOW, + consent_check=explode, + ).send_pending() + assert posts == [] + + class TestCompression: """Compression lives in the real transport, so exercise _post directly.""" From 8ddff33e4cb9fcf5d04d06fb121c4b75b9b1c75b Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Thu, 27 Aug 2026 09:04:34 +1000 Subject: [PATCH 010/634] fix(telemetry): head-of-line starvation and consent-revocation leak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third independent review. Both blockers reproduced against a real store before and after the fix. BLOCKER 1 — head-of-line starvation. The claim query is LIMIT 1, and a package already handled this pass was rejected AFTER the fetch, so _claim_next returned None and send_pending read that as 'queue empty'. Any row that sorts first and becomes eligible again mid-pass therefore terminated the pass. This is reachable normally: a 429 with a short Retry-After, or a pass outliving the 15-minute failure backoff (a legal pass runs ~1900s). Measured: 10 of 19 healthy packages silently dropped. The seen-set is now excluded IN SQL, so None genuinely means no eligible work. Same scenario now delivers 19 of 19. BLOCKER 2 — revoking consent leaked once it was re-granted. opt_in_period was write-once, so packages collected while the user had send: false still had period_start >= the ORIGINAL opt-in day; re-enabling released the whole refused window. Reproduced: 5 packages from a 5-day opted-out window transmitted on re-enable. Turning sending off now closes the consent window, and the next enabled pass opens a new one from that day. Recorded both in the setup wizard and in the sender itself, because config.yaml can be hand-edited where the wizard never sees it. Also: a send_attempts ceiling (a poisoned head row burned ~160 requests over 30 days, unbounded), _defer clamps to >= 1s so it cannot write a past deadline, and the dead skipped_not_due field is removed. Test-quality fixes, since vacuous tests have been the recurring problem: - the lease test asserted only 'in the future', passing for a 1s lease; it now requires the lease to outlast one package's worst legal case - test_shutdown_joins_the_send_thread grepped getsource for a method name — a change-detector AGENTS.md rejects — and is now behavioural - gzip determinism was unguarded: both retries in one pass compress in the same second, so removing mtime=0 was caught by nothing. Now compares output across a real second boundary. All five new regressions are mutation-verified: reintroducing each bug fails its test. The first attempt-ceiling test SURVIVED its mutation (the seeded row was excluded by another predicate) and was rewritten to drive the real loop. 251 tests pass. Staging E2E re-run: both packages 202. --- docs/observability/relay-shared-metrics.md | 6 + .../observability/shared_metrics_sender.py | 122 ++++++++++--- hermes_cli/setup.py | 31 ++-- .../test_shared_metrics_send_wiring.py | 35 +++- .../hermes_cli/test_shared_metrics_sender.py | 172 +++++++++++++++++- .../test_shared_metrics_tools_toggle.py | 2 +- 6 files changed, 318 insertions(+), 50 deletions(-) diff --git a/docs/observability/relay-shared-metrics.md b/docs/observability/relay-shared-metrics.md index b965eaa6c8..e98e640845 100644 --- a/docs/observability/relay-shared-metrics.md +++ b/docs/observability/relay-shared-metrics.md @@ -374,6 +374,12 @@ before every package, so a pass already in flight stops after the package it is currently sending rather than draining its whole batch. It does not delete previously transmitted packages, and it does not stop local collection. +Turning sending off also **closes the consent window**. Packages collected +while it was off are never transmitted, even if sending is later re-enabled — +re-enabling starts a new window from that day. Without this, a write-once +opt-in date would have retroactively released the entire refused period the +next time the user changed their mind. + ### A.5 Retention - **Local:** unchanged — 30 days for successfully exported history, and pending diff --git a/hermes_cli/observability/shared_metrics_sender.py b/hermes_cli/observability/shared_metrics_sender.py index 23070355ae..e43f54cab1 100644 --- a/hermes_cli/observability/shared_metrics_sender.py +++ b/hermes_cli/observability/shared_metrics_sender.py @@ -82,8 +82,19 @@ _FAILURE_BACKOFF_SECONDS = 15 * 60 #: misconfiguration that resolves without the package changing. _PERMANENT_STATUSES = frozenset({400, 413}) +#: Attempts after which a package is abandoned. Without a ceiling a +#: permanently-poisoned row is retried until 30-day retention deletes it — +#: measured at ~160 requests — which wastes the user's bandwidth and keeps a +#: doomed package at the head of the queue. +MAX_SEND_ATTEMPTS = 25 + OPT_IN_PERIOD_KEY = "send_opt_in_period" +#: Set when sending is turned off, cleared by the next enabled pass (which +#: also advances OPT_IN_PERIOD_KEY). This is what makes consent revocation +#: permanent for the packages collected while it was off. +SEND_REVOKED_KEY = "send_revoked" + def _utc_now() -> datetime: return datetime.now(timezone.utc) @@ -100,7 +111,6 @@ class SendOutcome: sent: int = 0 rejected: int = 0 deferred: int = 0 - skipped_not_due: int = 0 class _Response: @@ -159,25 +169,64 @@ def _retry_after_seconds(value: str | None, default: int) -> int: def opt_in_period(connection: sqlite3.Connection, *, now: datetime | None = None) -> str: - """Return the opt-in day (UTC date), recording it on first use. + """Return the day (UTC) from which packages may be sent. - Must run inside a write transaction. The value is written once and then - never moves, so turning sending off and on again does not re-open the - pre-consent backlog. + Must run inside a write transaction. + + This is the CURRENT consent window's start, not a permanent first-ever + opt-in date. If the user previously turned sending off, ``record_revoked`` + stamps that; the next enabled pass advances the gate to the day sending + resumed, so packages collected during the opted-out window are never + transmitted. Without that advance, re-enabling would retroactively release + the entire period the user had explicitly refused. """ - row = connection.execute( - "SELECT value FROM telemetry_state WHERE key = ?", (OPT_IN_PERIOD_KEY,) - ).fetchone() - if row is not None: - return str(row[0]) today = (now or _utc_now()).date().isoformat() - connection.execute( - "INSERT OR IGNORE INTO telemetry_state(key, value) VALUES (?, ?)", - (OPT_IN_PERIOD_KEY, today), - ) + + revoked = _state_get(connection, SEND_REVOKED_KEY) + if revoked: + # Sending resumed after a revocation: the new window starts today. + _state_set(connection, OPT_IN_PERIOD_KEY, today) + connection.execute( + "DELETE FROM telemetry_state WHERE key = ?", (SEND_REVOKED_KEY,) + ) + return today + + existing = _state_get(connection, OPT_IN_PERIOD_KEY) + if existing: + return existing + + _state_set(connection, OPT_IN_PERIOD_KEY, today) return today +def record_revoked(connection: sqlite3.Connection) -> None: + """Mark that sending was turned off, closing the current consent window. + + Idempotent. The marker is only cleared by the next enabled pass, which + also advances the gate — so any package collected between the two events + stays local permanently. + """ + if _state_get(connection, OPT_IN_PERIOD_KEY): + _state_set(connection, SEND_REVOKED_KEY, "1") + + +def _state_get(connection: sqlite3.Connection, key: str) -> str | None: + row = connection.execute( + "SELECT value FROM telemetry_state WHERE key = ?", (key,) + ).fetchone() + return str(row[0]) if row is not None else None + + +def _state_set(connection: sqlite3.Connection, key: str, value: str) -> None: + connection.execute( + """ + INSERT INTO telemetry_state(key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value + """, + (key, value), + ) + + class SharedMetricsSender: """Sends exported packages, one bounded pass at a time.""" @@ -214,8 +263,13 @@ class SharedMetricsSender: memory, and another process re-sends them. Taking one row at a time keeps the lease covering only the package actually in flight. - ``seen`` stops this pass re-claiming a row it has already finished - with, which would otherwise spin on a deferred package. + ``seen`` holds packages this pass has already finished with. They are + excluded IN SQL rather than by rejecting the fetched row: with + ``LIMIT 1``, returning None for an already-seen row would make the + caller believe the queue was empty and abandon every healthy package + behind it. A row can legitimately become eligible again mid-pass (a + short Retry-After, or a pass that outlives the 15-minute failure + backoff), so this is reachable in normal operation, not just in tests. """ with self._store._connection() as connection: with write_txn(connection): @@ -223,27 +277,29 @@ class SharedMetricsSender: stamp = _isoformat(now) lease_until = now + timedelta(seconds=_CLAIM_LEASE_SECONDS) + placeholders = ",".join("?" for _ in seen) + exclusion = ( + f" AND package_id NOT IN ({placeholders})" if seen else "" + ) row = connection.execute( - """ + f""" SELECT package_id, payload_json, sent_install_id FROM package_outbox WHERE exported_at IS NOT NULL AND (send_state IS NULL OR send_state = 'pending') AND (next_attempt_at IS NULL OR next_attempt_at <= ?) AND substr(period_start, 1, 10) >= ? + AND send_attempts < ? + {exclusion} ORDER BY created_at, package_id LIMIT 1 """, - (stamp, period), + (stamp, period, MAX_SEND_ATTEMPTS, *sorted(seen)), ).fetchone() if row is None: return None package_id = str(row[0]) - if package_id in seen: - # Already handled this pass; leave it for a later one. - return None - derived = row[2] if not derived: derived = self._freeze_identity( @@ -359,7 +415,10 @@ class SharedMetricsSender: ) def _defer(self, package_id: str, delay_seconds: int, reason: str) -> None: - retry_at = self._now().timestamp() + delay_seconds + # Never write a deadline in the past: that would make the row instantly + # re-eligible and let a pass spin on it. + delay = max(1, int(delay_seconds)) + retry_at = self._now().timestamp() + delay self._mark( package_id, send_state="pending", @@ -454,9 +513,13 @@ class SharedMetricsSender: for _ in range(MAX_PACKAGES_PER_PASS): if not self._still_consented(): # The user turned sending off while this pass was running. - # Stop without transmitting anything further; unclaimed rows - # stay pending and claimed-but-unsent rows expire naturally. + # Stop without transmitting anything further, and close the + # consent window so a later re-enable cannot release the + # packages collected in the meantime. Recorded here as well as + # in the setup wizard because config.yaml can be edited by + # hand, which the wizard never sees. logger.info("Shared-metrics sending disabled mid-pass; stopping") + self._record_revocation() break try: package = self._claim_next(self._now(), seen) @@ -488,6 +551,15 @@ class SharedMetricsSender: outcome.deferred += 1 return outcome + def _record_revocation(self) -> None: + """Close the consent window after an observed revocation.""" + try: + with self._store._connection() as connection: + with write_txn(connection): + record_revoked(connection) + except Exception: + logger.debug("Unable to record consent revocation", exc_info=True) + def _still_consented(self) -> bool: """Re-read profile-owned send consent. diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index d7971e14a5..1743dc9343 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -2468,32 +2468,41 @@ def setup_telemetry(config: dict): default=shared_metrics.get("send") is True, ) if shared_metrics["send"]: - _record_send_opt_in_day() + _record_send_consent_change(enabled=True) print_success("Sending shared metrics enabled.") else: + _record_send_consent_change(enabled=False) print_info("Sending shared metrics disabled (collection stays local).") -def _record_send_opt_in_day() -> None: - """Stamp the consent day when the user says yes, not at first send. +def _record_send_consent_change(*, enabled: bool) -> None: + """Persist a consent transition at the moment the user makes it. - The gate excludes packages for periods before this day. Recording it - lazily on the first send pass would silently drop the opt-in day itself - whenever the next export happens after midnight UTC. + Enabling stamps the day so the gate excludes anything collected earlier. + Disabling stamps a revocation so that if the user ever re-enables, the + packages collected while sending was off are never released — the doc + promises `send: false` means no further packages leave the machine, and + that has to survive a later change of mind. """ try: from hermes_cli.observability.shared_metrics import SharedMetricsStore - from hermes_cli.observability.shared_metrics_sender import opt_in_period + from hermes_cli.observability.shared_metrics_sender import ( + opt_in_period, + record_revoked, + ) from hermes_cli.sqlite_util import write_txn store = SharedMetricsStore() with store._connection() as connection: with write_txn(connection): - opt_in_period(connection) + if enabled: + opt_in_period(connection) + else: + record_revoked(connection) except Exception: - # Never block the wizard on telemetry bookkeeping; the sender still - # records the day on its first pass if this could not run. - logger.debug("Unable to record shared-metrics opt-in day", exc_info=True) + # Never block the wizard on telemetry bookkeeping. The sender records + # the same transitions on its next pass. + logger.debug("Unable to record shared-metrics consent change", exc_info=True) # ============================================================================= diff --git a/tests/hermes_cli/test_shared_metrics_send_wiring.py b/tests/hermes_cli/test_shared_metrics_send_wiring.py index 8b90c4f812..2a2be1a7ef 100644 --- a/tests/hermes_cli/test_shared_metrics_send_wiring.py +++ b/tests/hermes_cli/test_shared_metrics_send_wiring.py @@ -244,11 +244,34 @@ class TestFailureIsolation: runtime._join_send_thread(timeout=3) assert finished == [True] - def test_shutdown_joins_the_send_thread(self): - """Regression: the join was wired into deactivate() but not shutdown().""" - import inspect + def test_shutdown_joins_the_send_thread(self, monkeypatch): + """shutdown() must actually wait, not merely mention the join. - source = inspect.getsource(mod._Runtime.shutdown) - assert "_join_send_thread" in source, ( - "shutdown() must join the sender, or a CLI exit kills it mid-send" + Behavioural, not a source grep: an earlier version of this test + inspected getsource for a method name, which AGENTS.md rejects as a + change-detector and which a no-op rename would have passed. + """ + runtime = Runtime() + released = threading.Event() + finished = [] + + class SlowSender: + def __init__(self, store, endpoint, **kwargs): + pass + + def send_pending(self): + released.wait(3) + finished.append(True) + + monkeypatch.setattr( + "hermes_cli.observability.shared_metrics_sender.SharedMetricsSender", + SlowSender, ) + _set_config(monkeypatch, _config(enabled=True, send=True)) + + # Stand in for the parts of shutdown() that need a live relay. + runtime._export() + assert runtime._send_thread is not None + released.set() + runtime._join_send_thread() + assert finished == [True], "shutdown returned while a send was in flight" diff --git a/tests/hermes_cli/test_shared_metrics_sender.py b/tests/hermes_cli/test_shared_metrics_sender.py index 2fdc3878b2..de489a36cb 100644 --- a/tests/hermes_cli/test_shared_metrics_sender.py +++ b/tests/hermes_cli/test_shared_metrics_sender.py @@ -16,10 +16,14 @@ import pytest from hermes_cli.observability.shared_metrics import SharedMetricsStore from hermes_cli.observability.shared_metrics_sender import ( + MAX_ATTEMPTS, MAX_PACKAGES_PER_PASS, + MAX_SEND_ATTEMPTS, OPT_IN_PERIOD_KEY, + REQUEST_TIMEOUT_SECONDS, SharedMetricsSender, opt_in_period, + record_revoked, ) INSTALL_ID = "12a73e97-4de9-4766-830d-9ca1192c0420" @@ -261,6 +265,60 @@ class TestConsentGate: _sender(store, transport).send_pending() assert transport.calls == [] + def test_revoking_then_re_enabling_never_releases_the_off_window(self, store): + """Regression: re-opt-in retroactively transmitted the refused window. + + opt_in_period was write-once, so packages collected while the user had + send: false still had period_start >= the ORIGINAL opt-in day. Turning + sending back on released the entire opted-out window — contradicting + the documented promise that `send: false` means no further packages + leave the machine. + """ + _add_package(store, "consented", "2026-08-26") + with store._connection() as connection: + with __import__( + "hermes_cli.sqlite_util", fromlist=["write_txn"] + ).write_txn(connection): + opt_in_period(connection, now=NOW) + + # User turns sending off; packages keep being collected. + with store._connection() as connection: + with __import__( + "hermes_cli.sqlite_util", fromlist=["write_txn"] + ).write_txn(connection): + record_revoked(connection) + for day in ("2026-08-27", "2026-08-28", "2026-08-29"): + _add_package(store, f"refused-{day}", day) + + # User re-enables a few days later. + later = NOW + timedelta(days=5) + transport = FakeTransport(*[FakeResponse(202)] * 10) + SharedMetricsSender( + store, ENDPOINT, post=transport, sleep=lambda _s: None, now=lambda: later + ).send_pending() + + sent = [json.loads(c["payload"])["package_id"] for c in transport.calls] + assert not any("refused" in pid for pid in sent), ( + f"transmitted packages collected while sending was off: {sent}" + ) + + def test_a_package_from_after_re_enabling_is_sent(self, store): + """The revocation fix must not wedge sending off permanently.""" + with store._connection() as connection: + with __import__( + "hermes_cli.sqlite_util", fromlist=["write_txn"] + ).write_txn(connection): + opt_in_period(connection, now=NOW) + record_revoked(connection) + + later = NOW + timedelta(days=5) + _add_package(store, "after-re-optin", later.date().isoformat()) + transport = FakeTransport(FakeResponse(202)) + SharedMetricsSender( + store, ENDPOINT, post=transport, sleep=lambda _s: None, now=lambda: later + ).send_pending() + assert len(transport.calls) == 1 + class TestIdentity: def test_install_id_is_never_transmitted(self, store): @@ -397,12 +455,23 @@ class TestClaimingAndBounds: "a concurrent pass claimed a package already in flight" ) - def test_a_claim_leases_the_row_into_the_future(self, store): - """The lease, not the send result, is what blocks a concurrent pass.""" + def test_a_claim_leases_the_row_long_enough_to_cover_a_worst_case_send( + self, store + ): + """The lease must outlast one package's worst legal duration. + + Asserting merely "in the future" passed for a 1-second lease, which is + useless: a package can legally take three 30s timeouts plus backoff. + """ _add_package(store, "pkg-1", "2026-08-26") claimed = _sender(store, FakeTransport())._claim_next(NOW, set()) assert claimed is not None - assert _row(store, "pkg-1")["next_attempt_at"] > "2026-08-26T12:00:00Z" + + worst_case = REQUEST_TIMEOUT_SECONDS * MAX_ATTEMPTS + 1 + 5 + 25 + deadline = NOW + timedelta(seconds=worst_case) + assert _row(store, "pkg-1")["next_attempt_at"] >= _iso(deadline), ( + "lease expires before a single package can legally finish" + ) def test_a_slow_multi_package_pass_does_not_lose_its_lease(self, store): """Regression: a batch-wide lease expired while later rows were sent. @@ -448,6 +517,85 @@ class TestClaimingAndBounds: f"a concurrent pass re-sent {second_posts} after a lease expired" ) + def test_a_re_eligible_head_row_does_not_starve_the_tail(self, store): + """Regression: `seen` terminated the pass instead of skipping a row. + + The claim query is LIMIT 1. When the oldest row was already handled + this pass but had become eligible again (short Retry-After, or a pass + outliving the 15-minute failure backoff), _claim_next returned None + and send_pending read that as "queue empty", abandoning every healthy + package behind it. Measured: 10 of 19 delivered. + """ + _add_package(store, "aaa-head", "2026-08-26") + for i in range(5): + _add_package(store, f"zzz-{i}", "2026-08-26") + # Order by created_at puts the head first. + with store._connection() as connection: + connection.execute( + "UPDATE package_outbox SET created_at = '2026-08-26T00:00:00Z'" + " WHERE package_id = 'aaa-head'" + ) + + posts = [] + + def transport(endpoint, payload, *, timeout): + pid = json.loads(payload)["package_id"] + posts.append(pid) + if pid == "aaa-head": + # Well-behaved service: retry in one second, so the head is + # eligible again immediately. + return FakeResponse(429, retry_after="1") + return FakeResponse(202) + + clock = {"t": NOW} + SharedMetricsSender( + store, + ENDPOINT, + post=transport, + sleep=lambda _s: None, + now=lambda: clock["t"] + timedelta(seconds=30 * len(posts)), + ).send_pending() + + delivered = {p for p in posts if p.startswith("zzz")} + assert delivered == {f"zzz-{i}" for i in range(5)}, ( + f"tail starved by a re-eligible head row; delivered {delivered}" + ) + + def test_a_poisoned_package_is_abandoned_eventually(self, store): + """Without a ceiling a doomed row is retried ~160 times over 30 days. + + Drives the real loop rather than pre-setting a counter: a row seeded + at exactly the limit is also excluded by other predicates, so that + version of this test passed even with the ceiling removed. + """ + _add_package(store, "pkg-1", "2026-08-26") + + clock = {"t": NOW} + attempts = [] + + def transport(endpoint, payload, *, timeout): + attempts.append(1) + return FakeResponse(503) + + # Run many passes, always well past any backoff, as a month of hook + # fires against a permanently failing package would. + for i in range(60): + SharedMetricsSender( + store, + ENDPOINT, + post=transport, + sleep=lambda _s: None, + now=lambda: clock["t"] + timedelta(hours=i), + ).send_pending() + + row = _row(store, "pkg-1") + assert row["send_attempts"] <= MAX_SEND_ATTEMPTS, ( + f"package retried {row['send_attempts']} times with no ceiling" + ) + assert len(attempts) < 100, ( + f"{len(attempts)} requests burned on one doomed package" + ) + def test_an_expired_lease_is_reclaimed(self, store): """A process killed mid-pass must not strand its packages.""" _add_package(store, "pkg-1", "2026-08-26") @@ -647,12 +795,22 @@ class TestCompression: captured = self._captured_request(payload) assert len(captured["data"]) < len(payload) - def test_gzip_round_trips_to_the_original_bytes(self): - import gzip as gziplib + def test_gzip_is_deterministic_across_time(self): + """Kills the mtime footgun: gzip embeds a timestamp by default. + + The in-pass retry test cannot catch this — both attempts compress + within the same second. Compressing the same bytes at two different + wall-clock seconds is what actually exercises mtime=0. + """ + import time as _time payload = json.dumps({"filler": "x" * 20000}).encode("utf-8") - captured = self._captured_request(payload) - assert gziplib.decompress(captured["data"]) == payload + first = self._captured_request(payload)["data"] + _time.sleep(1.1) + second = self._captured_request(payload)["data"] + assert first == second, ( + "gzip output changed between seconds — mtime is being embedded" + ) def test_small_payloads_are_sent_plain(self): payload = b'{"small": true}' diff --git a/tests/hermes_cli/test_shared_metrics_tools_toggle.py b/tests/hermes_cli/test_shared_metrics_tools_toggle.py index 31718d5d18..462bfcbd90 100644 --- a/tests/hermes_cli/test_shared_metrics_tools_toggle.py +++ b/tests/hermes_cli/test_shared_metrics_tools_toggle.py @@ -52,7 +52,7 @@ class TestToggle: "hermes_cli.setup.prompt_yes_no", lambda *_a, **_k: True ) monkeypatch.setattr( - "hermes_cli.setup._record_send_opt_in_day", lambda: None + "hermes_cli.setup._record_send_consent_change", lambda **_k: None ) monkeypatch.setattr( "hermes_cli.tools_config.save_config", From 36f1e01eba64d35b4c517c0966d549679c681055 Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Thu, 27 Aug 2026 09:14:16 +1000 Subject: [PATCH 011/634] chore(ci): retrigger checks after a message-only amend From 613849c1905a67bd66dfe4a095a25b0d367b90c4 Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Thu, 27 Aug 2026 09:47:47 +1000 Subject: [PATCH 012/634] fix(telemetry): close the consent window on the config transition Fourth independent review. Two more consent leaks, both reproduced through the real relay entry point before and after the fix. Both are failures of my own round-3 fix, which recorded revocation in the wrong place. BLOCKER 1 - revoking while idle recorded nothing. _record_revocation lived inside send_pending's loop, but _send_exported_packages returns early when send is false, before a sender is ever constructed. The dominant case is a user turning sending off while no pass is running, so the loop that was meant to observe the revocation could never run. Reproduced: 6 periods collected during a refused window were transmitted on re-enable. The window now closes on the observed config EDGE, before the early return. Last-seen send state is persisted because each hook fires in a fresh process, so a true->false transition is only visible by comparison. The rising edge also opens the window explicitly: the sender only runs when there is something to send, so a user who opts in and out before any package exists would otherwise have no window for record_revoked to close. BLOCKER 2 - turning COLLECTION off never recorded revocation. The not-enabled branch in setup.py force-set send=false and returned without calling _record_send_consent_change, so `hermes tools` -> disable shared metrics silently dropped consent while leaving the window open. Same retroactive release on re-enable. Both consent surfaces now record, and setup keeps the relay's edge detector in step. Also, from the same review's mutation sweep: - the scheme check is now pinned as an allowlist. Replacing the http test with `if True` survived the entire suite, because every non-http case targeted a REMOTE host where the loopback branch rejects anyway. Only a non-http scheme on loopback distinguishes the two. Shipped behaviour was already correct; nothing guarded it. - A.3 no longer claims rotation bounds long-term linkability outright. Measured against 11 real packages: resource is a stable low-entropy tuple and periods are contiguous across a rotation, so for a RARE configuration those can bridge windows. The honest claim is that rotation raises the cost, not that it makes correlation impossible. Two mutants are documented as unkillable rather than papered over with tests that only appear to cover them: the _defer clamp is unreachable from any current caller, and widening the falling-edge check to an unconditional else is behaviourally equivalent because record_revoked is idempotent and no-ops without an open window. An earlier version of the anti-spurious-revocation test could not fail either - it used a never-consented store, where record_revoked no-ops regardless. Rewritten to opt in, revoke, re-enable, and then assert that a steady enabled state does not re-close the reopened window. 259 tests pass. Staging E2E re-run: both packages 202. --- docs/observability/relay-shared-metrics.md | 15 ++ .../observability/relay_shared_metrics.py | 70 +++++++++ .../observability/shared_metrics_sender.py | 22 ++- hermes_cli/setup.py | 16 ++ tests/hermes_cli/test_setup_telemetry.py | 45 ++++++ .../test_shared_metrics_send_config.py | 22 +++ .../test_shared_metrics_send_wiring.py | 147 ++++++++++++++++++ 7 files changed, 331 insertions(+), 6 deletions(-) diff --git a/docs/observability/relay-shared-metrics.md b/docs/observability/relay-shared-metrics.md index e98e640845..f5af4e8e80 100644 --- a/docs/observability/relay-shared-metrics.md +++ b/docs/observability/relay-shared-metrics.md @@ -357,6 +357,21 @@ Rotation bounds long-term linkability without destroying short-term cohort analysis. A profile is one identity for the length of a window, and an unrelated identity after it. +**What rotation does not bound.** The identifier changes; the rest of the +envelope does not. `resource` (`os_family`, `architecture`, `install_method`, +`hermes_version`) is stable and low-entropy, and `period_start` / +`period_end` are contiguous across a rotation boundary. For a common +configuration this is no help to an observer — measured against the 11 real +packages in a development outbox, every one shares the same +`arm64 / macos / git` tuple. For a **rare** configuration it is a plausible +re-identification aid: an unusual architecture or install method, combined +with an uninterrupted daily period sequence, can bridge two windows. The +claim this design makes is therefore "rotation raises the cost of long-term +correlation", not "rotation makes it impossible". Narrowing that residue +would mean coarsening `resource` or jittering period boundaries, and neither +is worth the analytical loss today — but it should be a conscious decision, +not an unexamined one. + ### A.4 Reset behavior Removing `$HERMES_HOME/telemetry/shared_metrics` still resets local identity, diff --git a/hermes_cli/observability/relay_shared_metrics.py b/hermes_cli/observability/relay_shared_metrics.py index 94a1eac64f..2d7ec0583a 100644 --- a/hermes_cli/observability/relay_shared_metrics.py +++ b/hermes_cli/observability/relay_shared_metrics.py @@ -1083,6 +1083,67 @@ class _Runtime: if exported is not None: self._safe(self._send_exported_packages) + def _observe_send_consent(self, send_enabled: bool) -> None: + """Close the consent window on a true->false transition. + + Persists the last-seen send state so a change is detected even though + this runs in a fresh process each time. Only the falling edge matters: + opening a new window is the sender's job, on the next enabled pass. + + Failures here must never break the export hook, but they are logged at + warning rather than debug: silently failing to close a consent window + is a privacy-relevant event, not routine bookkeeping. + """ + try: + from hermes_cli.observability.shared_metrics_sender import ( + LAST_SEEN_SEND_KEY, + opt_in_period, + record_revoked, + ) + from hermes_cli.sqlite_util import write_txn + + current = "1" if send_enabled else "0" + with self.subscriber.store._connection() as connection: + with write_txn(connection): + row = connection.execute( + "SELECT value FROM telemetry_state WHERE key = ?", + (LAST_SEEN_SEND_KEY,), + ).fetchone() + previous = str(row[0]) if row is not None else None + + if send_enabled: + # Open the window HERE, on the rising edge, rather than + # leaving it to the sender's first claim. The sender + # only runs when there is something to send, so a user + # who opts in and then opts out before any package + # exists would otherwise have no window to close, and + # record_revoked (which requires one) would no-op. + opt_in_period(connection) + elif previous == "1": + # `previous == "1"` is the true falling edge. Widening + # this to an unconditional else would be behaviourally + # equivalent today — record_revoked is idempotent and + # no-ops without an open window — so no test can tell + # the two apart. It is written as an edge anyway + # because that is the property intended, and a future + # change to record_revoked should not silently turn + # every disabled pass into a revocation. + record_revoked(connection) + + if previous != current: + connection.execute( + """ + INSERT INTO telemetry_state(key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value + """, + (LAST_SEEN_SEND_KEY, current), + ) + except Exception: + logger.warning( + "Unable to record a shared-metrics consent transition", + exc_info=True, + ) + def _send_exported_packages(self) -> None: from hermes_cli.observability.shared_metrics_send_config import ( resolve_send_config, @@ -1097,6 +1158,15 @@ class _Runtime: return resolved = resolve_send_config(config) + + # Observe the consent EDGE before deciding whether to send. Recording + # revocation inside the send loop (as an earlier fix did) can never + # work: the dominant case is the user turning sending off while no + # pass is running, and then this method returns below without ever + # constructing a sender. The window has to close on the transition, + # not on the next transmission that by definition will not happen. + self._observe_send_consent(resolved.send) + if not resolved.send: return diff --git a/hermes_cli/observability/shared_metrics_sender.py b/hermes_cli/observability/shared_metrics_sender.py index e43f54cab1..49c2397f6e 100644 --- a/hermes_cli/observability/shared_metrics_sender.py +++ b/hermes_cli/observability/shared_metrics_sender.py @@ -95,6 +95,11 @@ OPT_IN_PERIOD_KEY = "send_opt_in_period" #: permanent for the packages collected while it was off. SEND_REVOKED_KEY = "send_revoked" +#: Last send-consent state this machine observed ("1"/"0"). Persisted because +#: each hook fires in a fresh process, so a true->false edge is only visible +#: by comparing against what was recorded last time. +LAST_SEEN_SEND_KEY = "send_last_seen" + def _utc_now() -> datetime: return datetime.now(timezone.utc) @@ -415,8 +420,13 @@ class SharedMetricsSender: ) def _defer(self, package_id: str, delay_seconds: int, reason: str) -> None: - # Never write a deadline in the past: that would make the row instantly - # re-eligible and let a pass spin on it. + # Defence in depth: no current caller can pass a non-positive delay + # (Retry-After is already clamped to [1, 86400] when parsed, and every + # other call site passes a positive constant), so this clamp is + # deliberately unreachable today and no test can distinguish it. It + # stays because a past deadline would make the row instantly + # re-eligible and let a pass spin on it — a cheap guard against a + # future caller that forgets. delay = max(1, int(delay_seconds)) retry_at = self._now().timestamp() + delay self._mark( @@ -514,10 +524,10 @@ class SharedMetricsSender: if not self._still_consented(): # The user turned sending off while this pass was running. # Stop without transmitting anything further, and close the - # consent window so a later re-enable cannot release the - # packages collected in the meantime. Recorded here as well as - # in the setup wizard because config.yaml can be edited by - # hand, which the wizard never sees. + # consent window. This covers only the mid-pass case; a + # revocation made while no pass is running is caught by the + # relay's edge detector before it early-returns, because this + # loop would never run to observe it. logger.info("Shared-metrics sending disabled mid-pass; stopping") self._record_revocation() break diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index 1743dc9343..5a000d0374 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -2454,6 +2454,11 @@ def setup_telemetry(config: dict): if shared_metrics.get("send") is True: shared_metrics["send"] = False print_info("Sending shared metrics disabled as well.") + # Turning collection off is also a withdrawal of send consent, and it + # has to close the window like any other. Recorded unconditionally: + # the send key may already be false in config while the consent window + # is still open, and that window must not survive to be reopened. + _record_send_consent_change(enabled=False) return print_success("Local shared metrics enabled.") @@ -2487,6 +2492,7 @@ def _record_send_consent_change(*, enabled: bool) -> None: try: from hermes_cli.observability.shared_metrics import SharedMetricsStore from hermes_cli.observability.shared_metrics_sender import ( + LAST_SEEN_SEND_KEY, opt_in_period, record_revoked, ) @@ -2499,6 +2505,16 @@ def _record_send_consent_change(*, enabled: bool) -> None: opt_in_period(connection) else: record_revoked(connection) + # Keep the relay's edge detector in step. Without this the + # wizard's change looks like "no transition" on the next hook + # fire, and a later true->false edge could be missed. + connection.execute( + """ + INSERT INTO telemetry_state(key, value) VALUES (?, ?) + ON CONFLICT(key) DO UPDATE SET value = excluded.value + """, + (LAST_SEEN_SEND_KEY, "1" if enabled else "0"), + ) except Exception: # Never block the wizard on telemetry bookkeeping. The sender records # the same transitions on its next pass. diff --git a/tests/hermes_cli/test_setup_telemetry.py b/tests/hermes_cli/test_setup_telemetry.py index e6ebcb428c..4f66259eaa 100644 --- a/tests/hermes_cli/test_setup_telemetry.py +++ b/tests/hermes_cli/test_setup_telemetry.py @@ -25,6 +25,51 @@ def test_setup_telemetry_enables_shared_metrics(monkeypatch): assert config["telemetry"]["shared_metrics"]["enabled"] is True +def test_disabling_collection_closes_the_send_consent_window(monkeypatch, tmp_path): + """`hermes tools` -> disable shared metrics must withdraw send consent. + + The not-enabled branch returned early without recording anything, so the + consent window stayed open and re-enabling later would release every + package collected in between. + """ + from hermes_cli.observability.shared_metrics import SharedMetricsStore + from hermes_cli.observability.shared_metrics_sender import SEND_REVOKED_KEY + + store = SharedMetricsStore( + database_path=tmp_path / "m.db", outbox_directory=tmp_path / "o" + ) + monkeypatch.setattr( + "hermes_cli.observability.shared_metrics.SharedMetricsStore", + lambda *a, **k: store, + ) + + # The user had consented; now they turn collection off entirely. + monkeypatch.setattr( + "hermes_cli.setup.prompt_yes_no", lambda _question, default: False + ) + config = {"telemetry": {"shared_metrics": {"enabled": True, "send": True}}} + # Consent was granted earlier, so a window is already open — that is + # precisely the state whose closure must be recorded. + from hermes_cli.sqlite_util import write_txn + from hermes_cli.observability.shared_metrics_sender import opt_in_period + + with store._connection() as connection: + with write_txn(connection): + opt_in_period(connection) + + setup_telemetry(config) + + assert config["telemetry"]["shared_metrics"]["enabled"] is False + assert config["telemetry"]["shared_metrics"]["send"] is False + with store._connection() as connection: + row = connection.execute( + "SELECT value FROM telemetry_state WHERE key = ?", (SEND_REVOKED_KEY,) + ).fetchone() + assert row is not None and row[0] == "1", ( + "disabling collection left the send consent window open" + ) + + def test_setup_parser_accepts_telemetry_section(): parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(dest="command") diff --git a/tests/hermes_cli/test_shared_metrics_send_config.py b/tests/hermes_cli/test_shared_metrics_send_config.py index 227c7a1cfa..2af8958a2b 100644 --- a/tests/hermes_cli/test_shared_metrics_send_config.py +++ b/tests/hermes_cli/test_shared_metrics_send_config.py @@ -136,6 +136,28 @@ class TestTransportSafety: ) assert resolved.send is False + @pytest.mark.parametrize( + "endpoint", + [ + "ftp://localhost/v1/telemetry", + "gopher://localhost/v1/telemetry", + "ws://127.0.0.1/v1/telemetry", + ], + ) + def test_a_non_http_scheme_on_loopback_is_still_refused(self, endpoint): + """The scheme is allowlisted, not merely checked for plaintext http. + + Gap found by mutation testing: replacing the `http` scheme test with + `if True` survived the whole suite, because every non-http scheme case + pointed at a REMOTE host, where the loopback branch rejects it anyway. + Only a non-http scheme aimed at loopback distinguishes an allowlist + from a plaintext-only check. + """ + resolved = resolve_send_config( + _config(enabled=True, send=True, endpoint=endpoint) + ) + assert resolved.send is False + def test_unsafe_endpoint_does_not_block_collection(self): resolved = resolve_send_config( _config(enabled=True, send=True, endpoint="http://example.test/v1") diff --git a/tests/hermes_cli/test_shared_metrics_send_wiring.py b/tests/hermes_cli/test_shared_metrics_send_wiring.py index 2a2be1a7ef..3900847955 100644 --- a/tests/hermes_cli/test_shared_metrics_send_wiring.py +++ b/tests/hermes_cli/test_shared_metrics_send_wiring.py @@ -23,6 +23,31 @@ class FakeStore: return [] +class RealBackedStore: + """A store with a genuine SQLite connection, for consent-state tests. + + The consent edge detector writes to telemetry_state, and it is wrapped in + a broad except. Against a stub without _connection it would swallow an + AttributeError and silently do nothing — which is exactly the failure this + file needs to be able to catch. + """ + + def __init__(self, tmp_path): + from hermes_cli.observability.shared_metrics import SharedMetricsStore + + self._real = SharedMetricsStore( + database_path=tmp_path / "m.db", outbox_directory=tmp_path / "o" + ) + self.exported = 0 + + def _connection(self): + return self._real._connection() + + def create_and_export_package_if_due(self): + self.exported += 1 + return [] + + class FakeSubscriber: def __init__(self): self.store = FakeStore() @@ -184,6 +209,128 @@ class TestInteractivePathIsNotBlocked: runtime._join_send_thread(timeout=5) +class TestConsentRevocationWindow: + """The falling edge must close the window even with no pass running. + + Round 3 recorded revocation inside the send loop, which cannot fire for + the dominant case: the user turns sending off while idle, so the relay + early-returns and no sender is ever built. Re-enabling then released + every package collected during the refused window. + """ + + def _runtime(self, tmp_path): + runtime = Runtime() + runtime.subscriber.store = RealBackedStore(tmp_path) + return runtime + + def _state(self, runtime, key): + with runtime.subscriber.store._connection() as connection: + row = connection.execute( + "SELECT value FROM telemetry_state WHERE key = ?", (key,) + ).fetchone() + return row[0] if row else None + + def test_revoking_while_idle_closes_the_window( + self, monkeypatch, tmp_path, capture_sender + ): + from hermes_cli.observability.shared_metrics_sender import ( + SEND_REVOKED_KEY, + ) + + runtime = self._runtime(tmp_path) + + _set_config(monkeypatch, _config(enabled=True, send=True)) + runtime._send_exported_packages() + + # User edits config.yaml: send: false. Hooks keep firing normally. + _set_config(monkeypatch, _config(enabled=True, send=False)) + for _ in range(6): + runtime._send_exported_packages() + + assert self._state(runtime, SEND_REVOKED_KEY) == "1", ( + "revoking while no pass was running left the consent window open" + ) + + def test_no_spurious_revocation_when_nothing_changes( + self, monkeypatch, tmp_path, capture_sender + ): + """The detector must key on an EDGE, not on every disabled pass. + + A level trigger re-closes a window the user has since REOPENED: each + later disabled pass stamps revoked again, so the next enabled pass + advances the gate and silently drops packages the user did consent to. + Mutation-checked — an earlier version of this test used a + never-consented store, where record_revoked no-ops regardless, and so + could not tell an edge trigger from a level trigger. + """ + from hermes_cli.observability.shared_metrics_sender import ( + OPT_IN_PERIOD_KEY, + SEND_REVOKED_KEY, + ) + + runtime = self._runtime(tmp_path) + + _set_config(monkeypatch, _config(enabled=True, send=True)) + runtime._send_exported_packages() + + _set_config(monkeypatch, _config(enabled=True, send=False)) + runtime._send_exported_packages() + assert self._state(runtime, SEND_REVOKED_KEY) == "1" + + # User changes their mind and re-enables. + _set_config(monkeypatch, _config(enabled=True, send=True)) + runtime._send_exported_packages() + assert self._state(runtime, SEND_REVOKED_KEY) is None, ( + "re-enabling must clear the revocation marker" + ) + reopened = self._state(runtime, OPT_IN_PERIOD_KEY) + + # Further ENABLED passes must not disturb the reopened window. + for _ in range(4): + runtime._send_exported_packages() + + assert self._state(runtime, SEND_REVOKED_KEY) is None, ( + "a steady enabled state re-closed the consent window" + ) + assert self._state(runtime, OPT_IN_PERIOD_KEY) == reopened + + def test_a_never_consented_user_is_never_marked_revoked( + self, monkeypatch, tmp_path, capture_sender + ): + from hermes_cli.observability.shared_metrics_sender import ( + SEND_REVOKED_KEY, + ) + + runtime = self._runtime(tmp_path) + _set_config(monkeypatch, _config(enabled=True, send=False)) + for _ in range(5): + runtime._send_exported_packages() + + assert self._state(runtime, SEND_REVOKED_KEY) is None + + def test_re_enabling_after_an_idle_revocation_starts_a_new_window( + self, monkeypatch, tmp_path, capture_sender + ): + from hermes_cli.observability.shared_metrics_sender import ( + OPT_IN_PERIOD_KEY, + SEND_REVOKED_KEY, + ) + + runtime = self._runtime(tmp_path) + _set_config(monkeypatch, _config(enabled=True, send=True)) + runtime._send_exported_packages() + first_window = self._state(runtime, OPT_IN_PERIOD_KEY) + + _set_config(monkeypatch, _config(enabled=True, send=False)) + runtime._send_exported_packages() + assert self._state(runtime, SEND_REVOKED_KEY) == "1" + + # Re-enabling must not simply resume the original window. + _set_config(monkeypatch, _config(enabled=True, send=True)) + runtime._send_exported_packages() + assert first_window is not None + + class TestFailureIsolation: def test_a_sender_crash_does_not_propagate(self, runtime, monkeypatch): class Exploding: From 5e380d95ba76484fc58118b9fd4299c025075bc0 Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Thu, 27 Aug 2026 10:42:31 +1000 Subject: [PATCH 013/634] refactor(telemetry): replace consent day-stamp with explicit intervals Structural fix after five review rounds put four blockers in the same subsystem. The root cause was representational: consent history is a sequence of on/off intervals, but it was stored as ONE moving day-stamp plus a revoked flag. Every fix had to mutate that scalar at exactly the right moment from exactly the right place, and each round the mutation was missing from some reachable path (write-once stamp in R3; recorded inside a loop that never runs when sending is off in R4; dead code whenever collection was off in R5). Consent is now recorded as explicit intervals (send_consent_windows) and eligibility is a pure derivation: a package is sent only when its whole period falls inside a recorded window. One writer - reconcile_send_consent - derives window state from an observation of (config, now). It is idempotent and order-independent, so the wizard, the relay, and the mid-pass check all call the same function and cannot disagree; there are no edges to detect and no ordering between writers to get wrong. The relay reconciles once per process BEFORE the collection gate, which fixes round-5 D1 (enabled:false made the only idle-path observer unreachable). The claim reads the table and never writes it, removing the read-path mutation (D2's rewrite vector). Timestamp discipline, each rule load-bearing and mutation-tested: - 'obs' high-water mark: monotonic, advanced only by observations; confirms an open window forward (last_confirmed_at). - 'data' high-water mark: advanced only by stored package period_end; clamps window OPENS so a rolled-back clock cannot slide a window under refused packages already on disk (round-5 D2). - A close stamps last_confirmed_at, never "now": consent is asserted only for observed time, so a hand-edited config with no process running for 90 days fails closed (round-5 D1 strongest form). - The gate requires period containment, not period_start >=, so an intra-day revoke/re-enable holds back the day package (round-5 D3). - Unlike the day-stamp, a revoke/re-enable cycle no longer destroys the undelivered backlog from the earlier consented window (round-5 D4). The redesign was validated BEFORE implementation against all 13 reproduced defect scenarios on a real store; the first two drafts each failed scenarios in that harness (v1 leaked the unobserved-gap case by closing at "now"; v2 leaked refused windows by letting data stamps confirm consent). The harness ships as tests/hermes_cli/test_shared_metrics_consent_windows.py. Deleted: OPT_IN_PERIOD_KEY, SEND_REVOKED_KEY, LAST_SEEN_SEND_KEY, opt_in_period(), record_revoked(), the relay edge detector body, and the setup wizard's key bookkeeping (~170 lines of transition machinery). Schema: two additive tables, version deliberately unchanged; verified against a copy of the real production DB (13 rows intact, reopen no-op). Also kills round-5's M8 survivor: the seen-exclusion mutation now fails the suite. New mutation sweep: 8/8 killed, including one vacuous test of my own this round (obs-mark monotonicity was covered only by coincidence of the data mark; now pinned directly). Documented cost: a fresh package waits at most one process start after its period completes before release (fail-closed direction). 270 tests pass; ruff and windows-footguns clean. Staging E2E re-run through the interval gate: both packages 202. --- docs/observability/relay-shared-metrics.md | 32 ++- hermes_cli/config_defaults.py | 7 +- .../observability/relay_shared_metrics.py | 96 ++++---- hermes_cli/observability/shared_metrics.py | 60 +++++ .../observability/shared_metrics_sender.py | 153 +++++++----- hermes_cli/setup.py | 35 +-- scripts/e2e_shared_metrics_staging.py | 37 ++- tests/hermes_cli/test_setup_telemetry.py | 22 +- .../test_shared_metrics_consent_windows.py | 221 ++++++++++++++++++ .../test_shared_metrics_send_wiring.py | 146 ++++++------ .../hermes_cli/test_shared_metrics_sender.py | 137 +++++++---- .../test_shared_metrics_sender_e2e.py | 20 +- 12 files changed, 702 insertions(+), 264 deletions(-) create mode 100644 tests/hermes_cli/test_shared_metrics_consent_windows.py diff --git a/docs/observability/relay-shared-metrics.md b/docs/observability/relay-shared-metrics.md index f5af4e8e80..b538011675 100644 --- a/docs/observability/relay-shared-metrics.md +++ b/docs/observability/relay-shared-metrics.md @@ -301,10 +301,20 @@ telemetry: - Like `enabled`, `send` is profile-owned and is not overridden by managed-scope configuration. -**Only packages for periods on or after the opt-in day are ever sent.** The -opt-in day (UTC) is recorded when `send` first becomes true, and any package -whose `period_start` predates it is permanently excluded, however late it was -created. +**A package is only sent when its whole period falls inside a recorded +consent window.** Consent is stored as explicit intervals in the shared- +metrics SQLite store (`send_consent_windows`): a window opens when `send: +true` is first observed, is confirmed forward by every later observation, +and closes — at the last *confirmed* moment, never at the wall clock — when +`send: false` is observed. A single reconciler derives this table from the +config on every process start, so wizard changes, hand-edits to +`config.yaml`, and mid-pass revocations all take the same path, and no +transition can be missed by any of them. + +Any package whose period predates the first window, falls between windows, +or runs past the newest confirmed moment is excluded — the gate fails +closed. A fresh package therefore waits at most one process start after its +period completes before becoming eligible. The gate is on the **period**, not on the package's creation time. One period is split across several packages created on different days: a day's first @@ -389,11 +399,15 @@ before every package, so a pass already in flight stops after the package it is currently sending rather than draining its whole batch. It does not delete previously transmitted packages, and it does not stop local collection. -Turning sending off also **closes the consent window**. Packages collected -while it was off are never transmitted, even if sending is later re-enabled — -re-enabling starts a new window from that day. Without this, a write-once -opt-in date would have retroactively released the entire refused period the -next time the user changed their mind. +Turning sending off also **closes the consent window** — at the last moment +consent was actually observed, not at the wall clock. Packages whose periods +fall between one window and the next are never transmitted, even if sending +is later re-enabled, and this holds for any number of on/off cycles, across +hand-edits with no process running, and under a clock that jumps backwards +(window opens are clamped above every timestamp already in the store). +Unlike the earlier single moving opt-in date, closing and reopening does NOT +discard the still-undelivered backlog from a previous consented window — +those packages stay inside their own interval and remain eligible. ### A.5 Retention diff --git a/hermes_cli/config_defaults.py b/hermes_cli/config_defaults.py index 5303e020df..804a48ea38 100644 --- a/hermes_cli/config_defaults.py +++ b/hermes_cli/config_defaults.py @@ -3334,9 +3334,10 @@ DEFAULT_CONFIG = { # Transmit exported packages to the Nous telemetry service. # Requires ``enabled``: it never switches collection on by itself, # and ``send`` without ``enabled`` is logged as an error rather - # than silently doing nothing. Only packages whose period starts - # on or after the opt-in day are ever sent, so data collected - # before consent stays local. + # than silently doing nothing. A package is only sent when its + # whole period falls inside a recorded consent window, so data + # collected before consent — or while it was withdrawn — stays + # local. "send": False, # Ingest endpoint. Production by default; override for staging or # a local test server. Deliberately NOT overridable by an diff --git a/hermes_cli/observability/relay_shared_metrics.py b/hermes_cli/observability/relay_shared_metrics.py index 2d7ec0583a..978497945d 100644 --- a/hermes_cli/observability/relay_shared_metrics.py +++ b/hermes_cli/observability/relay_shared_metrics.py @@ -1084,60 +1084,26 @@ class _Runtime: self._safe(self._send_exported_packages) def _observe_send_consent(self, send_enabled: bool) -> None: - """Close the consent window on a true->false transition. + """Reconcile consent windows with the observed config state. - Persists the last-seen send state so a change is detected even though - this runs in a fresh process each time. Only the falling edge matters: - opening a new window is the sender's job, on the next enabled pass. + Thin wrapper over the SINGLE consent writer. The old edge-detection + body (last-seen key, rising/falling branches) is gone: reconciliation + derives the correct window state from what it observes, so there is + no transition to miss and no ordering between callers to get wrong. - Failures here must never break the export hook, but they are logged at + Failures must never break the export hook, but they are logged at warning rather than debug: silently failing to close a consent window is a privacy-relevant event, not routine bookkeeping. """ try: from hermes_cli.observability.shared_metrics_sender import ( - LAST_SEEN_SEND_KEY, - opt_in_period, - record_revoked, + reconcile_send_consent, ) from hermes_cli.sqlite_util import write_txn - current = "1" if send_enabled else "0" with self.subscriber.store._connection() as connection: with write_txn(connection): - row = connection.execute( - "SELECT value FROM telemetry_state WHERE key = ?", - (LAST_SEEN_SEND_KEY,), - ).fetchone() - previous = str(row[0]) if row is not None else None - - if send_enabled: - # Open the window HERE, on the rising edge, rather than - # leaving it to the sender's first claim. The sender - # only runs when there is something to send, so a user - # who opts in and then opts out before any package - # exists would otherwise have no window to close, and - # record_revoked (which requires one) would no-op. - opt_in_period(connection) - elif previous == "1": - # `previous == "1"` is the true falling edge. Widening - # this to an unconditional else would be behaviourally - # equivalent today — record_revoked is idempotent and - # no-ops without an open window — so no test can tell - # the two apart. It is written as an edge anyway - # because that is the property intended, and a future - # change to record_revoked should not silently turn - # every disabled pass into a revocation. - record_revoked(connection) - - if previous != current: - connection.execute( - """ - INSERT INTO telemetry_state(key, value) VALUES (?, ?) - ON CONFLICT(key) DO UPDATE SET value = excluded.value - """, - (LAST_SEEN_SEND_KEY, current), - ) + reconcile_send_consent(connection, send_enabled) except Exception: logger.warning( "Unable to record a shared-metrics consent transition", @@ -1259,8 +1225,54 @@ def handles_hook(hook_name: str) -> bool: return hook_name in HANDLED_HOOKS and enabled() +_consent_reconcile_done = False + + +def _reconcile_send_consent_once() -> None: + """Reconcile consent windows with config, once per process. + + Runs BEFORE and INDEPENDENT of the collection gate — that placement is + the fix for the round-5 D1 leak, where the only idle-path consent + observer sat behind ``handles_hook()`` and became dead code the moment + ``enabled: false`` was set. A user with collection off still gets their + send-consent windows reconciled here. + + Skipped only when there is no store on disk AND consent is off: with no + store there are no packages, so there is nothing a window could protect, + and creating ``~/.hermes/telemetry`` for every fully-disabled user would + be a behaviour change in the wrong direction. + """ + global _consent_reconcile_done + if _consent_reconcile_done: + return + _consent_reconcile_done = True + try: + from hermes_cli.config import read_raw_config_readonly + from hermes_cli.observability.shared_metrics import SharedMetricsStore + from hermes_cli.observability.shared_metrics_send_config import ( + resolve_send_config, + ) + from hermes_cli.observability.shared_metrics_sender import ( + reconcile_send_consent, + ) + from hermes_cli.sqlite_util import write_txn + + resolved = resolve_send_config(read_raw_config_readonly() or {}) + store = SharedMetricsStore() + if not resolved.send and not store.database_path.exists(): + return + with store._connection() as connection: + with write_txn(connection): + reconcile_send_consent(connection, resolved.send) + except Exception: + logger.warning( + "Unable to reconcile shared-metrics send consent", exc_info=True + ) + + def observe_lifecycle(hook_name: str, **kwargs: Any) -> None: """Project one Hermes lifecycle event into the core Relay integration.""" + _reconcile_send_consent_once() if not handles_hook(hook_name): return if not relay_runtime.relay_instrumentation_enabled(): diff --git a/hermes_cli/observability/shared_metrics.py b/hermes_cli/observability/shared_metrics.py index bf5c1fb0bf..32dfc4fff6 100644 --- a/hermes_cli/observability/shared_metrics.py +++ b/hermes_cli/observability/shared_metrics.py @@ -338,6 +338,7 @@ class SharedMetricsStore: """ ) SharedMetricsStore._add_send_columns(connection) + SharedMetricsStore._add_consent_tables(connection) connection.execute( """ INSERT INTO telemetry_state(key, value) @@ -383,6 +384,55 @@ class SharedMetricsStore: f"ALTER TABLE package_outbox ADD COLUMN {column} {declaration}" ) + @staticmethod + def _add_consent_tables(connection: sqlite3.Connection) -> None: + """Create the consent-window tables, idempotently. + + Additive like ``_add_send_columns`` — the schema version is + deliberately NOT bumped, and old readers never touch these tables. + + ``send_consent_windows`` records consent as explicit intervals rather + than a moving day-stamp: a window is opened when send consent is + observed, heartbeat-confirmed on every later observation, and closed + at the LAST CONFIRMED moment (never "now") when consent is observed + withdrawn. Consent is asserted only for time that was actually + observed, so unobserved gaps — a hand-edited config with no process + running — fail closed by construction. + + ``consent_marks`` holds two monotonic high-water marks with strictly + separated roles: + + - ``obs``: the latest observation stamp ever seen. Advanced only by + the reconciler. Confirms consent and clamps window closes. + - ``data``: the latest package ``period_end`` ever stored. Advanced + only by the package writer. Clamps window OPENS, so a rolled-back + clock can never open a window underneath packages that already + exist on disk. + + The separation is load-bearing: letting data stamps confirm consent + re-created a refused-window leak (packages stored during an off + window would vouch for it), and letting observation stamps clamp + opens is not enough on its own to stop a rollback sliding a window + under existing refused data. + """ + connection.execute( + """ + CREATE TABLE IF NOT EXISTS send_consent_windows ( + opened_at TEXT NOT NULL, + last_confirmed_at TEXT NOT NULL, + closed_at TEXT + ) + """ + ) + connection.execute( + """ + CREATE TABLE IF NOT EXISTS consent_marks ( + name TEXT PRIMARY KEY CHECK (name IN ('obs', 'data')), + stamp TEXT NOT NULL + ) + """ + ) + @staticmethod def _create_counter_aggregates_table(connection: sqlite3.Connection) -> None: connection.execute( @@ -617,6 +667,16 @@ class SharedMetricsStore: payload["generated_at"], ), ) + # Advance the data high-water mark. This is the ONLY writer of the + # 'data' mark: it clamps consent-window opens so a rolled-back clock + # can never open a window underneath packages that already exist. + connection.execute( + """ + INSERT INTO consent_marks(name, stamp) VALUES ('data', ?) + ON CONFLICT(name) DO UPDATE SET stamp = MAX(stamp, excluded.stamp) + """, + (payload["period_end"],), + ) for row in rows: connection.execute( """ diff --git a/hermes_cli/observability/shared_metrics_sender.py b/hermes_cli/observability/shared_metrics_sender.py index 49c2397f6e..389611ed37 100644 --- a/hermes_cli/observability/shared_metrics_sender.py +++ b/hermes_cli/observability/shared_metrics_sender.py @@ -17,7 +17,10 @@ file. See Appendix A.7 of ``docs/observability/relay-shared-metrics.md``. **Consent is gated on the package's PERIOD, not its creation time.** One period is split across packages created on different days, so a created-at gate would send a period's tail while dropping its head and silently -undercount the opt-in day. +undercount the first consented day. The gate itself is interval containment: +the period must fall entirely inside a recorded consent window +(``send_consent_windows``), maintained by the single ``reconcile_send_consent`` +writer below. """ from __future__ import annotations @@ -88,18 +91,6 @@ _PERMANENT_STATUSES = frozenset({400, 413}) #: doomed package at the head of the queue. MAX_SEND_ATTEMPTS = 25 -OPT_IN_PERIOD_KEY = "send_opt_in_period" - -#: Set when sending is turned off, cleared by the next enabled pass (which -#: also advances OPT_IN_PERIOD_KEY). This is what makes consent revocation -#: permanent for the packages collected while it was off. -SEND_REVOKED_KEY = "send_revoked" - -#: Last send-consent state this machine observed ("1"/"0"). Persisted because -#: each hook fires in a fresh process, so a true->false edge is only visible -#: by comparing against what was recorded last time. -LAST_SEEN_SEND_KEY = "send_last_seen" - def _utc_now() -> datetime: return datetime.now(timezone.utc) @@ -173,46 +164,86 @@ def _retry_after_seconds(value: str | None, default: int) -> int: return default -def opt_in_period(connection: sqlite3.Connection, *, now: datetime | None = None) -> str: - """Return the day (UTC) from which packages may be sent. +def reconcile_send_consent( + connection: sqlite3.Connection, + send_enabled: bool, + *, + now: datetime | None = None, +) -> None: + """Reconcile the consent-window table with the observed config state. - Must run inside a write transaction. + THE ONLY writer of consent state. Must run inside a write transaction. + A pure function of (config, now, store): call it from anywhere, any + number of times, in any order — the resulting windows are the same. This + replaces the previous edge-detection design, whose three partial + observers (wizard, relay, mid-pass) each covered a different subset of + transitions and repeatedly leaked the transitions between the subsets. - This is the CURRENT consent window's start, not a permanent first-ever - opt-in date. If the user previously turned sending off, ``record_revoked`` - stamps that; the next enabled pass advances the gate to the day sending - resumed, so packages collected during the opted-out window are never - transmitted. Without that advance, re-enabling would retroactively release - the entire period the user had explicitly refused. + Timestamp discipline (each rule is load-bearing; see the validation + harness in tests/hermes_cli/test_shared_metrics_consent_windows.py): + + - The 'obs' mark advances to every observation stamp, monotonically. + An open window's ``last_confirmed_at`` follows it: consent is asserted + only for time that was actually observed. + - A close is stamped at ``last_confirmed_at`` — never "now" — so an + unobserved gap (hand-edited config, machine off for 90 days) is never + inside a window and fails closed. + - An open clamps to ``max(now, obs, data)``: a rolled-back clock cannot + open a window underneath refused packages already on disk, and cannot + make the new window adjacent to the previous close. """ - today = (now or _utc_now()).date().isoformat() + stamp = _isoformat(now or _utc_now()) + connection.execute( + """ + INSERT INTO consent_marks(name, stamp) VALUES ('obs', ?) + ON CONFLICT(name) DO UPDATE SET stamp = MAX(stamp, excluded.stamp) + """, + (stamp,), + ) + marks = dict( + connection.execute("SELECT name, stamp FROM consent_marks").fetchall() + ) + obs = marks["obs"] # >= stamp; immune to clock rollback + data = marks.get("data") - revoked = _state_get(connection, SEND_REVOKED_KEY) - if revoked: - # Sending resumed after a revocation: the new window starts today. - _state_set(connection, OPT_IN_PERIOD_KEY, today) + open_row = connection.execute( + "SELECT rowid FROM send_consent_windows WHERE closed_at IS NULL" + ).fetchone() + + if send_enabled: + if open_row is None: + opened = max(x for x in (obs, data) if x is not None) + connection.execute( + "INSERT INTO send_consent_windows(opened_at, last_confirmed_at)" + " VALUES (?, ?)", + (opened, opened), + ) + else: + connection.execute( + "UPDATE send_consent_windows" + " SET last_confirmed_at = MAX(last_confirmed_at, ?)" + " WHERE rowid = ?", + (obs, open_row[0]), + ) + elif open_row is not None: connection.execute( - "DELETE FROM telemetry_state WHERE key = ?", (SEND_REVOKED_KEY,) + "UPDATE send_consent_windows SET closed_at = last_confirmed_at" + " WHERE rowid = ?", + (open_row[0],), ) - return today - - existing = _state_get(connection, OPT_IN_PERIOD_KEY) - if existing: - return existing - - _state_set(connection, OPT_IN_PERIOD_KEY, today) - return today -def record_revoked(connection: sqlite3.Connection) -> None: - """Mark that sending was turned off, closing the current consent window. - - Idempotent. The marker is only cleared by the next enabled pass, which - also advances the gate — so any package collected between the two events - stays local permanently. - """ - if _state_get(connection, OPT_IN_PERIOD_KEY): - _state_set(connection, SEND_REVOKED_KEY, "1") +#: Claim-time consent predicate: the package's period must fall entirely +#: inside SOME recorded consent window. An open window vouches only up to its +#: last confirmed moment, so a package whose period runs past it waits for +#: the next reconcile heartbeat (fail-closed; released within one hook fire). +CONSENT_GATE_SQL = """EXISTS ( + SELECT 1 FROM send_consent_windows w + WHERE package_outbox.period_start >= w.opened_at + AND package_outbox.period_end <= + CASE WHEN w.closed_at IS NULL THEN w.last_confirmed_at + ELSE w.closed_at END +)""" def _state_get(connection: sqlite3.Connection, key: str) -> str | None: @@ -278,7 +309,6 @@ class SharedMetricsSender: """ with self._store._connection() as connection: with write_txn(connection): - period = opt_in_period(connection, now=now) stamp = _isoformat(now) lease_until = now + timedelta(seconds=_CLAIM_LEASE_SECONDS) @@ -286,6 +316,10 @@ class SharedMetricsSender: exclusion = ( f" AND package_id NOT IN ({placeholders})" if seen else "" ) + # Consent is a READ here — the claim must never mutate the + # window table. The old design's opt_in_period() call at this + # exact spot meant selecting a row could rewrite what was + # permitted to be sent (and did, under a rolled-back clock). row = connection.execute( f""" SELECT package_id, payload_json, sent_install_id @@ -293,13 +327,13 @@ class SharedMetricsSender: WHERE exported_at IS NOT NULL AND (send_state IS NULL OR send_state = 'pending') AND (next_attempt_at IS NULL OR next_attempt_at <= ?) - AND substr(period_start, 1, 10) >= ? + AND {CONSENT_GATE_SQL} AND send_attempts < ? {exclusion} ORDER BY created_at, package_id LIMIT 1 """, - (stamp, period, MAX_SEND_ATTEMPTS, *sorted(seen)), + (stamp, MAX_SEND_ATTEMPTS, *sorted(seen)), ).fetchone() if row is None: return None @@ -523,13 +557,12 @@ class SharedMetricsSender: for _ in range(MAX_PACKAGES_PER_PASS): if not self._still_consented(): # The user turned sending off while this pass was running. - # Stop without transmitting anything further, and close the - # consent window. This covers only the mid-pass case; a - # revocation made while no pass is running is caught by the - # relay's edge detector before it early-returns, because this - # loop would never run to observe it. + # Stop without transmitting anything further, and reconcile + # so the window closes at its last confirmed moment. This is + # the same single writer every other observation point uses — + # not a separate recording mechanism. logger.info("Shared-metrics sending disabled mid-pass; stopping") - self._record_revocation() + self._reconcile(send_enabled=False) break try: package = self._claim_next(self._now(), seen) @@ -561,14 +594,18 @@ class SharedMetricsSender: outcome.deferred += 1 return outcome - def _record_revocation(self) -> None: - """Close the consent window after an observed revocation.""" + def _reconcile(self, *, send_enabled: bool) -> None: + """Run the single consent writer from within a pass.""" try: with self._store._connection() as connection: with write_txn(connection): - record_revoked(connection) + reconcile_send_consent( + connection, send_enabled, now=self._now() + ) except Exception: - logger.debug("Unable to record consent revocation", exc_info=True) + logger.warning( + "Unable to reconcile shared-metrics consent", exc_info=True + ) def _still_consented(self) -> bool: """Re-read profile-owned send consent. diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index 5a000d0374..6af33a000d 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -2481,43 +2481,28 @@ def setup_telemetry(config: dict): def _record_send_consent_change(*, enabled: bool) -> None: - """Persist a consent transition at the moment the user makes it. + """Reconcile consent windows at the moment the user decides. - Enabling stamps the day so the gate excludes anything collected earlier. - Disabling stamps a revocation so that if the user ever re-enables, the - packages collected while sending was off are never released — the doc - promises `send: false` means no further packages leave the machine, and - that has to survive a later change of mind. + Same single writer as the relay and the sender — reconciliation derives + the window state from the observation, so wizard, relay, and mid-pass + callers cannot disagree. The relay's once-per-process reconcile would + catch this on the next hook fire anyway; running it here just makes the + wizard's effect immediate. """ try: from hermes_cli.observability.shared_metrics import SharedMetricsStore from hermes_cli.observability.shared_metrics_sender import ( - LAST_SEEN_SEND_KEY, - opt_in_period, - record_revoked, + reconcile_send_consent, ) from hermes_cli.sqlite_util import write_txn store = SharedMetricsStore() with store._connection() as connection: with write_txn(connection): - if enabled: - opt_in_period(connection) - else: - record_revoked(connection) - # Keep the relay's edge detector in step. Without this the - # wizard's change looks like "no transition" on the next hook - # fire, and a later true->false edge could be missed. - connection.execute( - """ - INSERT INTO telemetry_state(key, value) VALUES (?, ?) - ON CONFLICT(key) DO UPDATE SET value = excluded.value - """, - (LAST_SEEN_SEND_KEY, "1" if enabled else "0"), - ) + reconcile_send_consent(connection, enabled) except Exception: - # Never block the wizard on telemetry bookkeeping. The sender records - # the same transitions on its next pass. + # Never block the wizard on telemetry bookkeeping. The relay runs the + # same reconciliation on the next lifecycle hook. logger.debug("Unable to record shared-metrics consent change", exc_info=True) diff --git a/scripts/e2e_shared_metrics_staging.py b/scripts/e2e_shared_metrics_staging.py index 3e52e61a7a..e0c497a342 100644 --- a/scripts/e2e_shared_metrics_staging.py +++ b/scripts/e2e_shared_metrics_staging.py @@ -62,6 +62,31 @@ def main() -> int: ) today = datetime.now(timezone.utc).date().isoformat() + # The generator only exports COMPLETED periods, so the realistic E2E + # package is yesterday's. It also has to be: the consent gate only + # releases a package once its whole period is confirmed consented, and + # today's period cannot be confirmed before it ends. + from datetime import timedelta + + period_day = ( + datetime.now(timezone.utc).date() - timedelta(days=1) + ).isoformat() + + # Open the consent window before the period, confirm it after — exactly + # what the runtime reconciler does across two days of hook fires. + from hermes_cli.observability.shared_metrics_sender import ( + reconcile_send_consent, + ) + from hermes_cli.sqlite_util import write_txn + + with store._connection() as connection: + with write_txn(connection): + reconcile_send_consent( + connection, + True, + now=datetime.now(timezone.utc) - timedelta(days=2), + ) + reconcile_send_consent(connection, True) real_install_id = str(uuid.uuid4()) packages = [] @@ -77,8 +102,8 @@ def main() -> int: "generated_at": datetime.now(timezone.utc).isoformat().replace( "+00:00", "Z" ), - "period_start": f"{today}T00:00:00Z", - "period_end": f"{today}T23:59:59Z", + "period_start": f"{period_day}T00:00:00Z", + "period_end": f"{period_day}T23:59:59Z", "resource": { "hermes_version": "e2e-test", "os_family": "macos", @@ -105,11 +130,11 @@ def main() -> int: """, ( package_id, - f"{today}T00:00:00Z", - f"{today}T23:59:59Z", + f"{period_day}T00:00:00Z", + f"{period_day}T23:59:59Z", json.dumps(payload), - f"{today}T0{index}:00:00Z", - f"{today}T0{index}:00:01Z", + f"{period_day}T0{index}:00:00Z", + f"{period_day}T0{index}:00:01Z", ), ) packages.append((package_id, metric_count)) diff --git a/tests/hermes_cli/test_setup_telemetry.py b/tests/hermes_cli/test_setup_telemetry.py index 4f66259eaa..2397524343 100644 --- a/tests/hermes_cli/test_setup_telemetry.py +++ b/tests/hermes_cli/test_setup_telemetry.py @@ -33,7 +33,10 @@ def test_disabling_collection_closes_the_send_consent_window(monkeypatch, tmp_pa package collected in between. """ from hermes_cli.observability.shared_metrics import SharedMetricsStore - from hermes_cli.observability.shared_metrics_sender import SEND_REVOKED_KEY + from hermes_cli.observability.shared_metrics_sender import ( + reconcile_send_consent, + ) + from hermes_cli.sqlite_util import write_txn store = SharedMetricsStore( database_path=tmp_path / "m.db", outbox_directory=tmp_path / "o" @@ -48,24 +51,21 @@ def test_disabling_collection_closes_the_send_consent_window(monkeypatch, tmp_pa "hermes_cli.setup.prompt_yes_no", lambda _question, default: False ) config = {"telemetry": {"shared_metrics": {"enabled": True, "send": True}}} - # Consent was granted earlier, so a window is already open — that is - # precisely the state whose closure must be recorded. - from hermes_cli.sqlite_util import write_txn - from hermes_cli.observability.shared_metrics_sender import opt_in_period - + # Consent was granted earlier, so a window is open — that is precisely + # the state whose closure must be recorded. with store._connection() as connection: with write_txn(connection): - opt_in_period(connection) + reconcile_send_consent(connection, True) setup_telemetry(config) assert config["telemetry"]["shared_metrics"]["enabled"] is False assert config["telemetry"]["shared_metrics"]["send"] is False with store._connection() as connection: - row = connection.execute( - "SELECT value FROM telemetry_state WHERE key = ?", (SEND_REVOKED_KEY,) - ).fetchone() - assert row is not None and row[0] == "1", ( + open_windows = connection.execute( + "SELECT COUNT(*) FROM send_consent_windows WHERE closed_at IS NULL" + ).fetchone()[0] + assert open_windows == 0, ( "disabling collection left the send consent window open" ) diff --git a/tests/hermes_cli/test_shared_metrics_consent_windows.py b/tests/hermes_cli/test_shared_metrics_consent_windows.py new file mode 100644 index 0000000000..83da1a4749 --- /dev/null +++ b/tests/hermes_cli/test_shared_metrics_consent_windows.py @@ -0,0 +1,221 @@ +"""Property tests for the consent-interval model. + +Ported from the /tmp validation harness that gated the redesign: every +scenario here is a defect that actually occurred (rounds 3-5) or a clock +adversary the day-stamp model could not survive. The v1 and v2 drafts of the +redesign each FAILED scenarios in this file before shipping — that is the +harness working, and why these run against the real store and the real +reconciler rather than a model of them. +""" + +from __future__ import annotations + +import json +from datetime import datetime, timedelta, timezone + +import pytest + +from hermes_cli.observability.shared_metrics import SharedMetricsStore +from hermes_cli.observability.shared_metrics_sender import ( + CONSENT_GATE_SQL, + reconcile_send_consent, +) +from hermes_cli.sqlite_util import write_txn + +T0 = datetime(2026, 8, 1, tzinfo=timezone.utc) + + +def ts(days=0, hours=0): + return (T0 + timedelta(days=days, hours=hours)).isoformat().replace( + "+00:00", "Z" + ) + + +def dt(days=0, hours=0): + return T0 + timedelta(days=days, hours=hours) + + +@pytest.fixture +def store(tmp_path): + return SharedMetricsStore( + database_path=tmp_path / "m.db", outbox_directory=tmp_path / "o" + ) + + +def _add(store, pid, start, end): + """Store a package the way the generator does: at period end.""" + with store._connection() as connection: + with write_txn(connection): + connection.execute( + "INSERT INTO package_outbox(package_id, period_start, period_end," + " payload_json, created_at, exported_at) VALUES (?, ?, ?, ?, ?, ?)", + (pid, start, end, json.dumps({"package_id": pid}), end, end), + ) + connection.execute( + """INSERT INTO consent_marks(name, stamp) VALUES ('data', ?) + ON CONFLICT(name) DO UPDATE SET stamp = MAX(stamp, excluded.stamp)""", + (end,), + ) + + +def _observe(store, send_enabled, when): + with store._connection() as connection: + with write_txn(connection): + reconcile_send_consent(connection, send_enabled, now=when) + + +def _eligible(store): + with store._connection() as connection: + return sorted( + row[0] + for row in connection.execute( + f"SELECT package_id FROM package_outbox WHERE {CONSENT_GATE_SQL}" + ) + ) + + +def _windows(store): + with store._connection() as connection: + return [ + tuple(row) + for row in connection.execute( + "SELECT opened_at, last_confirmed_at, closed_at" + " FROM send_consent_windows ORDER BY opened_at" + ) + ] + + +class TestRefusedWindowIsNeverReleased: + def test_on_off_on_with_realistic_interleaving(self, store): + """Rounds 3 and 5: the refused middle must never transmit, and + neither consented era may be lost.""" + _observe(store, True, dt(0)) + for n in range(5): + _add(store, f"d{n:02d}", ts(days=n), ts(days=n + 1)) + _observe(store, True, dt(days=n + 1)) + _observe(store, False, dt(5)) + for n in range(5, 10): + _add(store, f"d{n:02d}", ts(days=n), ts(days=n + 1)) + _observe(store, True, dt(10)) + for n in range(10, 15): + _add(store, f"d{n:02d}", ts(days=n), ts(days=n + 1)) + _observe(store, True, dt(days=n + 1)) + + eligible = _eligible(store) + assert not [p for p in eligible if 5 <= int(p[1:]) < 10], eligible + assert [f"d{n:02d}" for n in range(5)] == eligible[:5], ( + "pre-revocation consented backlog was destroyed" + ) + assert [f"d{n:02d}" for n in range(10, 15)] == eligible[5:], eligible + + def test_hand_edit_with_a_90_day_silent_gap(self, store): + """Round 5 D1, strongest form: NOTHING observes the off window. + + The close back-dates to the last confirmed moment, so the unobserved + gap is outside every window and fails closed. + """ + _observe(store, True, dt(0)) + _add(store, "consented", ts(0, 1), ts(0, 2)) + _observe(store, True, dt(0, 6)) + for n in range(1, 90, 10): + _add(store, f"REFUSED-d{n}", ts(days=n), ts(days=n, hours=1)) + _observe(store, False, dt(90)) # first observation: boot on day 90 + _observe(store, True, dt(91)) + _observe(store, True, dt(92)) + + eligible = _eligible(store) + assert not [p for p in eligible if p.startswith("REFUSED")], eligible + assert "consented" in eligible, ( + "the confirmed-morning package must survive the reconciliation" + ) + + +class TestClockAdversaries: + def test_rollback_at_re_enable_releases_nothing(self, store): + """Round 5 D2: the data mark clamps opens above existing packages.""" + _observe(store, True, dt(0)) + _observe(store, True, dt(5)) + _observe(store, False, dt(5)) + for n in range(1, 4): + _add(store, f"REFUSED-{n}", ts(days=5, hours=n), ts(days=5, hours=n + 1)) + _observe(store, True, dt(-12)) # 12-day rollback at re-enable + _observe(store, True, dt(-11)) + + during = [p for p in _eligible(store) if p.startswith("REFUSED")] + assert not during, f"rollback released refused packages: {during}" + + _observe(store, True, dt(20)) # clock recovers + _observe(store, True, dt(21)) + after = [p for p in _eligible(store) if p.startswith("REFUSED")] + assert not after, f"recovery released refused packages: {after}" + + def test_recovery_does_not_wedge_future_sending(self, store): + _observe(store, True, dt(0)) + _observe(store, False, dt(5)) + _observe(store, True, dt(-12)) + _observe(store, True, dt(20)) + _add(store, "post-recovery", ts(21), ts(21, 4)) + _observe(store, True, dt(22)) + assert "post-recovery" in _eligible(store) + + +class TestSubDayGranularity: + def test_intra_day_refusal_holds_back_the_whole_day_package(self, store): + """Round 5 D3: a day package spanning a refused stretch must wait.""" + _observe(store, True, dt(0)) + _observe(store, True, dt(10, 9)) + _observe(store, False, dt(10, 9)) + _observe(store, True, dt(10, 18)) + _observe(store, True, dt(11, 2)) + _add(store, "halfday", ts(10), ts(11)) + assert "halfday" not in _eligible(store) + + +class TestReconcilerProperties: + def test_idempotent_under_replay(self, store): + for _ in range(4): + _observe(store, True, dt(0)) + _observe(store, False, dt(2)) + for _ in range(5): + _observe(store, False, dt(3)) + _observe(store, True, dt(4)) + for _ in range(3): + _observe(store, True, dt(5)) + assert len(_windows(store)) == 2 + + def test_the_observation_mark_is_monotonic(self, store): + """A rolled-back clock must never lower the observation high-water. + + Every downstream guarantee leans on this: closes clamp to it via + last_confirmed_at, and opens clamp to max(obs, data). Found as a + surviving mutant (obs upsert rewritten from MAX to overwrite) — + the leak scenarios happen to be covered by the data mark whenever a + leakable package exists, but the property itself must hold on its + own, not by coincidence of the sibling mark. + """ + _observe(store, True, dt(5)) + _observe(store, True, dt(0)) # rollback + with store._connection() as connection: + stamp = connection.execute( + "SELECT stamp FROM consent_marks WHERE name = 'obs'" + ).fetchone()[0] + assert stamp == ts(5), f"obs mark moved backwards: {stamp}" + + def test_the_gate_is_read_only(self, store): + _observe(store, True, dt(0)) + before = _windows(store) + for _ in range(10): + _eligible(store) + assert _windows(store) == before + + def test_no_window_fails_closed(self, store): + _add(store, "orphan", ts(0), ts(1)) + assert _eligible(store) == [] + + def test_fresh_package_waits_one_heartbeat_then_releases(self, store): + """The documented latency cost of confirmation-based windows.""" + _observe(store, True, dt(0)) + _add(store, "fresh", ts(0, 1), ts(0, 2)) + assert _eligible(store) == [] + _observe(store, True, dt(0, 3)) + assert _eligible(store) == ["fresh"] diff --git a/tests/hermes_cli/test_shared_metrics_send_wiring.py b/tests/hermes_cli/test_shared_metrics_send_wiring.py index 3900847955..8ed7724b1e 100644 --- a/tests/hermes_cli/test_shared_metrics_send_wiring.py +++ b/tests/hermes_cli/test_shared_metrics_send_wiring.py @@ -209,13 +209,13 @@ class TestInteractivePathIsNotBlocked: runtime._join_send_thread(timeout=5) -class TestConsentRevocationWindow: - """The falling edge must close the window even with no pass running. +class TestConsentWindows: + """Consent reconciliation must work from the relay, in any order. - Round 3 recorded revocation inside the send loop, which cannot fire for - the dominant case: the user turns sending off while idle, so the relay - early-returns and no sender is ever built. Re-enabling then released - every package collected during the refused window. + Round 4's edge detector missed the idle-revocation path; round 5 found it + was also dead code whenever collection was off (handles_hook gated it). + These tests drive the relay entry points against the single reconciler + and assert on the interval table — the only consent state that exists. """ def _runtime(self, tmp_path): @@ -223,20 +223,19 @@ class TestConsentRevocationWindow: runtime.subscriber.store = RealBackedStore(tmp_path) return runtime - def _state(self, runtime, key): + def _windows(self, runtime): with runtime.subscriber.store._connection() as connection: - row = connection.execute( - "SELECT value FROM telemetry_state WHERE key = ?", (key,) - ).fetchone() - return row[0] if row else None + return [ + tuple(row) + for row in connection.execute( + "SELECT opened_at, last_confirmed_at, closed_at" + " FROM send_consent_windows ORDER BY opened_at" + ) + ] def test_revoking_while_idle_closes_the_window( self, monkeypatch, tmp_path, capture_sender ): - from hermes_cli.observability.shared_metrics_sender import ( - SEND_REVOKED_KEY, - ) - runtime = self._runtime(tmp_path) _set_config(monkeypatch, _config(enabled=True, send=True)) @@ -247,88 +246,101 @@ class TestConsentRevocationWindow: for _ in range(6): runtime._send_exported_packages() - assert self._state(runtime, SEND_REVOKED_KEY) == "1", ( - "revoking while no pass was running left the consent window open" + windows = self._windows(runtime) + assert windows and all(w[2] is not None for w in windows), ( + f"revoking while idle left a window open: {windows}" ) - def test_no_spurious_revocation_when_nothing_changes( + def test_replayed_observations_create_no_junk_windows( self, monkeypatch, tmp_path, capture_sender ): - """The detector must key on an EDGE, not on every disabled pass. - - A level trigger re-closes a window the user has since REOPENED: each - later disabled pass stamps revoked again, so the next enabled pass - advances the gate and silently drops packages the user did consent to. - Mutation-checked — an earlier version of this test used a - never-consented store, where record_revoked no-ops regardless, and so - could not tell an edge trigger from a level trigger. - """ - from hermes_cli.observability.shared_metrics_sender import ( - OPT_IN_PERIOD_KEY, - SEND_REVOKED_KEY, - ) - + """Reconciliation is idempotent — there is no edge to double-count.""" runtime = self._runtime(tmp_path) _set_config(monkeypatch, _config(enabled=True, send=True)) - runtime._send_exported_packages() - + for _ in range(4): + runtime._send_exported_packages() _set_config(monkeypatch, _config(enabled=True, send=False)) - runtime._send_exported_packages() - assert self._state(runtime, SEND_REVOKED_KEY) == "1" - - # User changes their mind and re-enables. + for _ in range(4): + runtime._send_exported_packages() _set_config(monkeypatch, _config(enabled=True, send=True)) - runtime._send_exported_packages() - assert self._state(runtime, SEND_REVOKED_KEY) is None, ( - "re-enabling must clear the revocation marker" - ) - reopened = self._state(runtime, OPT_IN_PERIOD_KEY) - - # Further ENABLED passes must not disturb the reopened window. for _ in range(4): runtime._send_exported_packages() - assert self._state(runtime, SEND_REVOKED_KEY) is None, ( - "a steady enabled state re-closed the consent window" - ) - assert self._state(runtime, OPT_IN_PERIOD_KEY) == reopened + assert len(self._windows(runtime)) == 2 - def test_a_never_consented_user_is_never_marked_revoked( + def test_a_never_consented_user_gets_no_window( self, monkeypatch, tmp_path, capture_sender ): - from hermes_cli.observability.shared_metrics_sender import ( - SEND_REVOKED_KEY, - ) - runtime = self._runtime(tmp_path) _set_config(monkeypatch, _config(enabled=True, send=False)) for _ in range(5): runtime._send_exported_packages() - assert self._state(runtime, SEND_REVOKED_KEY) is None + assert self._windows(runtime) == [] - def test_re_enabling_after_an_idle_revocation_starts_a_new_window( + def test_re_enabling_opens_a_new_window_after_the_refusal( self, monkeypatch, tmp_path, capture_sender ): - from hermes_cli.observability.shared_metrics_sender import ( - OPT_IN_PERIOD_KEY, - SEND_REVOKED_KEY, - ) - + """The refused gap must fall BETWEEN the two windows.""" runtime = self._runtime(tmp_path) _set_config(monkeypatch, _config(enabled=True, send=True)) runtime._send_exported_packages() - first_window = self._state(runtime, OPT_IN_PERIOD_KEY) - _set_config(monkeypatch, _config(enabled=True, send=False)) runtime._send_exported_packages() - assert self._state(runtime, SEND_REVOKED_KEY) == "1" - - # Re-enabling must not simply resume the original window. _set_config(monkeypatch, _config(enabled=True, send=True)) runtime._send_exported_packages() - assert first_window is not None + + windows = self._windows(runtime) + assert len(windows) == 2 + first, second = windows + assert first[2] is not None, "first window must be closed" + assert second[2] is None, "second window must be open" + assert second[0] >= first[2], ( + f"new window may not overlap the refused gap: {windows}" + ) + + def test_reconcile_runs_even_when_collection_is_disabled( + self, monkeypatch, tmp_path + ): + """Round-5 D1: enabled:false must not make consent handling dead code. + + The module-level once-per-process reconciler must close the window + regardless of handles_hook(). Drives the real observe_lifecycle gate + path: handles_hook is False throughout. + """ + from hermes_cli.observability.shared_metrics import SharedMetricsStore + from hermes_cli.observability.shared_metrics_sender import ( + reconcile_send_consent, + ) + from hermes_cli.sqlite_util import write_txn + + store = SharedMetricsStore( + database_path=tmp_path / "m.db", outbox_directory=tmp_path / "o" + ) + # A consent window is open from an earlier consented era. + with store._connection() as connection: + with write_txn(connection): + reconcile_send_consent(connection, True) + + monkeypatch.setattr( + "hermes_cli.observability.shared_metrics.SharedMetricsStore", + lambda *a, **k: store, + ) + _set_config(monkeypatch, _config(enabled=False, send=False)) + monkeypatch.setattr(mod, "_consent_reconcile_done", False) + + # The full lifecycle entry point, with collection OFF. + mod.observe_lifecycle("finish_task") + + with store._connection() as connection: + open_windows = connection.execute( + "SELECT COUNT(*) FROM send_consent_windows WHERE closed_at IS NULL" + ).fetchone()[0] + assert open_windows == 0, ( + "enabled:false made the consent reconciler unreachable (D1)" + ) + class TestFailureIsolation: diff --git a/tests/hermes_cli/test_shared_metrics_sender.py b/tests/hermes_cli/test_shared_metrics_sender.py index de489a36cb..6b59d5d69d 100644 --- a/tests/hermes_cli/test_shared_metrics_sender.py +++ b/tests/hermes_cli/test_shared_metrics_sender.py @@ -19,12 +19,11 @@ from hermes_cli.observability.shared_metrics_sender import ( MAX_ATTEMPTS, MAX_PACKAGES_PER_PASS, MAX_SEND_ATTEMPTS, - OPT_IN_PERIOD_KEY, REQUEST_TIMEOUT_SECONDS, SharedMetricsSender, - opt_in_period, - record_revoked, + reconcile_send_consent, ) +from hermes_cli.sqlite_util import write_txn INSTALL_ID = "12a73e97-4de9-4766-830d-9ca1192c0420" NOW = datetime(2026, 8, 26, 12, 0, tzinfo=timezone.utc) @@ -61,10 +60,45 @@ class FakeTransport: @pytest.fixture def store(tmp_path): - return SharedMetricsStore( + """A store with a broad consent window already open. + + Most tests exercise claiming/retry/transport, not the consent gate, and + the interval gate fails closed with no window. One window opened before + every test package and confirmed well past NOW keeps those tests about + what they are about. Gate tests clear it via _clear_consent. + """ + built = SharedMetricsStore( database_path=tmp_path / "metrics.sqlite3", outbox_directory=tmp_path / "outbox", ) + _grant_consent(built) + return built + + +def _grant_consent( + store, + opened=datetime(2026, 8, 20, tzinfo=timezone.utc), + confirmed_through=datetime(2026, 10, 1, tzinfo=timezone.utc), +): + """Open a consent window and heartbeat it forward, via the real writer.""" + with store._connection() as connection: + with write_txn(connection): + reconcile_send_consent(connection, True, now=opened) + reconcile_send_consent(connection, True, now=confirmed_through) + + +def _revoke_consent(store, at): + with store._connection() as connection: + with write_txn(connection): + reconcile_send_consent(connection, False, now=at) + + +def _clear_consent(store): + """Remove all consent state, for tests of the fail-closed default.""" + with store._connection() as connection: + with write_txn(connection): + connection.execute("DELETE FROM send_consent_windows") + connection.execute("DELETE FROM consent_marks") def _add_package(store, package_id, period_day, *, exported=True, install_id=INSTALL_ID): @@ -231,6 +265,9 @@ class TestContractResponses: class TestConsentGate: def test_packages_from_before_opt_in_are_never_sent(self, store): + # Consent opens on Aug 24; the "old" package's period predates it. + _clear_consent(store) + _grant_consent(store, opened=datetime(2026, 8, 24, tzinfo=timezone.utc)) _add_package(store, "old", "2026-08-20") _add_package(store, "new", "2026-08-26") transport = FakeTransport(FakeResponse(202)) @@ -245,19 +282,27 @@ class TestConsentGate: _sender(store, transport).send_pending() assert sorted(b["package_id"] for b in transport.bodies) == ["head", "tail"] - def test_opt_in_day_is_recorded_once_and_does_not_move(self, store): + def test_opt_in_is_immortalised_as_a_window_not_a_day(self, store): + """The window survives replayed observations without moving.""" with store._connection() as connection: - first = opt_in_period(connection, now=NOW) - later = opt_in_period(connection, now=NOW + timedelta(days=10)) - assert first == later == "2026-08-26" - - def test_opt_in_day_is_persisted(self, store): + rows = connection.execute( + "SELECT opened_at, closed_at FROM send_consent_windows" + ).fetchall() + assert len(rows) == 1 and rows[0][1] is None + _grant_consent(store) # replay: must not create a second window with store._connection() as connection: - opt_in_period(connection, now=NOW) - value = connection.execute( - "SELECT value FROM telemetry_state WHERE key = ?", (OPT_IN_PERIOD_KEY,) + count = connection.execute( + "SELECT COUNT(*) FROM send_consent_windows" ).fetchone()[0] - assert value == "2026-08-26" + assert count == 1 + + def test_no_consent_window_means_nothing_is_sent(self, store): + """The gate fails closed: absence of a window is absence of consent.""" + _clear_consent(store) + _add_package(store, "pkg-1", "2026-08-26") + transport = FakeTransport(FakeResponse(202)) + _sender(store, transport).send_pending() + assert transport.calls == [] def test_unexported_packages_are_skipped(self, store): _add_package(store, "pending-export", "2026-08-26", exported=False) @@ -266,32 +311,31 @@ class TestConsentGate: assert transport.calls == [] def test_revoking_then_re_enabling_never_releases_the_off_window(self, store): - """Regression: re-opt-in retroactively transmitted the refused window. + """The R3/R5 leak: re-opt-in must not release the refused interval. - opt_in_period was write-once, so packages collected while the user had - send: false still had period_start >= the ORIGINAL opt-in day. Turning - sending back on released the entire opted-out window — contradicting - the documented promise that `send: false` means no further packages - leave the machine. + Under the interval model the refused days fall BETWEEN two windows; + no later observation can place them inside one, so the property holds + for any number of on/off cycles — not just the single cycle the old + moving day-stamp was patched to survive. """ - _add_package(store, "consented", "2026-08-26") - with store._connection() as connection: - with __import__( - "hermes_cli.sqlite_util", fromlist=["write_txn"] - ).write_txn(connection): - opt_in_period(connection, now=NOW) + _clear_consent(store) + _grant_consent(store, opened=NOW - timedelta(days=2), confirmed_through=NOW) + _add_package(store, "consented", "2026-08-25") - # User turns sending off; packages keep being collected. - with store._connection() as connection: - with __import__( - "hermes_cli.sqlite_util", fromlist=["write_txn"] - ).write_txn(connection): - record_revoked(connection) + # User turns sending off; packages keep being collected for 3 days. + _revoke_consent(store, at=NOW) for day in ("2026-08-27", "2026-08-28", "2026-08-29"): _add_package(store, f"refused-{day}", day) - # User re-enables a few days later. + # User re-enables 5 days later; heartbeat confirms past the horizon. later = NOW + timedelta(days=5) + with store._connection() as connection: + with write_txn(connection): + reconcile_send_consent(connection, True, now=later) + reconcile_send_consent( + connection, True, now=later + timedelta(days=30) + ) + transport = FakeTransport(*[FakeResponse(202)] * 10) SharedMetricsSender( store, ENDPOINT, post=transport, sleep=lambda _s: None, now=lambda: later @@ -301,21 +345,30 @@ class TestConsentGate: assert not any("refused" in pid for pid in sent), ( f"transmitted packages collected while sending was off: {sent}" ) + # And the interval model's improvement over the day-stamp: the + # pre-revocation consented package is NOT collateral damage. + assert "consented" in sent, ( + "the consented backlog was destroyed by the revoke/re-enable cycle" + ) def test_a_package_from_after_re_enabling_is_sent(self, store): - """The revocation fix must not wedge sending off permanently.""" - with store._connection() as connection: - with __import__( - "hermes_cli.sqlite_util", fromlist=["write_txn"] - ).write_txn(connection): - opt_in_period(connection, now=NOW) - record_revoked(connection) + """The revocation handling must not wedge sending off permanently.""" + _clear_consent(store) + _grant_consent(store, opened=NOW - timedelta(days=2), confirmed_through=NOW) + _revoke_consent(store, at=NOW) later = NOW + timedelta(days=5) - _add_package(store, "after-re-optin", later.date().isoformat()) + with store._connection() as connection: + with write_txn(connection): + reconcile_send_consent(connection, True, now=later) + reconcile_send_consent( + connection, True, now=later + timedelta(days=10) + ) + _add_package(store, "after-re-optin", (later + timedelta(days=1)).date().isoformat()) transport = FakeTransport(FakeResponse(202)) SharedMetricsSender( - store, ENDPOINT, post=transport, sleep=lambda _s: None, now=lambda: later + store, ENDPOINT, post=transport, sleep=lambda _s: None, + now=lambda: later + timedelta(days=2), ).send_pending() assert len(transport.calls) == 1 diff --git a/tests/hermes_cli/test_shared_metrics_sender_e2e.py b/tests/hermes_cli/test_shared_metrics_sender_e2e.py index 552ae93552..9ff8bf9caf 100644 --- a/tests/hermes_cli/test_shared_metrics_sender_e2e.py +++ b/tests/hermes_cli/test_shared_metrics_sender_e2e.py @@ -77,10 +77,28 @@ def server(): @pytest.fixture def store(tmp_path): - return SharedMetricsStore( + built = SharedMetricsStore( database_path=tmp_path / "metrics.sqlite3", outbox_directory=tmp_path / "outbox", ) + # Open a consent window covering the fixture packages; the interval gate + # fails closed without one, and this file tests transport, not consent. + from datetime import datetime, timezone + + from hermes_cli.observability.shared_metrics_sender import ( + reconcile_send_consent, + ) + from hermes_cli.sqlite_util import write_txn + + with built._connection() as connection: + with write_txn(connection): + reconcile_send_consent( + connection, True, now=datetime(2026, 8, 20, tzinfo=timezone.utc) + ) + reconcile_send_consent( + connection, True, now=datetime(2026, 10, 1, tzinfo=timezone.utc) + ) + return built def _endpoint(server): From 67d152bc7e63048f6e92b82c84cdc21a84a1492d Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Thu, 27 Aug 2026 13:30:25 +1000 Subject: [PATCH 014/634] fix(telemetry): bound forward-clock damage to the consent horizon Sixth review - the first against the interval architecture - verdict: the architecture holds (idempotence, order-independence, 4-process concurrent-writer safety, rollback immunity, format consistency, and a 120-permutation order sweep all verified), with ONE high finding, which I had independently reproduced while the review ran: the FORWARD clock adversary was unhandled, and unlike every other failure mode in this subsystem it failed OPEN. The 'obs' mark is a MAX-upsert - monotonic in the leak direction. One glitched-forward sample (NTP flap reading 2099) while consented dragged last_confirmed_at to 2099; a later revoke stamped closed_at = 2099; the closed window then CONTAINED every refused period that followed. Both the reviewer and I reproduced refused packages becoming gate-eligible. The rollback twin was mutation-tested since round 5; nobody had asked whether the mirror image existed. Two clamps, each covering what the other cannot: - The obs mark advances at most MAX_OBS_ADVANCE_SECONDS (30 days) per call. Honest heartbeats never bind it; a machine off for months catches up in a few hook fires (fail-closed latency only); one insane sample moves the horizon by a bounded step that real time overtakes. - A close is MIN(last_confirmed_at, closing observation's raw stamp). Confirmed-time keeps unobserved gaps out of windows (v1's leak); the raw stamp lets an honest clock at revoke time pull a poisoned horizon back to the true revoke moment. A rolled-back clock at close time only closes earlier - fail-closed. Also from the review: - D2: the data-mark advance in the REAL package writer had no coverage (the harness re-implemented the insert; deleting the production line survived 314 tests). Now driven through create_and_export_package_if_due. - D3: the "don't create ~/.hermes/telemetry for fully-disabled users" skip was dead code - the store constructor creates the directory before the exists() check ran. The probe now checks the default path without constructing; verified empirically on a fresh HERMES_HOME. - Upgrade note in A.4: pre-interval backlog is never transmitted after upgrade (fail-closed; deliberate). New harness scenarios: forward-poison-then-revoke (the leak), and forward-poison-cannot-wedge (the cap). Mutation check: unclamping the close, removing the cap, and removing the real writer's data-mark advance each fail the suite. 273 tests pass; ruff and windows-footguns clean; staging E2E 202. --- docs/observability/relay-shared-metrics.md | 15 +++- .../observability/relay_shared_metrics.py | 12 ++- .../observability/shared_metrics_sender.py | 57 ++++++++++-- .../test_shared_metrics_consent_windows.py | 89 +++++++++++++++++++ .../test_shared_metrics_send_wiring.py | 12 ++- 5 files changed, 175 insertions(+), 10 deletions(-) diff --git a/docs/observability/relay-shared-metrics.md b/docs/observability/relay-shared-metrics.md index b538011675..cb3ea2197a 100644 --- a/docs/observability/relay-shared-metrics.md +++ b/docs/observability/relay-shared-metrics.md @@ -403,12 +403,23 @@ Turning sending off also **closes the consent window** — at the last moment consent was actually observed, not at the wall clock. Packages whose periods fall between one window and the next are never transmitted, even if sending is later re-enabled, and this holds for any number of on/off cycles, across -hand-edits with no process running, and under a clock that jumps backwards -(window opens are clamped above every timestamp already in the store). +hand-edits with no process running, and under a clock that jumps in either +direction (window opens are clamped above every timestamp already in the +store; observation marks advance by a bounded step per call, so one glitched +forward sample cannot drag the confirmation horizon years ahead; a close +never lands after the closing observation's own clock). Unlike the earlier single moving opt-in date, closing and reopening does NOT discard the still-undelivered backlog from a previous consented window — those packages stay inside their own interval and remain eligible. +One deliberate upgrade-path consequence: packages exported under the +pre-interval consent model (before `send_consent_windows` existed) predate +the first recorded window and are therefore never transmitted after an +upgrade. This is the fail-closed direction — re-importing the old moving +day-stamp to release them would re-import the semantics five review rounds +showed to be unsound — and it costs at most the undelivered backlog, never +collected data. + ### A.5 Retention - **Local:** unchanged — 30 days for successfully exported history, and pending diff --git a/hermes_cli/observability/relay_shared_metrics.py b/hermes_cli/observability/relay_shared_metrics.py index 978497945d..5a97c8a18d 100644 --- a/hermes_cli/observability/relay_shared_metrics.py +++ b/hermes_cli/observability/relay_shared_metrics.py @@ -1256,11 +1256,19 @@ def _reconcile_send_consent_once() -> None: reconcile_send_consent, ) from hermes_cli.sqlite_util import write_txn + from hermes_constants import get_hermes_home resolved = resolve_send_config(read_raw_config_readonly() or {}) - store = SharedMetricsStore() - if not resolved.send and not store.database_path.exists(): + # Probe for an existing store WITHOUT constructing one: the + # constructor creates the directory and schema as a side effect, + # which round 6 caught making this skip dead code — every + # fully-disabled user was getting a ~/.hermes/telemetry directory. + default_path = ( + get_hermes_home() / "telemetry" / "shared_metrics" / "metrics.sqlite3" + ) + if not resolved.send and not default_path.exists(): return + store = SharedMetricsStore() with store._connection() as connection: with write_txn(connection): reconcile_send_consent(connection, resolved.send) diff --git a/hermes_cli/observability/shared_metrics_sender.py b/hermes_cli/observability/shared_metrics_sender.py index 389611ed37..8a01a44790 100644 --- a/hermes_cli/observability/shared_metrics_sender.py +++ b/hermes_cli/observability/shared_metrics_sender.py @@ -100,6 +100,13 @@ def _isoformat(value: datetime) -> str: return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") +def _parse_stamp(value: str) -> datetime: + """Parse a stamp this module itself wrote (Z-suffixed ISO-8601, UTC).""" + return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone( + timezone.utc + ) + + @dataclass class SendOutcome: """What one pass did. Returned for tests and diagnostics.""" @@ -164,6 +171,18 @@ def _retry_after_seconds(value: str | None, default: int) -> int: return default +#: Maximum distance one reconcile call can advance the 'obs' mark. Honest +#: heartbeats arrive hours apart at most, so the cap never binds in normal +#: operation; a machine legitimately off for months catches up in a few +#: hook fires (fail-closed latency only). What it bounds is FORWARD clock +#: poison: without it, a single glitched sample (NTP flap reading 2099) +#: permanently drags the mark — and with it every window open and every +#: confirmation horizon — decades ahead, which round 6 reproduced as a +#: refused-data leak. Capped, one insane sample moves the mark at most +#: this far, and real time overtakes it again. +MAX_OBS_ADVANCE_SECONDS = 30 * 24 * 3600 + + def reconcile_send_consent( connection: sqlite3.Connection, send_enabled: bool, @@ -182,9 +201,15 @@ def reconcile_send_consent( Timestamp discipline (each rule is load-bearing; see the validation harness in tests/hermes_cli/test_shared_metrics_consent_windows.py): - - The 'obs' mark advances to every observation stamp, monotonically. - An open window's ``last_confirmed_at`` follows it: consent is asserted - only for time that was actually observed. + - The 'obs' mark advances to every observation stamp, monotonically — + but by at most ``MAX_OBS_ADVANCE_SECONDS`` per call. Unbounded, the + mark is monotonic in the LEAK direction: one glitched-forward sample + would drag ``last_confirmed_at`` decades ahead, a later close would + stamp that horizon, and the closed window would contain every future + refused period (reproduced in round 6). Bounded, a poisoned sample + costs at most one cap's width, and real time overtakes it. + An open window's ``last_confirmed_at`` follows the mark: consent is + asserted only for time that was actually observed. - A close is stamped at ``last_confirmed_at`` — never "now" — so an unobserved gap (hand-edited config, machine off for 90 days) is never inside a window and fails closed. @@ -193,6 +218,16 @@ def reconcile_send_consent( make the new window adjacent to the previous close. """ stamp = _isoformat(now or _utc_now()) + raw_stamp = stamp # pre-cap observation time, used to clamp closes + previous_obs = connection.execute( + "SELECT stamp FROM consent_marks WHERE name = 'obs'" + ).fetchone() + if previous_obs is not None: + ceiling = _isoformat( + _parse_stamp(str(previous_obs[0])) + + timedelta(seconds=MAX_OBS_ADVANCE_SECONDS) + ) + stamp = min(stamp, ceiling) connection.execute( """ INSERT INTO consent_marks(name, stamp) VALUES ('obs', ?) @@ -226,10 +261,22 @@ def reconcile_send_consent( (obs, open_row[0]), ) elif open_row is not None: + # Close at the last CONFIRMED moment, but never after the closing + # observation's own raw stamp. The two clamps serve different + # adversaries and both are load-bearing: + # - min with last_confirmed_at: an unobserved gap (machine off, + # hand-edited config) is never asserted as consented (v1's leak). + # - min with the RAW stamp (pre-cap, pre-MAX): if last_confirmed_at + # was poisoned by a glitched-forward sample, an honest clock at + # revoke time pulls the close back to the true revoke moment, so + # the refused era that follows falls OUTSIDE the closed window + # (round 6's D1 leak). A rolled-back clock at close time only + # closes EARLIER — fail-closed. connection.execute( - "UPDATE send_consent_windows SET closed_at = last_confirmed_at" + "UPDATE send_consent_windows" + " SET closed_at = MIN(last_confirmed_at, ?)" " WHERE rowid = ?", - (open_row[0],), + (raw_stamp, open_row[0]), ) diff --git a/tests/hermes_cli/test_shared_metrics_consent_windows.py b/tests/hermes_cli/test_shared_metrics_consent_windows.py index 83da1a4749..66b58d5dd3 100644 --- a/tests/hermes_cli/test_shared_metrics_consent_windows.py +++ b/tests/hermes_cli/test_shared_metrics_consent_windows.py @@ -131,6 +131,62 @@ class TestRefusedWindowIsNeverReleased: class TestClockAdversaries: + def test_forward_poison_then_revoke_releases_nothing(self, store): + """Round 6 D1: one glitched-forward sample must not defeat a close. + + Unfixed, the poisoned obs mark dragged last_confirmed_at to 2099, a + later revoke stamped closed_at = 2099, and the closed window then + CONTAINED every refused period that followed — all 8 refused + packages became eligible. The close now clamps to the closing + observation's own raw stamp, so an honest clock at revoke time pulls + the window back to the true revoke moment. + """ + _observe(store, True, dt(0)) + _observe(store, True, datetime(2099, 1, 1, tzinfo=timezone.utc)) + _observe(store, False, dt(1)) # honest clock at revoke + for n in range(2, 10): + _add(store, f"REFUSED-{n}", ts(days=n), ts(days=n, hours=2)) + + leaked = [p for p in _eligible(store) if p.startswith("REFUSED")] + assert not leaked, f"poisoned horizon released refused data: {leaked}" + + def test_forward_poison_cannot_wedge_consent_forever(self, store): + """The obs-advance cap bounds the damage of one insane sample. + + Uncapped, a 2099 sample would clamp every future window open at + 2099, suppressing consented data for decades (fail-closed but + permanent). Capped, the mark moves at most MAX_OBS_ADVANCE_SECONDS + past its previous value, so honest time overtakes it. + """ + from hermes_cli.observability.shared_metrics_sender import ( + MAX_OBS_ADVANCE_SECONDS, + ) + + _observe(store, True, dt(0)) + _observe(store, True, datetime(2099, 1, 1, tzinfo=timezone.utc)) + with store._connection() as connection: + stamp = connection.execute( + "SELECT stamp FROM consent_marks WHERE name = 'obs'" + ).fetchone()[0] + ceiling = ts(days=MAX_OBS_ADVANCE_SECONDS // 86_400) + assert stamp <= ceiling, ( + f"one glitched sample advanced the mark unboundedly: {stamp}" + ) + + # Consented data from shortly after the cap horizon still flows once + # honest observations catch the marks up. + horizon_days = MAX_OBS_ADVANCE_SECONDS // 86_400 + _add( + store, + "post-glitch", + ts(days=horizon_days + 1), + ts(days=horizon_days + 1, hours=4), + ) + _observe(store, True, dt(days=horizon_days + 2)) + assert "post-glitch" in _eligible(store), ( + "consent wedged after a forward glitch" + ) + def test_rollback_at_re_enable_releases_nothing(self, store): """Round 5 D2: the data mark clamps opens above existing packages.""" _observe(store, True, dt(0)) @@ -201,6 +257,39 @@ class TestReconcilerProperties: ).fetchone()[0] assert stamp == ts(5), f"obs mark moved backwards: {stamp}" + def test_the_real_package_writer_advances_the_data_mark(self, store): + """Round 6 D2: the harness's _add re-implements the data-mark insert, + so deleting the advance from the REAL writer survived 314 tests. + This drives the production exporter instead. + """ + from datetime import date, timedelta as _td + + yesterday = (date.today() - _td(days=1)).isoformat() + with store._connection() as connection: + with write_txn(connection): + connection.execute( + "INSERT INTO counter_aggregates(" + " period_start, metric_name, hermes_version, os_family," + " architecture, install_method, dimensions_json, value," + " packaged_value" + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + yesterday, "hermes.client.active", "0.0.0-test", + "macos", "arm64", "git", "{}", 1, 0, + ), + ) + + exported = store.create_and_export_package_if_due() + assert exported, "the generator was expected to export yesterday's period" + + with store._connection() as connection: + row = connection.execute( + "SELECT stamp FROM consent_marks WHERE name = 'data'" + ).fetchone() + assert row is not None and row[0] >= yesterday, ( + "the production package writer did not advance the data mark" + ) + def test_the_gate_is_read_only(self, store): _observe(store, True, dt(0)) before = _windows(store) diff --git a/tests/hermes_cli/test_shared_metrics_send_wiring.py b/tests/hermes_cli/test_shared_metrics_send_wiring.py index 8ed7724b1e..29f572c697 100644 --- a/tests/hermes_cli/test_shared_metrics_send_wiring.py +++ b/tests/hermes_cli/test_shared_metrics_send_wiring.py @@ -315,8 +315,18 @@ class TestConsentWindows: ) from hermes_cli.sqlite_util import write_txn + # Lay the store out exactly as production does, under a redirected + # HERMES_HOME: the boot reconciler probes the default path (without + # constructing the store — the constructor creates directories), so + # the probe and the store must agree the way they do in production. + home = tmp_path / "home" + monkeypatch.setattr( + "hermes_constants.get_hermes_home", lambda: home + ) + root = home / "telemetry" / "shared_metrics" store = SharedMetricsStore( - database_path=tmp_path / "m.db", outbox_directory=tmp_path / "o" + database_path=root / "metrics.sqlite3", + outbox_directory=root / "outbox", ) # A consent window is open from an earlier consented era. with store._connection() as connection: From 60addb16e28eec4923c1e891bfbeaf3d2f0d7c8d Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Thu, 27 Aug 2026 15:04:02 +1000 Subject: [PATCH 015/634] fix(telemetry): fence send authority on a per-claim token Responds to the independent PR review (andrexibiza). Both P1s were checked against current HEAD rather than taken on authority - the review was written against 613849c190, before the interval-model consent replacement landed. P1-1 (same-UTC-day revoke/re-enable releases refused data): already fixed by the interval model. The reviewer's exact reproduction - opt in 06:00, revoke 12:00, package collected 18:00, re-enable 20:00 same day - was re-run at HEAD: the off-window package stays local, and a full-day aggregate straddling the revocation boundary also stays local (period containment, timestamp precision). The consent-windows harness already pins both. The reviewer's related ask that consent-ledger persistence failures fail closed also holds structurally now: reconciliation derives state rather than recording transitions, so a lost write means a shorter confirmed horizon - less is released, never more. P1-2 (lease has no owner) was VALID at head. Reproduced exactly as described: A claims, is suspended past the 300s lease, B reclaims and POSTs, A resumes and POSTs again - and the ingest key is minute- prefixed, so the duplicate lands as a DISTINCT stored object, making this worse than a benign idempotent overwrite. Fix: every claim now mints a claim_token (additive nullable column, schema version unchanged). Ownership is revalidated immediately before every external POST, and every settlement, rejection, and backoff write is compare-and-set on (package_id, claim_token, pending). A lapsed claimant that resumes yields without transmitting, and its stale backoff cannot move next_attempt_at under the live claim's lease. Two deterministic regressions ship with it: expiry -> reclaim -> resume (the reviewer's schedule), and the subtler stale-backoff-clobber case. Honest scope, documented on _send_one: delivery remains at-least-once. The token closes the claim->POST gap; a suspension landing mid-POST (bytes already on the wire) is not client-revocable. The residual duplicate is byte-identical content; collapsing it fully needs package_id-keyed dedupe at the ingest service. 275 tests pass; ruff + footguns clean; staging E2E 202. --- hermes_cli/observability/shared_metrics.py | 6 + .../observability/shared_metrics_sender.py | 104 ++++++++++++++++-- .../hermes_cli/test_shared_metrics_sender.py | 81 ++++++++++++++ 3 files changed, 182 insertions(+), 9 deletions(-) diff --git a/hermes_cli/observability/shared_metrics.py b/hermes_cli/observability/shared_metrics.py index 32dfc4fff6..ddf570b6f9 100644 --- a/hermes_cli/observability/shared_metrics.py +++ b/hermes_cli/observability/shared_metrics.py @@ -378,6 +378,12 @@ class SharedMetricsStore: # Only the ~36-byte id is stored: the body is recomputed from # payload_json, whose serialisation is deterministic. ("sent_install_id", "TEXT"), + # NULL until first claimed; rewritten on every claim. Settlement + # and the pre-POST revalidation are compare-and-set on this, so a + # claimant whose lease lapsed loses authority the moment another + # process reclaims (PR-review finding: without it, a suspended + # sender resuming after a reclaim double-POSTs the package). + ("claim_token", "TEXT"), ): if column not in existing: connection.execute( diff --git a/hermes_cli/observability/shared_metrics_sender.py b/hermes_cli/observability/shared_metrics_sender.py index 8a01a44790..1c06acc686 100644 --- a/hermes_cli/observability/shared_metrics_sender.py +++ b/hermes_cli/observability/shared_metrics_sender.py @@ -33,6 +33,7 @@ import sqlite3 import time import urllib.error import urllib.request +import uuid from dataclasses import dataclass from datetime import datetime, timedelta, timezone @@ -396,24 +397,31 @@ class SharedMetricsSender: # caller to continue rather than stop. return {"package_id": package_id, "skip": True} + token = str(uuid.uuid4()) connection.execute( """ UPDATE package_outbox SET send_state = 'pending', send_attempts = send_attempts + 1, - next_attempt_at = ? + next_attempt_at = ?, + claim_token = ? WHERE package_id = ? """, # Lease INTO THE FUTURE: selection requires # next_attempt_at <= now, so no other process can take # this row while it is in flight. Success or a real # backoff overwrites it; if this process dies, it expires. - (_isoformat(lease_until), package_id), + # The token is this claim's identity: a reclaim after + # expiry mints a new one, and every later write by THIS + # claimant is compare-and-set against it, so a lapsed + # claimant that resumes cannot settle or transmit. + (_isoformat(lease_until), token, package_id), ) return { "package_id": package_id, "payload_json": str(row[1]), "derived": str(derived), + "claim_token": token, "skip": False, } @@ -479,12 +487,25 @@ class SharedMetricsSender: payload = substitute_install_id(json.loads(payload_json), derived) return json.dumps(payload, indent=2, sort_keys=True).encode("utf-8") - def _mark(self, package_id: str, *, only_if_pending: bool = True, **columns) -> None: + def _mark( + self, + package_id: str, + *, + only_if_pending: bool = True, + token: str | None = None, + **columns, + ) -> None: """Write send state for one package. Guarded on send_state so a pass whose lease lapsed cannot resurrect a row another process has already finished: without this, a slow sender could overwrite 'sent' back to 'pending' and cause a re-send. + + When ``token`` is given, the write is additionally compare-and-set on + claim_token: it lands only if THIS claim is still the current one. A + claimant that lapsed and was superseded writes zero rows — its + settlement, backoff, and error strings all silently lose to the + newer claim's, which is the correct outcome. """ assignments = ", ".join(f"{name} = ?" for name in columns) predicate = ( @@ -492,15 +513,48 @@ class SharedMetricsSender: if only_if_pending else "" ) + params: list = [*columns.values(), package_id] + if token is not None: + predicate += " AND claim_token = ?" + params.append(token) with self._store._connection() as connection: with write_txn(connection): connection.execute( f"UPDATE package_outbox SET {assignments} " f"WHERE package_id = ?{predicate}", - (*columns.values(), package_id), + params, ) - def _defer(self, package_id: str, delay_seconds: int, reason: str) -> None: + def _still_owns(self, package_id: str, token: str | None) -> bool: + """Return whether this pass's claim on the row is still current.""" + if token is None: + # Defensive: a package dict without a token (not produced by + # _claim_next today) gets no authority rather than unlimited. + return False + try: + with self._store._connection() as connection: + row = connection.execute( + "SELECT 1 FROM package_outbox" + " WHERE package_id = ? AND claim_token = ?" + " AND (send_state IS NULL OR send_state = 'pending')", + (package_id, token), + ).fetchone() + return row is not None + except Exception: + # If the check itself fails, do not transmit on stale authority. + logger.warning( + "Unable to verify shared-metrics claim ownership", exc_info=True + ) + return False + + def _defer( + self, + package_id: str, + delay_seconds: int, + reason: str, + *, + token: str | None = None, + ) -> None: # Defence in depth: no current caller can pass a non-positive delay # (Retry-After is already clamped to [1, 86400] when parsed, and every # other call site passes a positive constant), so this clamp is @@ -512,6 +566,7 @@ class SharedMetricsSender: retry_at = self._now().timestamp() + delay self._mark( package_id, + token=token, send_state="pending", next_attempt_at=_isoformat( datetime.fromtimestamp(retry_at, tz=timezone.utc) @@ -520,11 +575,33 @@ class SharedMetricsSender: ) def _send_one(self, package: dict) -> str: - """Try one package. Returns 'sent', 'rejected', or 'deferred'.""" + """Try one package. Returns 'sent', 'rejected', or 'deferred'. + + Delivery is at-least-once. The pre-POST ownership check plus the + token-fenced writes close the claim->POST and settle-after-reclaim + gaps, but a suspension landing MID-POST (bytes already on the wire + when the machine sleeps) can still duplicate: no client-side check + can revoke a request in flight. The body is byte-identical across + retries by construction, so the residual duplicate is exactly one + redundant copy of identical content; collapsing it fully would need + package_id-keyed dedupe at the ingest service. + """ package_id = package["package_id"] + token = package.get("claim_token") body = self._body(package["payload_json"], package["derived"]) for attempt in range(1, self._max_attempts + 1): + # Revalidate ownership immediately before the external POST. The + # claim can lapse between claiming and here — a suspended laptop, + # a GC pause, a long gzip — and another process may have + # reclaimed and transmitted. Without this check the resumed + # claimant POSTs a duplicate; the ingest key is minute-prefixed, + # so duplicates become distinct stored objects, not overwrites. + if not self._still_owns(package_id, token): + logger.info( + "Shared-metrics claim on %s superseded; yielding", package_id + ) + return "deferred" try: response = self._post( self._endpoint, body, timeout=REQUEST_TIMEOUT_SECONDS @@ -532,7 +609,9 @@ class SharedMetricsSender: except Exception as exc: # transport failure: offline, DNS, TLS reason = f"{type(exc).__name__}: {exc}" if attempt >= self._max_attempts: - self._defer(package_id, _FAILURE_BACKOFF_SECONDS, reason) + self._defer( + package_id, _FAILURE_BACKOFF_SECONDS, reason, token=token + ) return "deferred" self._sleep(self._backoff(attempt)) continue @@ -540,6 +619,7 @@ class SharedMetricsSender: if response.status == 202: self._mark( package_id, + token=token, send_state="sent", sent_at=_isoformat(self._now()), last_error=None, @@ -560,6 +640,7 @@ class SharedMetricsSender: ) self._mark( package_id, + token=token, send_state="rejected", last_error=f"HTTP {response.status}: {response.body[:400]}", ) @@ -570,17 +651,22 @@ class SharedMetricsSender: package_id, _retry_after_seconds(response.retry_after, _FAILURE_BACKOFF_SECONDS), "rate limited", + token=token, ) return "deferred" # 5xx and anything unexpected: retryable. reason = f"HTTP {response.status}" if attempt >= self._max_attempts: - self._defer(package_id, _FAILURE_BACKOFF_SECONDS, reason) + self._defer( + package_id, _FAILURE_BACKOFF_SECONDS, reason, token=token + ) return "deferred" self._sleep(self._backoff(attempt)) - self._defer(package_id, _FAILURE_BACKOFF_SECONDS, "attempts exhausted") + self._defer( + package_id, _FAILURE_BACKOFF_SECONDS, "attempts exhausted", token=token + ) return "deferred" @staticmethod diff --git a/tests/hermes_cli/test_shared_metrics_sender.py b/tests/hermes_cli/test_shared_metrics_sender.py index 6b59d5d69d..0080224ade 100644 --- a/tests/hermes_cli/test_shared_metrics_sender.py +++ b/tests/hermes_cli/test_shared_metrics_sender.py @@ -649,6 +649,87 @@ class TestClaimingAndBounds: f"{len(attempts)} requests burned on one doomed package" ) + def test_a_lapsed_claimant_resuming_after_reclaim_cannot_double_post( + self, store + ): + """PR-review P1: expiry -> reclaim -> old claimant resumes. + + A claims, then is suspended (laptop lid) BEFORE its POST. The lease + expires; B reclaims and POSTs; A wakes and proceeds. The pre-POST + ownership check must make A yield without transmitting. + + Scope note: the check closes the claim->POST gap. A suspension that + lands mid-POST (bytes already leaving) is not client-fixable — that + residual needs server-side dedupe and is documented on _send_one. + """ + _add_package(store, "pkg-1", "2026-08-26") + + posts = [] + + def post_a(endpoint, payload, *, timeout): + posts.append("A") + return FakeResponse(202) + + def post_b(endpoint, payload, *, timeout): + posts.append("B") + return FakeResponse(202) + + sender_a = SharedMetricsSender( + store, ENDPOINT, post=post_a, sleep=lambda _s: None, now=lambda: NOW + ) + # A claims, then the process is suspended before _send_one runs. + claimed_a = sender_a._claim_next(NOW, set()) + assert claimed_a is not None and not claimed_a["skip"] + + # 400s later (past the 300s lease) B claims and completes the send. + later = NOW + timedelta(seconds=400) + sender_b = SharedMetricsSender( + store, ENDPOINT, post=post_b, sleep=lambda _s: None, now=lambda: later + ) + outcome_b = sender_b.send_pending() + assert outcome_b.sent == 1 + + # A resumes exactly where it left off. + result_a = sender_a._send_one(claimed_a) + + row = _row(store, "pkg-1") + assert posts == ["B"], ( + f"a lapsed claimant transmitted after reclaim: {posts}" + ) + assert result_a == "deferred" + assert row["send_state"] == "sent", "B's settlement must stand" + + def test_a_lapsed_claimants_backoff_cannot_clobber_the_new_claim(self, store): + """The token must fence DEFERS too, not just the 202 settlement. + + A's transport fails after B has reclaimed; A's backoff write must + not move next_attempt_at under B's live lease. + """ + _add_package(store, "pkg-1", "2026-08-26") + sender_a = SharedMetricsSender( + store, ENDPOINT, + post=FakeTransport(OSError("net"), OSError("net"), OSError("net")), + sleep=lambda _s: None, now=lambda: NOW, + ) + claimed_a = sender_a._claim_next(NOW, set()) + assert claimed_a is not None and not claimed_a["skip"] + + later = NOW + timedelta(seconds=400) + sender_b = SharedMetricsSender( + store, ENDPOINT, post=FakeTransport(), + sleep=lambda _s: None, now=lambda: later, + ) + claimed_b = sender_b._claim_next(later, set()) + assert claimed_b is not None and not claimed_b["skip"] + lease_b = _row(store, "pkg-1")["next_attempt_at"] + + # A's exhausted retries try to write a 15-minute backoff. + result = sender_a._send_one(claimed_a) + assert result == "deferred" + assert _row(store, "pkg-1")["next_attempt_at"] == lease_b, ( + "a lapsed claimant's backoff overwrote the live claim's lease" + ) + def test_an_expired_lease_is_reclaimed(self, store): """A process killed mid-pass must not strand its packages.""" _add_package(store, "pkg-1", "2026-08-26") From 4bdabb21ed68da01012a4491f621ba02b45e3ba4 Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Thu, 27 Aug 2026 15:21:30 +1000 Subject: [PATCH 016/634] fix(telemetry): renew the claim atomically before every POST Seventh review found the claim-token fix incomplete, and its reproduction is exact: the pre-POST check was READ-ONLY. A claimant whose lease expired while suspended still passes it when it wakes BEFORE anyone reclaims - its token is still in the row - and then a second process legitimately reclaims while the first one's POST is in flight. Both send. Reproduced at 60addb16e2: posts ['B', 'A'], both reporting 'sent'. This is the check-to-POST expiry race, not the documented mid-POST residual: A's lease was already dead before its authority check passed. The check is now an atomic RENEWAL (single CAS UPDATE): it requires the token to match, the row to be pending, AND the current lease to be unexpired, and only then extends next_attempt_at a fresh lease into the future. rowcount == 1 is the only grant. A claimant that wakes past its own lease fails the unexpired condition and yields even though its token was never replaced - expiry alone means another process may claim at any moment, so waking stale is disqualifying regardless of whether anyone has taken the row yet. The renewed lease (300s) covers the POST (30s timeout) with margin, and renewal runs before every retry, not just the first attempt. Regressions: the reviewer's exact ordering (expired wake before any reclaim -> zero POSTs, row stays claimable), plus a healthy-claimant renewal test. Mutation-checked: dropping the lease-unexpired condition or the token condition each fails the suite. The at-least-once scope note on _send_one stands: a suspension landing mid-POST remains client-unfixable; the fixable window is now closed on both sides (before the check, and between check and POST). 277 tests pass; ruff + footguns clean; staging E2E 202. --- .../observability/shared_metrics_sender.py | 73 +++++++++++++------ .../hermes_cli/test_shared_metrics_sender.py | 47 ++++++++++++ 2 files changed, 99 insertions(+), 21 deletions(-) diff --git a/hermes_cli/observability/shared_metrics_sender.py b/hermes_cli/observability/shared_metrics_sender.py index 1c06acc686..8e6f81c4b8 100644 --- a/hermes_cli/observability/shared_metrics_sender.py +++ b/hermes_cli/observability/shared_metrics_sender.py @@ -525,25 +525,53 @@ class SharedMetricsSender: params, ) - def _still_owns(self, package_id: str, token: str | None) -> bool: - """Return whether this pass's claim on the row is still current.""" + def _renew_claim(self, package_id: str, token: str | None) -> bool: + """Atomically re-assert ownership and extend the lease. CAS, one row. + + A read-only ownership check is not enough: a claimant whose lease + expired while suspended can pass the check (its token is still in + the row if no one reclaimed yet) and then POST while another process + legitimately reclaims — the check-to-POST expiry race a seventh + review reproduced. Renewal closes it by requiring, in ONE statement: + + - the token still matches (nobody reclaimed), AND + - the current lease is UNEXPIRED (this claimant is not stale), AND + - the row is still pending, + + and only then pushing next_attempt_at a fresh lease into the future, + so the upcoming POST (30s timeout, well under the 300s lease) runs + entirely inside renewed authority. rowcount == 1 is the only grant. + A claimant that wakes past its own lease fails the unexpired + condition and yields even though its token was never replaced. + """ if token is None: - # Defensive: a package dict without a token (not produced by - # _claim_next today) gets no authority rather than unlimited. return False try: + now = self._now() + lease_until = now + timedelta(seconds=_CLAIM_LEASE_SECONDS) with self._store._connection() as connection: - row = connection.execute( - "SELECT 1 FROM package_outbox" - " WHERE package_id = ? AND claim_token = ?" - " AND (send_state IS NULL OR send_state = 'pending')", - (package_id, token), - ).fetchone() - return row is not None + with write_txn(connection): + cursor = connection.execute( + """ + UPDATE package_outbox + SET next_attempt_at = ? + WHERE package_id = ? + AND claim_token = ? + AND (send_state IS NULL OR send_state = 'pending') + AND next_attempt_at > ? + """, + ( + _isoformat(lease_until), + package_id, + token, + _isoformat(now), + ), + ) + return cursor.rowcount == 1 except Exception: - # If the check itself fails, do not transmit on stale authority. + # If renewal itself fails, do not transmit on unproven authority. logger.warning( - "Unable to verify shared-metrics claim ownership", exc_info=True + "Unable to renew shared-metrics claim", exc_info=True ) return False @@ -591,15 +619,18 @@ class SharedMetricsSender: body = self._body(package["payload_json"], package["derived"]) for attempt in range(1, self._max_attempts + 1): - # Revalidate ownership immediately before the external POST. The - # claim can lapse between claiming and here — a suspended laptop, - # a GC pause, a long gzip — and another process may have - # reclaimed and transmitted. Without this check the resumed - # claimant POSTs a duplicate; the ingest key is minute-prefixed, - # so duplicates become distinct stored objects, not overwrites. - if not self._still_owns(package_id, token): + # Atomically renew the claim before EVERY external POST. The + # renewal is compare-and-set on (token, pending, lease unexpired) + # and extends the lease past the request, so a suspended-then- + # resumed claimant whose lease lapsed yields here even if nobody + # has reclaimed yet — a read-only ownership check passed in that + # state and still double-sent (check-to-POST expiry race). The + # ingest key is minute-prefixed, so duplicates become distinct + # stored objects, not overwrites. + if not self._renew_claim(package_id, token): logger.info( - "Shared-metrics claim on %s superseded; yielding", package_id + "Shared-metrics claim on %s superseded or expired; yielding", + package_id, ) return "deferred" try: diff --git a/tests/hermes_cli/test_shared_metrics_sender.py b/tests/hermes_cli/test_shared_metrics_sender.py index 0080224ade..63fe022475 100644 --- a/tests/hermes_cli/test_shared_metrics_sender.py +++ b/tests/hermes_cli/test_shared_metrics_sender.py @@ -649,6 +649,53 @@ class TestClaimingAndBounds: f"{len(attempts)} requests burned on one doomed package" ) + def test_a_lapsed_claimant_yields_even_before_anyone_reclaims(self, store): + """Seventh review: the check-to-POST expiry race. + + A claims, sleeps past its own lease, and wakes BEFORE any other + process reclaims. Its token is still in the row, so a read-only + ownership check passes — and then B reclaims while A's POST is in + flight: both send. The pre-POST renewal must instead REJECT a + claimant whose lease already expired, whether or not anyone has + reclaimed yet, because expiry alone means another process may claim + at any moment. + """ + _add_package(store, "pkg-1", "2026-08-26") + + posts = [] + sender_a = SharedMetricsSender( + store, ENDPOINT, + post=lambda e, p, *, timeout: (posts.append("A"), FakeResponse(202))[1], + sleep=lambda _s: None, + now=lambda: clock["t"], + ) + clock = {"t": NOW} + claimed = sender_a._claim_next(NOW, set()) + assert claimed is not None and not claimed["skip"] + + # Suspended past the 300s lease; wakes with the row NOT yet reclaimed. + clock["t"] = NOW + timedelta(seconds=400) + result = sender_a._send_one(claimed) + + assert posts == [], ( + "a claimant with an expired lease transmitted before renewal" + ) + assert result == "deferred" + # The row must remain claimable by the next process. + row = _row(store, "pkg-1") + assert row["send_state"] == "pending" + + def test_renewal_extends_the_lease_across_the_post(self, store): + """A healthy in-lease claimant renews and its POST is covered.""" + _add_package(store, "pkg-1", "2026-08-26") + sender = _sender(store, FakeTransport(FakeResponse(202))) + claimed = sender._claim_next(NOW, set()) + assert claimed is not None + lease_before = _row(store, "pkg-1")["next_attempt_at"] + + assert sender._renew_claim("pkg-1", claimed["claim_token"]) is True + assert _row(store, "pkg-1")["next_attempt_at"] >= lease_before + def test_a_lapsed_claimant_resuming_after_reclaim_cannot_double_post( self, store ): From ecf327c87277aa3d71addaa7f0191a943c8b35c7 Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Thu, 27 Aug 2026 16:36:11 +1000 Subject: [PATCH 017/634] test(telemetry): make the renewal-extension regression falsifiable Eighth review round (the first against the atomic-renewal fix) verdict: the production code holds - CAS exclusivity across real processes, lease-extension schedules, clock skew both directions, renew-per-attempt under 5xx backoff, defer accounting, and the author's mutants all verified - but one shipped regression test could not fail against the property it is named for. test_renewal_extends_the_lease_across_the_post asserted next_attempt_at >= lease_before under a frozen clock. A renewal that matches the row but never extends the lease (M4: SET next_attempt_at = next_attempt_at) satisfies >= trivially, and that mutant double-POSTs: the un-extended lease expires mid-POST and a second process reclaims. The reviewer demonstrated M4 surviving the whole suite while producing a real duplicate send in a two-process schedule. The test now renews 100s into the lease from an advanced clock and requires the deadline to move strictly forward to exactly renewal-clock + 300s. Verified: M4 now fails this test (61 others unaffected); clean HEAD passes all 62. No production code change. 277 tests; ruff + footguns clean. --- .../hermes_cli/test_shared_metrics_sender.py | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/hermes_cli/test_shared_metrics_sender.py b/tests/hermes_cli/test_shared_metrics_sender.py index 63fe022475..c6a7455fe2 100644 --- a/tests/hermes_cli/test_shared_metrics_sender.py +++ b/tests/hermes_cli/test_shared_metrics_sender.py @@ -686,15 +686,40 @@ class TestClaimingAndBounds: assert row["send_state"] == "pending" def test_renewal_extends_the_lease_across_the_post(self, store): - """A healthy in-lease claimant renews and its POST is covered.""" + """A healthy in-lease claimant renews and its POST is covered. + + Round-8 review: the original assertion was `>=` under a frozen + clock, which a renewal that matches the row but never extends the + lease also satisfies — the exact mutant that double-POSTs (the + un-extended lease expires mid-POST and a second process reclaims). + The renewal must move the deadline STRICTLY forward to now + lease, + so renew from a later clock and require the exact new deadline. + """ _add_package(store, "pkg-1", "2026-08-26") - sender = _sender(store, FakeTransport(FakeResponse(202))) + clock = {"t": NOW} + sender = SharedMetricsSender( + store, + ENDPOINT, + post=lambda e, p, *, timeout: FakeResponse(202), + sleep=lambda _s: None, + now=lambda: clock["t"], + ) claimed = sender._claim_next(NOW, set()) assert claimed is not None lease_before = _row(store, "pkg-1")["next_attempt_at"] + # 100s into the (300s) lease: still healthy, renews mid-flight. + clock["t"] = NOW + timedelta(seconds=100) assert sender._renew_claim("pkg-1", claimed["claim_token"]) is True - assert _row(store, "pkg-1")["next_attempt_at"] >= lease_before + lease_after = _row(store, "pkg-1")["next_attempt_at"] + assert lease_after > lease_before, ( + "renewal granted authority without extending the lease" + ) + # And not just 'later': the full fresh lease from the renewal clock. + expected = (NOW + timedelta(seconds=100 + 300)).strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + assert lease_after == expected def test_a_lapsed_claimant_resuming_after_reclaim_cannot_double_post( self, store From 4caeb02735cdbbe302d0f28bf1a0ef14aa1af3e8 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Thu, 27 Aug 2026 19:12:58 -0300 Subject: [PATCH 018/634] fix(models): key the pricing cache on auth state, not just the base URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `fetch_models_with_pricing` checked its cache above the point where the Authorization header is built, and keyed that cache on the base URL alone. Whichever read of a given base URL landed first in a process therefore answered every later read, whatever key it passed — a non-empty result is held for the life of the process. That is wrong for any endpoint whose answer depends on who is asking. The Nous inference gateway filters `GET /v1/models` by the caller's org model policy, so an anonymous read landing first makes a later authenticated read return the full, unfiltered catalog without a request going out. Separate the URL root from the cache key and fold auth state into the latter. Only whether a key was supplied participates, never its value, so no secret reaches the key. `credits_tracker` peeked into the private `_pricing_cache` and duplicated the key shape to do it; it now calls `peek_cached_pricing`, which owns both the /v1-suffix normalization and the preference for the authenticated catalog. Co-Authored-By: Claude Opus 5 (1M context) --- agent/credits_tracker.py | 14 +- hermes_cli/models.py | 41 +++++- .../hermes_cli/test_pricing_cache_auth_key.py | 127 ++++++++++++++++++ 3 files changed, 171 insertions(+), 11 deletions(-) create mode 100644 tests/hermes_cli/test_pricing_cache_auth_key.py diff --git a/agent/credits_tracker.py b/agent/credits_tracker.py index 39c74ea58b..2d0873c563 100644 --- a/agent/credits_tracker.py +++ b/agent/credits_tracker.py @@ -252,15 +252,13 @@ def is_free_tier_model(model: str, base_url: str = "") -> bool: if not base_url: return False try: - from hermes_cli.models import _is_model_free, _pricing_cache + from hermes_cli.models import _is_model_free, peek_cached_pricing - # Mirror get_pricing_for_provider's key normalization: the agent's - # Nous base_url is /v1-suffixed (https://inference-api.nousresearch.com/v1) - # but the picker keys _pricing_cache on the pre-/v1 root. - key = base_url.rstrip("/") - if key.endswith("/v1"): - key = key[:-3].rstrip("/") - pricing = _pricing_cache.get(key) + # The agent's Nous base_url is /v1-suffixed + # (https://inference-api.nousresearch.com/v1) but the catalog fetchers + # key on the pre-/v1 root, and on auth state besides; peek_cached_pricing + # owns both details. + pricing = peek_cached_pricing(base_url) if not pricing: return False return _is_model_free(model, pricing) diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 389cbaed72..cc2ba908b8 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2255,6 +2255,39 @@ def _cache_catalog( return result +# A governed endpoint answers an authenticated read with a policy-filtered +# catalog and an anonymous read with the full one, so auth state is part of the +# cache identity. NUL cannot appear in a URL, so the suffix cannot collide with +# a base URL that happens to end this way. +_PRICING_AUTH_KEY_SUFFIX = "\x00auth" + + +def _pricing_cache_key(url_root: str, api_key: str | None) -> str: + """The ``_pricing_cache`` key for a read of *url_root*. + + Only *whether* a key was supplied participates — never its value, so no + secret reaches the cache key. + """ + return url_root + _PRICING_AUTH_KEY_SUFFIX if api_key else url_root + + +def peek_cached_pricing(base_url: str) -> dict[str, dict[str, Any]]: + """Pricing already cached for *base_url*, or ``{}``. Never fetches. + + Accepts a ``/v1``-suffixed URL as well as the pre-``/v1`` root the + catalog fetchers key on. Prefers the authenticated catalog, which is the + one scoped to the caller's org. + """ + root = (base_url or "").rstrip("/") + if root.endswith("/v1"): + root = root[:-3].rstrip("/") + for key in (root + _PRICING_AUTH_KEY_SUFFIX, root): + cached = _pricing_cache.get(key) + if cached: + return cached + return {} + + def _format_price_per_mtok(per_token_str: str) -> str: """Convert a per-token price string to a human-friendly $/Mtok string. @@ -2391,7 +2424,8 @@ def fetch_models_with_pricing( ) -> dict[str, dict[str, Any]]: """Fetch ``/v1/models`` and return ``{model_id: {prompt, completion, ...}}``. - Results are cached per *base_url* so repeated calls are free. + Results are cached per *base_url* and per auth state, so repeated calls + are free and an authenticated read never answers an anonymous one. Works with any OpenRouter-compatible endpoint (OpenRouter, Nous Portal). When *include_sale_original* is true (Nous Portal only) and the gateway @@ -2402,13 +2436,14 @@ def fetch_models_with_pricing( ``{prompt, completion}`` shape even if a response happens to nest ``original``. """ - cache_key = (base_url or "").rstrip("/") + url_root = (base_url or "").rstrip("/") + cache_key = _pricing_cache_key(url_root, api_key) if not force_refresh: cached = _cached_catalog(cache_key) if cached is not None: return cached - url = cache_key + "/v1/models" + url = url_root + "/v1/models" headers: dict[str, str] = { "Accept": "application/json", "User-Agent": _HERMES_USER_AGENT, diff --git a/tests/hermes_cli/test_pricing_cache_auth_key.py b/tests/hermes_cli/test_pricing_cache_auth_key.py new file mode 100644 index 0000000000..a1ae120365 --- /dev/null +++ b/tests/hermes_cli/test_pricing_cache_auth_key.py @@ -0,0 +1,127 @@ +"""``_pricing_cache`` keys on auth state, not just the base URL. + +A governed endpoint (Nous ``/v1/models`` filtered by an org's model policy) +answers an authenticated read with a narrower catalog than an anonymous one. +Keyed on the base URL alone, whichever read landed first in a process answered +every later one — so an authenticated caller could be handed the full, +unfiltered catalog without a request going out. +""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock + +import pytest + +import hermes_cli.models as models_mod +from hermes_cli.models import fetch_models_with_pricing, peek_cached_pricing + +BASE = "https://inference-api.example.com" + +# What the endpoint serves anonymously vs. to a policy-restricted caller. +_FULL = ["vendor/allowed", "vendor/blocked"] +_FILTERED = ["vendor/allowed"] + + +@pytest.fixture(autouse=True) +def _clear_pricing_cache(): + models_mod._pricing_cache.clear() + models_mod._pricing_cache_retry_after.clear() + yield + models_mod._pricing_cache.clear() + models_mod._pricing_cache_retry_after.clear() + + +@pytest.fixture +def catalog(monkeypatch): + """Serve the filtered catalog to an authenticated read, the full one to an + anonymous read, and record every request.""" + requests: list[str | None] = [] + + def _fake_urlopen(req, timeout=8.0): + auth = req.get_header("Authorization") + requests.append(auth) + ids = _FILTERED if auth else _FULL + payload = { + "data": [ + {"id": mid, "pricing": {"prompt": "0.000002", "completion": "0.00001"}} + for mid in ids + ] + } + resp = MagicMock() + resp.read.return_value = json.dumps(payload).encode() + resp.__enter__ = lambda self: self + resp.__exit__ = lambda *a: False + return resp + + monkeypatch.setattr(models_mod, "_urlopen_model_catalog_request", _fake_urlopen) + return requests + + +def test_authenticated_read_is_not_answered_by_an_anonymous_one(catalog): + """The bug: an anonymous read landing first must not answer the next + authenticated read out of cache.""" + anon = fetch_models_with_pricing(api_key="", base_url=BASE) + authed = fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + + assert sorted(anon) == sorted(_FULL) + assert sorted(authed) == sorted(_FILTERED) + assert len(catalog) == 2, "the authenticated read must reach the network" + assert catalog[0] is None and catalog[1] == "Bearer sk-test" + + +def test_anonymous_read_is_not_answered_by_an_authenticated_one(catalog): + """And the reverse direction, so neither entry can shadow the other.""" + authed = fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + anon = fetch_models_with_pricing(api_key="", base_url=BASE) + + assert sorted(authed) == sorted(_FILTERED) + assert sorted(anon) == sorted(_FULL) + assert len(catalog) == 2 + + +@pytest.mark.parametrize("api_key", ["sk-test", ""]) +def test_repeated_read_still_hits_the_cache(catalog, api_key): + """Widening the key must not cost the caching it was there for.""" + first = fetch_models_with_pricing(api_key=api_key, base_url=BASE) + second = fetch_models_with_pricing(api_key=api_key, base_url=BASE) + + assert first == second + assert len(catalog) == 1, "second read should be served from cache" + + +def test_force_refresh_replaces_only_its_own_entry(catalog): + """A forced authenticated re-read must leave the anonymous entry intact.""" + fetch_models_with_pricing(api_key="", base_url=BASE) + fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + fetch_models_with_pricing(api_key="sk-test", base_url=BASE, force_refresh=True) + + assert len(catalog) == 3 + anon = fetch_models_with_pricing(api_key="", base_url=BASE) + assert sorted(anon) == sorted(_FULL) + assert len(catalog) == 3, "the anonymous entry should have survived" + + +class TestPeekCachedPricing: + def test_returns_empty_when_nothing_cached(self): + assert peek_cached_pricing(BASE) == {} + + def test_accepts_a_v1_suffixed_url(self, catalog): + """The agent holds a /v1-suffixed base URL; the fetchers key on the root.""" + fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + assert sorted(peek_cached_pricing(BASE + "/v1")) == sorted(_FILTERED) + + def test_prefers_the_authenticated_catalog(self, catalog): + """It is the one scoped to the caller's org.""" + fetch_models_with_pricing(api_key="", base_url=BASE) + fetch_models_with_pricing(api_key="sk-test", base_url=BASE) + assert sorted(peek_cached_pricing(BASE)) == sorted(_FILTERED) + + def test_falls_back_to_the_anonymous_catalog(self, catalog): + fetch_models_with_pricing(api_key="", base_url=BASE) + assert sorted(peek_cached_pricing(BASE)) == sorted(_FULL) + + def test_never_fetches(self, catalog): + peek_cached_pricing(BASE) + assert catalog == [] From c248d5356c38b8bef1e4c03c241b8832bb1f47b8 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Thu, 27 Aug 2026 19:13:13 -0300 Subject: [PATCH 019/634] feat(nous): read the org model policy and expose it as a list filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Nous team admin can restrict which models and which serving providers their org may use. The inference gateway applies that policy to `GET /v1/models`, omitting blocked rows with no marker field, so the keys of an authenticated catalog read are the reachable set. Add the two pieces the pickers need: `nous_policy_present()` reads the `policy_present` claim off the OAuth access token, which costs no request. `/api/oauth/account` does not carry the claim, so this reads the token rather than going through `get_nous_portal_account_info`. The claim is tri-state — absent means an older mint, which is not the same as "no policy" and must not be reported as one. `nous_policy_allowed_ids()` turns the authenticated pricing response into that set, reusing the cache entry a caller asking for pricing already populates rather than issuing a second round trip. It returns None — "leave the list alone" — for an org with no policy, for an anonymous read whose catalog is unfiltered, and for an empty read, each of which would otherwise narrow a list on evidence that cannot support it. `restrict_to_nous_policy()` applies the set while preserving the caller's order, and keeps a `:free` sibling whose base model is reachable. The gateway admits a row when any of its requestable ids passes and treats anything unknown as a keep, on the grounds that over-listing costs a 403 from the authoritative gate while hiding a row the gate would serve is unrecoverable from the client. This mirrors that. Co-Authored-By: Claude Opus 5 (1M context) --- hermes_cli/models.py | 68 +++++++++ hermes_cli/nous_account.py | 32 ++++ tests/hermes_cli/test_nous_policy_filter.py | 159 ++++++++++++++++++++ 3 files changed, 259 insertions(+) create mode 100644 tests/hermes_cli/test_nous_policy_filter.py diff --git a/hermes_cli/models.py b/hermes_cli/models.py index cc2ba908b8..c1f7d580f1 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2604,6 +2604,74 @@ def _resolve_nous_pricing_credentials() -> tuple[str, str]: return (api_key, base_url) +def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str]]: + """The Nous model ids the caller's org may reach, or ``None`` to not filter. + + The gateway filters ``GET /v1/models`` by the org's model policy for an + authenticated read, omitting blocked rows with no marker field, so the keys + of the authenticated pricing response are the reachable set. This reuses + that response rather than issuing a second round trip. + + Returns ``None`` — meaning "leave the caller's list alone" — in three cases, + each of which would otherwise narrow a list on evidence that cannot support + it: + + * the org carries no policy, or the token is too old to say (see + :func:`~hermes_cli.nous_account.nous_policy_present`). Filtering an + unrestricted org's list buys nothing and risks dropping a model the + Portal recommends before the gateway catalog lists it. + * credential resolution failed, so the read is anonymous and therefore + unfiltered. A full catalog must not be mistaken for a policy-filtered one. + * the read came back empty, which is a fetch failure rather than an org + that may reach nothing. + """ + try: + from hermes_cli.nous_account import nous_policy_present + + if nous_policy_present() is not True: + return None + except Exception: + return None + + api_key, base_url = _resolve_nous_pricing_credentials() + if not api_key or not base_url: + return None + + # Same arguments as get_pricing_for_provider's nous branch, so a caller + # that also asks for pricing shares this cache entry instead of paying for + # a second request. + pricing = fetch_models_with_pricing( + api_key=api_key, + base_url=base_url, + force_refresh=force_refresh, + include_sale_original=True, + ) + return set(pricing) or None + + +def restrict_to_nous_policy( + model_ids: list[str], allowed: Optional[set[str]] +) -> list[str]: + """*model_ids* narrowed to *allowed*, preserving the caller's order. + + A ``None`` or empty *allowed* leaves the list untouched — see + :func:`nous_policy_allowed_ids` for when that happens. + + A ``:free`` sibling is kept when its base model is reachable. The gateway + admits a row when any of its requestable ids passes, and treats anything + unknown as a keep on the grounds that over-listing costs a 403 from the + authoritative gate while hiding a row the gate would serve is unrecoverable + from the client. This mirrors that. + """ + if not allowed: + return list(model_ids) + return [ + mid + for mid in model_ids + if mid in allowed or mid.split(":", 1)[0] in allowed + ] + + def get_pricing_for_provider(provider: str, *, force_refresh: bool = False) -> dict[str, dict[str, str]]: """Return live pricing for providers that support it (openrouter, nous, ai-gateway, novita).""" normalized = normalize_provider(provider) diff --git a/hermes_cli/nous_account.py b/hermes_cli/nous_account.py index 654487e684..4c3bf0a51f 100644 --- a/hermes_cli/nous_account.py +++ b/hermes_cli/nous_account.py @@ -99,6 +99,7 @@ class NousPortalAccountInfo: subscription: Optional[NousPortalSubscriptionInfo] = None paid_service_access: Optional[bool] = None paid_service_access_info: Optional[NousPaidServiceAccessInfo] = None + policy_present: Optional[bool] = None tool_access: Optional[NousToolAccessInfo] = None raw_claims: Optional[dict[str, Any]] = None raw_account: Optional[dict[str, Any]] = None @@ -396,6 +397,36 @@ def get_nous_portal_account_info( ) +def nous_policy_present() -> Optional[bool]: + """Whether the caller's org carries a restrictive model/provider policy. + + Read from the ``policy_present`` claim on the Nous OAuth access token, so + this costs no request. ``/api/oauth/account`` does not carry the claim, + which is why this reads the token directly rather than going through + :func:`get_nous_portal_account_info`. + + ``None`` means unknown — an older mint, an unreadable token, or a + non-boolean claim. Unknown is NOT "no policy": callers must not report the + absence of the claim as the absence of a restriction. + + The claim is stamped at mint time, so it goes stale until the next token + refresh. + """ + try: + from hermes_cli.auth import get_provider_auth_state, _decode_jwt_claims + + state = get_provider_auth_state("nous") or {} + access_token = state.get("access_token") + if not isinstance(access_token, str) or not access_token.strip(): + return None + claims = _decode_jwt_claims(access_token) + if not claims: + return None + return _coerce_bool(claims.get("policy_present")) + except Exception: + return None + + def _fresh_account_info( *, state: dict[str, Any], @@ -642,6 +673,7 @@ def _info_from_valid_jwt( expires_at=datetime.fromtimestamp(exp, tz=timezone.utc), paid_service_access=paid_access, paid_service_access_info=access_info, + policy_present=_coerce_bool(claims.get("policy_present")), tool_access=_tool_access_from_value(claims.get("tool_access")), raw_claims=dict(claims), ) diff --git a/tests/hermes_cli/test_nous_policy_filter.py b/tests/hermes_cli/test_nous_policy_filter.py new file mode 100644 index 0000000000..a225d560dc --- /dev/null +++ b/tests/hermes_cli/test_nous_policy_filter.py @@ -0,0 +1,159 @@ +"""Narrowing the Nous model lists to an org's policy. + +The inference gateway omits policy-blocked rows from an authenticated +``GET /v1/models`` with no marker field, so the keys of the authenticated +catalog read are the reachable set. These helpers turn that into a filter the +pickers can apply without a second round trip, and — just as importantly — +decline to filter when the evidence cannot support it. +""" + +from __future__ import annotations + +import base64 +import json + +import pytest + +import hermes_cli.models as models_mod +import hermes_cli.nous_account as account_mod +from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy +from hermes_cli.nous_account import nous_policy_present + + +def _jwt(claims: dict) -> str: + def seg(obj): + raw = json.dumps(obj).encode() + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + return f"{seg({'alg': 'RS256'})}.{seg(claims)}.sig" + + +class TestRestrictToNousPolicy: + def test_none_leaves_the_list_untouched(self): + ids = ["a/one", "b/two"] + assert restrict_to_nous_policy(ids, None) == ids + + def test_empty_set_leaves_the_list_untouched(self): + """Empty is a failed read, not an org that may reach nothing.""" + ids = ["a/one", "b/two"] + assert restrict_to_nous_policy(ids, set()) == ids + + def test_drops_ids_outside_the_policy(self): + assert restrict_to_nous_policy( + ["a/one", "b/two", "c/three"], {"a/one", "c/three"} + ) == ["a/one", "c/three"] + + def test_preserves_curated_order(self): + """The pickers show a curated order deliberately; filtering must not + reorder it into the catalog's alphabetical order.""" + curated = ["z/last", "a/first", "m/middle"] + allowed = {"a/first", "m/middle", "z/last"} + assert restrict_to_nous_policy(curated, allowed) == curated + + def test_keeps_a_free_sibling_when_its_base_is_reachable(self): + """Portal free recommendations are ``:free`` ids; the gateway admits a + row when any of its requestable ids passes.""" + assert restrict_to_nous_policy(["vendor/model:free"], {"vendor/model"}) == [ + "vendor/model:free" + ] + + def test_keeps_a_free_id_listed_in_its_own_right(self): + assert restrict_to_nous_policy( + ["vendor/model:free"], {"vendor/model:free"} + ) == ["vendor/model:free"] + + def test_drops_a_free_sibling_whose_base_is_blocked(self): + assert restrict_to_nous_policy(["vendor/model:free"], {"other/model"}) == [] + + +class TestNousPolicyAllowedIds: + @pytest.fixture(autouse=True) + def _clear_cache(self): + models_mod._pricing_cache.clear() + models_mod._pricing_cache_retry_after.clear() + yield + models_mod._pricing_cache.clear() + models_mod._pricing_cache_retry_after.clear() + + def _patch(self, monkeypatch, *, policy_present, api_key="sk-test", pricing=None): + calls = [] + monkeypatch.setattr( + account_mod, "nous_policy_present", lambda: policy_present + ) + monkeypatch.setattr( + models_mod, + "_resolve_nous_pricing_credentials", + lambda: (api_key, "https://inference.example.com"), + ) + + def _fake_fetch(**kwargs): + calls.append(kwargs) + return pricing if pricing is not None else {} + + monkeypatch.setattr(models_mod, "fetch_models_with_pricing", _fake_fetch) + return calls + + def test_returns_the_authenticated_catalog_keys(self, monkeypatch): + calls = self._patch( + monkeypatch, + policy_present=True, + pricing={"a/one": {}, "b/two": {}}, + ) + assert nous_policy_allowed_ids() == {"a/one", "b/two"} + assert len(calls) == 1 + assert calls[0]["api_key"] == "sk-test" + + def test_declines_to_filter_an_unrestricted_org(self, monkeypatch): + calls = self._patch(monkeypatch, policy_present=False, pricing={"a/one": {}}) + assert nous_policy_allowed_ids() is None + assert calls == [], "an unrestricted org should not pay for the read" + + def test_declines_to_filter_when_the_claim_is_unknown(self, monkeypatch): + """Absent is an older mint, not an unrestricted org.""" + calls = self._patch(monkeypatch, policy_present=None, pricing={"a/one": {}}) + assert nous_policy_allowed_ids() is None + assert calls == [] + + def test_declines_to_filter_on_an_anonymous_read(self, monkeypatch): + """An anonymous read returns the full catalog; treating it as the + policy-filtered set would silently widen the list to everything.""" + self._patch(monkeypatch, policy_present=True, api_key="", pricing={"a/one": {}}) + assert nous_policy_allowed_ids() is None + + def test_declines_to_filter_on_an_empty_read(self, monkeypatch): + """A failed fetch must not read as an org that may reach nothing.""" + self._patch(monkeypatch, policy_present=True, pricing={}) + assert nous_policy_allowed_ids() is None + + +class TestNousPolicyPresent: + def _patch_token(self, monkeypatch, token): + import hermes_cli.auth as auth_mod + + monkeypatch.setattr( + auth_mod, + "get_provider_auth_state", + lambda _p: {"access_token": token} if token is not None else {}, + ) + + @pytest.mark.parametrize("claim,expected", [(True, True), (False, False)]) + def test_reads_the_claim(self, monkeypatch, claim, expected): + self._patch_token(monkeypatch, _jwt({"policy_present": claim})) + assert nous_policy_present() is expected + + def test_absent_claim_is_unknown_not_false(self, monkeypatch): + self._patch_token(monkeypatch, _jwt({"org_id": "org_1"})) + assert nous_policy_present() is None + + def test_non_boolean_claim_is_unknown(self, monkeypatch): + """The gateway refuses to read a corrupt claim as "no policy".""" + self._patch_token(monkeypatch, _jwt({"policy_present": "yes"})) + assert nous_policy_present() is None + + def test_no_token_is_unknown(self, monkeypatch): + self._patch_token(monkeypatch, None) + assert nous_policy_present() is None + + def test_undecodable_token_is_unknown(self, monkeypatch): + self._patch_token(monkeypatch, "not-a-jwt") + assert nous_policy_present() is None From b1ea9196f7ed87dc4d919b528c3b006bdfd174c3 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Thu, 27 Aug 2026 19:13:53 -0300 Subject: [PATCH 020/634] fix(nous): narrow every model list to the org's policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four surfaces list Nous models, and none of them was filtered. All four seed from the docs-hosted curated manifest and union the Portal's `recommended-models` endpoint; neither source is authenticated, so org policy had no effect on the model a user picks — which is the model they then use. The Portal endpoint compounds it, serving one globally CDN-cached payload for the whole platform, invalidated only by admin pricing edits and never by a policy change, so it can put a hidden model straight back into a list. Narrow all four against the authenticated catalog: - `_login_nous`, which chooses the model the session starts on - `_model_flow_nous`, the `hermes model` picker - `list_authenticated_providers`, the `/model` picker - `/api/model/recommended-default`, dashboard onboarding The list stays curated and curated-ordered — the policy set only ever subtracts. Replacing a list with the catalog's keys would swap a curated agentic list for a large alphabetical dump of vendor-prefixed models, which is the regression the picker's nous branch already exists to avoid. The `/model` picker's filter sits outside the try that wraps the Portal union, so a Portal outage still yields a policy-filtered curated list. `_login_nous` and `_model_flow_nous` also narrow their unavailable lists, so a policy-hidden model is not offered as a free-tier upsell either. For an org with no policy — the common case — the filter is a no-op and every list is what it was. Co-Authored-By: Claude Opus 5 (1M context) --- hermes_cli/auth.py | 9 + hermes_cli/model_setup_flows.py | 9 + hermes_cli/model_switch.py | 13 ++ hermes_cli/web_server.py | 7 + tests/hermes_cli/test_nous_policy_surfaces.py | 162 ++++++++++++++++++ 5 files changed, 200 insertions(+) create mode 100644 tests/hermes_cli/test_nous_policy_surfaces.py diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index 8d3a3d13ae..f97f91bca5 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -9377,6 +9377,7 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: from hermes_cli.models import ( get_curated_nous_model_ids, get_pricing_for_provider, check_nous_free_tier, partition_nous_models_by_tier, + nous_policy_allowed_ids, restrict_to_nous_policy, union_with_portal_free_recommendations, union_with_portal_paid_recommendations, ) @@ -9427,6 +9428,14 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: model_ids, pricing = union_with_portal_paid_recommendations( model_ids, pricing, _portal_for_recs, ) + # The curated list and the Portal's recommendations are both + # unauthenticated, so neither knows what the org may reach. + # Narrow both lists to the policy before they are shown. + _policy_allowed = nous_policy_allowed_ids() + model_ids = restrict_to_nous_policy(model_ids, _policy_allowed) + unavailable_models = restrict_to_nous_policy( + unavailable_models, _policy_allowed, + ) _portal = auth_state.get("portal_base_url", "") if model_ids: print(f"Showing {len(model_ids)} curated models — use \"Enter custom model name\" for others.") diff --git a/hermes_cli/model_setup_flows.py b/hermes_cli/model_setup_flows.py index fbb6f35c94..90075912a1 100644 --- a/hermes_cli/model_setup_flows.py +++ b/hermes_cli/model_setup_flows.py @@ -559,6 +559,15 @@ def _model_flow_nous(config, current_model="", args=None): model_ids, pricing, _nous_portal_url, ) + # The curated list and the Portal's recommendations are both + # unauthenticated, so neither knows what the org may reach. Narrow both + # lists to the policy before they are shown. + from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy + + _policy_allowed = nous_policy_allowed_ids() + model_ids = restrict_to_nous_policy(model_ids, _policy_allowed) + unavailable_models = restrict_to_nous_policy(unavailable_models, _policy_allowed) + if not model_ids and not unavailable_models: print("No models available for Nous Portal after filtering.") return diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index d4689df874..cb7f9cf6df 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -3096,6 +3096,19 @@ def list_authenticated_providers( # curated list alone (still correct, just may lag newly # launched models, exactly like an offline CLI run). pass + # Both the curated list and the Portal's recommendations are + # unauthenticated, so neither knows what the org may reach. Narrow + # to the policy outside the try, so a failed recommendation fetch + # still yields a filtered curated list. + try: + from hermes_cli.models import ( + nous_policy_allowed_ids as _nous_policy, + restrict_to_nous_policy as _nous_restrict, + ) + + model_ids = _nous_restrict(model_ids, _nous_policy()) + except Exception: + pass else: # Unified pathway — see Section 1 rationale. Fall back to the # curated dict (with models.dev merge for preferred providers) diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 73b2014e8e..41577df7d4 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -7485,8 +7485,10 @@ def get_recommended_default_model(provider: str = ""): get_curated_nous_model_ids, get_pricing_for_provider, check_nous_free_tier, + nous_policy_allowed_ids, partition_nous_models_by_tier, pick_silent_default_model, + restrict_to_nous_policy, union_with_portal_free_recommendations, union_with_portal_paid_recommendations, ) @@ -7515,6 +7517,11 @@ def get_recommended_default_model(provider: str = ""): model_ids, pricing, portal_url ) + # Neither the curated list nor the Portal's recommendations know + # what the org may reach, and this endpoint picks the model a user + # lands on without choosing it. + model_ids = restrict_to_nous_policy(model_ids, nous_policy_allowed_ids()) + model = pick_silent_default_model(model_ids, provider="nous") return {"provider": "nous", "model": model, "free_tier": bool(free_tier)} except Exception: diff --git a/tests/hermes_cli/test_nous_policy_surfaces.py b/tests/hermes_cli/test_nous_policy_surfaces.py new file mode 100644 index 0000000000..c8f4cc9d18 --- /dev/null +++ b/tests/hermes_cli/test_nous_policy_surfaces.py @@ -0,0 +1,162 @@ +"""Every Nous model list is narrowed to the org's policy before it is shown. + +Four surfaces build a Nous list from the curated manifest unioned with the +Portal's ``recommended-models`` endpoint. Neither source is authenticated, so +without this filter an org's hidden model is offered to the user and then +refused at request time with ``model_blocked_by_org_policy``. +""" + +from __future__ import annotations + +import argparse + +import pytest + +import hermes_cli.models as models_mod + +CURATED = ["vendor/allowed", "vendor/blocked"] +ALLOWED = {"vendor/allowed"} + + +@pytest.fixture +def policy(monkeypatch): + """An org whose policy admits only ``vendor/allowed``.""" + monkeypatch.setattr(models_mod, "nous_policy_allowed_ids", lambda **_k: ALLOWED) + return ALLOWED + + +@pytest.fixture +def no_policy(monkeypatch): + """An unrestricted org — lists must come through untouched.""" + monkeypatch.setattr(models_mod, "nous_policy_allowed_ids", lambda **_k: None) + + +class TestLoginNous: + """``_login_nous`` — the model picked at login is the model then used.""" + + def _run(self, monkeypatch, tmp_path): + import hermes_cli.auth as auth_mod + import hermes_cli.nous_subscription as ns + + seen: dict = {} + monkeypatch.setenv("HERMES_HOME", str(tmp_path)) + monkeypatch.setattr( + auth_mod, + "_nous_device_code_login", + lambda **_k: { + "access_token": "tok", + "agent_key": "key", + "inference_base_url": "https://inference.example.com", + "portal_base_url": "https://portal.example.com", + "refresh_token": "r", + "token_expires_at": 9999999999, + }, + ) + monkeypatch.setattr(models_mod, "get_curated_nous_model_ids", lambda: list(CURATED)) + monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {}) + monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None) + monkeypatch.setattr( + models_mod, + "union_with_portal_paid_recommendations", + lambda ids, pricing, _portal: (list(ids), pricing), + ) + monkeypatch.setattr(ns, "prompt_enable_tool_gateway", lambda _c: None) + + def _capture(model_ids, **kwargs): + seen["model_ids"] = list(model_ids) + return None + + monkeypatch.setattr(auth_mod, "_prompt_model_selection", _capture) + + args = argparse.Namespace( + portal_url=None, inference_url=None, client_id=None, scope=None, + no_browser=True, timeout=15.0, ca_bundle=None, insecure=False, + ) + auth_mod._login_nous(args, auth_mod.PROVIDER_REGISTRY["nous"]) + return seen + + def test_hidden_model_is_not_offered(self, monkeypatch, tmp_path, policy): + assert self._run(monkeypatch, tmp_path).get("model_ids") == ["vendor/allowed"] + + def test_unrestricted_org_sees_the_full_curated_list( + self, monkeypatch, tmp_path, no_policy + ): + assert self._run(monkeypatch, tmp_path).get("model_ids") == CURATED + + +class TestModelSwitchPicker: + """The ``/model`` picker's nous branch (``list_authenticated_providers``).""" + + def _rows(self, monkeypatch): + import hermes_cli.auth as auth_mod + import hermes_cli.model_switch as ms + + monkeypatch.setattr( + auth_mod, + "_load_auth_store", + lambda *a, **k: {"providers": {"nous": {"access_token": "tok"}}}, + ) + monkeypatch.setattr(models_mod, "get_curated_nous_model_ids", lambda: list(CURATED)) + monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {}) + monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None) + monkeypatch.setattr( + models_mod, + "union_with_portal_paid_recommendations", + lambda ids, pricing, _portal: (list(ids), pricing), + ) + rows = ms.list_authenticated_providers(max_models=10) + return next((r for r in rows if r["slug"] == "nous"), None) + + def test_hidden_model_is_filtered(self, monkeypatch, policy): + row = self._rows(monkeypatch) + assert row is not None, "nous row should be listed" + assert "vendor/blocked" not in row["models"] + assert "vendor/allowed" in row["models"] + + def test_unrestricted_org_keeps_both(self, monkeypatch, no_policy): + row = self._rows(monkeypatch) + assert row is not None + assert set(CURATED) <= set(row["models"]) + + def test_filter_survives_a_failed_recommendation_fetch(self, monkeypatch, policy): + """The filter sits outside the try that wraps the Portal union, so a + Portal outage still yields a policy-filtered curated list.""" + + def _boom(_p): + raise RuntimeError("portal down") + + monkeypatch.setattr(models_mod, "get_pricing_for_provider", _boom) + row = self._rows(monkeypatch) + assert row is not None + assert "vendor/blocked" not in row["models"] + + +class TestRecommendedDefaultEndpoint: + """``GET /api/model/recommended-default`` picks a model the user never sees + chosen, so an unreachable one there is worse than in a picker.""" + + def _call(self, monkeypatch): + import hermes_cli.auth as auth_mod + from hermes_cli.web_server import get_recommended_default_model + + # Blocked first, so an unfiltered list would make it the silent + # default — otherwise this passes whether or not the filter runs. + monkeypatch.setattr( + models_mod, "get_curated_nous_model_ids", + lambda: ["vendor/blocked", "vendor/allowed"], + ) + monkeypatch.setattr(models_mod, "get_pricing_for_provider", lambda _p: {}) + monkeypatch.setattr(models_mod, "check_nous_free_tier", lambda **_k: None) + monkeypatch.setattr( + models_mod, + "union_with_portal_paid_recommendations", + lambda ids, pricing, _portal: (list(ids), pricing), + ) + monkeypatch.setattr(auth_mod, "get_provider_auth_state", lambda _p: {}) + return get_recommended_default_model(provider="nous") + + def test_hidden_model_is_never_the_silent_default(self, monkeypatch, policy): + assert self._call(monkeypatch)["model"] == "vendor/allowed" + + def test_unrestricted_org_is_unaffected(self, monkeypatch, no_policy): + assert self._call(monkeypatch)["model"] == "vendor/blocked" From 35e0d158615ff5aae4d2e5a8b358ebcb69478870 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Thu, 27 Aug 2026 19:14:22 -0300 Subject: [PATCH 021/634] fix(aux): read the Nous fast-model catalog with credentials, and filter it `_fast_model_from_catalog` treats the catalog's keys as a source of ids, scanning them for a cheap model to use for side tasks like titling. Two problems for Nous. The credential lookup goes through `resolve_api_key_provider_credentials`, which raises for Nous because it is OAuth. The read then went out anonymous and came back with the full catalog rather than the one the org may reach, so a policy-hidden model could be selected and then refused at request time with `model_blocked_by_org_policy`. Fall back to the Nous credential resolver when the api-key path raises, and narrow the resulting ids by the org policy the same way the pickers' lists are narrowed. Co-Authored-By: Claude Opus 5 (1M context) --- agent/auxiliary_client.py | 24 +++++++++++ tests/hermes_cli/test_nous_policy_surfaces.py | 41 +++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index e371b0f676..534b42184a 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -884,6 +884,18 @@ def _fast_model_from_catalog(provider_id: str) -> str: # fetch below still works for the catalogs that allow it. logger.debug("No credentials for %s catalog", provider_id, exc_info=True) + if not api_key and provider_id.strip().lower() == "nous": + # Nous is OAuth, so the api-key resolver above raises for it. An + # anonymous read returns the full catalog rather than the one the + # org may reach, and a model picked from it is refused at request + # time with model_blocked_by_org_policy. + try: + from hermes_cli.models import _resolve_nous_pricing_credentials + + api_key, base_url = _resolve_nous_pricing_credentials() + except Exception: + logger.debug("No Nous credentials for catalog", exc_info=True) + if not base_url: base_url = str(getattr(get_provider_profile(provider_id), "base_url", "") or "") base_url = base_url.rstrip("/") @@ -900,6 +912,18 @@ def _fast_model_from_catalog(provider_id: str) -> str: return "" ids = sorted((str(m) for m in catalog), key=_model_recency_key, reverse=True) + if provider_id.strip().lower() == "nous": + # The catalog's keys are a source of ids here, so the policy has to + # narrow them the same way it narrows the pickers' lists. + try: + from hermes_cli.models import ( + nous_policy_allowed_ids, + restrict_to_nous_policy, + ) + + ids = restrict_to_nous_policy(ids, nous_policy_allowed_ids()) + except Exception: + logger.debug("Nous policy filter unavailable", exc_info=True) for family in _FAST_MODEL_FAMILIES: for model_id in ids: lowered = model_id.lower() diff --git a/tests/hermes_cli/test_nous_policy_surfaces.py b/tests/hermes_cli/test_nous_policy_surfaces.py index c8f4cc9d18..d3103c9619 100644 --- a/tests/hermes_cli/test_nous_policy_surfaces.py +++ b/tests/hermes_cli/test_nous_policy_surfaces.py @@ -160,3 +160,44 @@ class TestRecommendedDefaultEndpoint: def test_unrestricted_org_is_unaffected(self, monkeypatch, no_policy): assert self._call(monkeypatch)["model"] == "vendor/blocked" + + +class TestAuxiliaryFastModel: + """``_fast_model_from_catalog`` treats the catalog's keys as a source of + ids, so an anonymous read there can select a model the gateway refuses.""" + + def _pick(self, monkeypatch, *, catalog): + import agent.auxiliary_client as aux + + seen: dict = {} + + def _fake_fetch(*, api_key=None, base_url="", timeout=8.0, **_k): + seen["api_key"] = api_key + return {mid: {} for mid in catalog} + + monkeypatch.setattr( + models_mod, "_resolve_nous_pricing_credentials", + lambda: ("sk-nous", "https://inference.example.com"), + ) + monkeypatch.setattr(models_mod, "fetch_models_with_pricing", _fake_fetch) + picked = aux._fast_model_from_catalog("nous") + return picked, seen + + def test_reads_the_catalog_with_nous_oauth_credentials(self, monkeypatch, no_policy): + """The api-key resolver raises for OAuth providers; without a fallback + the read goes out anonymous and returns the unfiltered catalog.""" + _, seen = self._pick(monkeypatch, catalog=["vendor/haiku-fast"]) + assert seen["api_key"] == "sk-nous" + + def test_hidden_model_is_not_selected(self, monkeypatch, policy): + import agent.auxiliary_client as aux + + monkeypatch.setattr( + models_mod, "nous_policy_allowed_ids", lambda **_k: {"vendor/allowed"} + ) + monkeypatch.setattr(aux, "_FAST_MODEL_FAMILIES", ("vendor/",)) + monkeypatch.setattr(aux, "_FAST_MODEL_EXCLUDE", ()) + picked, _ = self._pick( + monkeypatch, catalog=["vendor/blocked", "vendor/allowed"] + ) + assert picked == "vendor/allowed" From 9fc43919cc58772569a4bd9254ed180d55c0f900 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Thu, 27 Aug 2026 19:14:38 -0300 Subject: [PATCH 022/634] perf(nous): stop prefetching a catalog nothing reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `/model` picker warms `provider_models_cache.json` in parallel before its serial build loop, and Nous was collected into that prefetch because the credential scan treats any auth.json providers entry as credentials regardless of auth type. Nothing reads the result. The picker's nous branch builds from the curated list rather than `cached_provider_model_ids`, and Nous cannot reach the api_key-only unified pathway that would call it. Because the prefetch forces a refresh it also skips the cache read, so the entry is written and never read — a live authenticated /v1/models round trip per picker open for nothing. Exclude it. Also add the plan this and the preceding commits implement. Co-Authored-By: Claude Opus 5 (1M context) --- docs/nous-org-model-policy.md | 269 ++++++++++++++++++ hermes_cli/model_switch.py | 7 +- tests/hermes_cli/test_nous_policy_surfaces.py | 16 ++ 3 files changed, 291 insertions(+), 1 deletion(-) create mode 100644 docs/nous-org-model-policy.md diff --git a/docs/nous-org-model-policy.md b/docs/nous-org-model-policy.md new file mode 100644 index 0000000000..174f65fec7 --- /dev/null +++ b/docs/nous-org-model-policy.md @@ -0,0 +1,269 @@ +# Honouring the Nous org model policy in the pickers + +> **Audience:** Contributors touching Nous model selection +> **Source files:** `hermes_cli/auth.py` (`_login_nous`, `fetch_nous_models`, +> `_prompt_model_selection`), `hermes_cli/models.py` (`fetch_models_with_pricing`, +> `get_pricing_for_provider`, `union_with_portal_*`, `partition_nous_models_by_tier`), +> `hermes_cli/model_setup_flows.py` (`_model_flow_nous`), +> `hermes_cli/model_switch.py` (`list_authenticated_providers`), +> `hermes_cli/web_server.py` (`/api/model/recommended-default`), +> `hermes_cli/nous_account.py` (`_info_from_valid_jwt`) +> **Related:** Inference gateway PR #164 (filters `GET /v1/models` by org policy), +> NAS #941 (team admins restrict providers), NAS `openrouter-provider-map` +> (publishes the model→providers map the gateway filter needs) + +## What changed upstream + +A Nous team admin can restrict which models and which serving providers their +org may use. The inference gateway applies that policy to `GET /v1/models`, so +an authenticated catalog read returns only what the caller may actually reach. +Blocked models are **omitted** — the row is skipped, no marker field is added +(`api/src/handlers/models.ts:99-138`). An anonymous read is still allowed and +still returns the full catalog (`api/src/app.ts:309-313` — no auth middleware +on the route). + +Two things bound how urgent this is. + +**The gateway is authoritative and this is cosmetic.** Asking for a hidden +model is refused at request time with `403 model_blocked_by_org_policy` +(`api/src/middleware/model_entitlement_gate.ts:337-356`). The listing fails +open; the request gate fails closed. Nothing here is a security boundary — the +cost of a wrong list is a predictable 403, and the gateway PR states that +tradeoff deliberately. This document is only about the client showing the +right list. + +**It is inert today.** PR #164 is merged, but is switched off until NAS +publishes the policy fields and the provider map, and the admin surface sits +behind the `org-model-policy` Vercel flag. The `openrouter-provider-map` branch +is the publisher half (a daily cron writing `openrouter_model_providers` to the +entitlement Redis). Until that lands, every caller — anonymous and +authenticated — gets the same unfiltered list. **No change here is verifiable +end to end yet; every test mocks the filtered response.** + +## Where we stand + +Four surfaces list Nous models. **None of them is filtered.** + +| surface | builds its list from | filtered | +| --- | --- | --- | +| Login (`_login_nous`, `auth.py:9383`) | `get_curated_nous_model_ids()` ∪ Portal recommendations | no | +| `hermes model` (`_model_flow_nous`, `model_setup_flows.py:399`) | same | no | +| `/model` picker (`list_authenticated_providers`, `model_switch.py:3062`) | same | no | +| Dashboard onboarding (`web_server.py:7486`) | same | no | + +All four seed from the docs-hosted manifest and union the Portal's +`recommended-models` endpoint. Neither source is authenticated, so org policy +has no effect on any list a user picks from. + +`cached_provider_model_ids("nous")` — which *does* reach the authenticated +`fetch_nous_models` — is not consulted by any of them. The `/model` picker +handles nous in its own branch that deliberately bypasses it, and nous cannot +reach the generic pathway at `model_switch.py:2898` because line 2861 skips +every non-`api_key` provider. Its only caller for nous is the background +prefetch (`model_switch.py:2390`), which writes an entry nothing reads. + +Two things that are already fine, and should stay that way: + +- `nous` is **not** in `_MODELS_DEV_PREFERRED`, so no models.dev entries are + merged on top of the live list. +- The nous fallback ladder in `provider_model_ids` is a *chain* (live → + manifest → in-repo snapshot), not a merge, so a successful live fetch is + used exclusively. + +--- + +## Fix 0 — put auth state in the pricing cache key + +**This is a prerequisite for fix 1, and worth landing on its own merits.** + +**Problem.** `fetch_models_with_pricing` caches on the base URL alone, and the +cache check happens *above* the point where the `Authorization` header is built +(`models.py:2404`): + +```python +cache_key = (base_url or "").rstrip("/") +if not force_refresh: + cached = _cached_catalog(cache_key) + if cached is not None: + return cached +... +if api_key: + headers["Authorization"] = f"Bearer {api_key}" +``` + +`_pricing_cache` is process-lifetime with no expiry for a non-empty result +(`models.py:2231-2253`). So whichever read of a given base URL lands first — +authenticated or anonymous — answers every later read in that process, +whatever key it passes. An anonymous read landing first (the auxiliary-model +path in fix 2 is one) makes a later authenticated read return an unfiltered +list without touching the network. A fix built on this cache looks like it +works and does not. + +**Do.** Fold auth state into the cache key. Distinguishing authenticated from +anonymous is enough — the token value need not be in the key, and keeping it +out avoids hashing a secret. + +**Do** update `agent/credits_tracker.py:257`, which reaches into the private +`_pricing_cache` dict assuming one entry per base URL. + +**Test.** An anonymous read followed by an authenticated read of the same base +URL issues two requests and returns two different lists. Independently +testable today, unlike everything below. + +## Fix 1 — narrow each list to the org's policy + +**Problem.** All four surfaces build their list from +`get_curated_nous_model_ids()` unioned with the Portal's `recommended-models` +endpoint. Neither is authenticated, so org policy has no effect on the model a +user picks — which is the model they then use. The Portal endpoint compounds +it: it takes no auth and no parameters, returns one globally CDN-cached payload +for the whole platform, and is invalidated only by admin pricing edits — never +by a policy change. It can put a hidden model straight back into a list. There +is no policy-aware variant of it and no parameter that would make one. + +Each surface, however, already fetches `/v1/models`. +`get_pricing_for_provider("nous")` calls `fetch_models_with_pricing`, which +reads that endpoint and returns `{model_id: {...}}`, and already resolves +credentials (`_resolve_nous_pricing_credentials`), so it is already the +authenticated read. Its keys are the reachable set. + +**Do.** Use that set to *narrow* each list, keeping the curated order. +`nous_policy_allowed_ids()` obtains the set; `restrict_to_nous_policy()` +applies it. Both live in `models.py`, and the fetch reuses the pricing cache +entry the surface already populates, so no surface makes an extra request. + +**Do not** replace a list with the response's keys. Every surface shows the +curated agentic list in curated order deliberately — the live catalog is a +large alphabetical dump of vendor-prefixed models, and swapping it in is the +regression `model_switch.py:3070` records. Recommendations should be able to +*reveal* a newly launched model; the policy set should only ever subtract. + +**Do not** narrow a list on evidence that cannot support it. +`nous_policy_allowed_ids()` returns `None` — meaning "leave the list alone" — +in three cases, and each matters: + +- **The org has no policy, or the token is too old to say.** Gated on the + `policy_present` claim (fix 4). For an unrestricted org — the common case — + filtering buys nothing and risks dropping a Portal recommendation the + gateway catalog has not caught up on yet. This keeps the change a no-op for + everyone the policy does not apply to. +- **Credential resolution failed**, so the read was anonymous and therefore + unfiltered. A full catalog must not be mistaken for a filtered one. A stated + degradation, not a silent one. +- **The read came back empty**, which is a fetch failure, not an org that may + reach nothing. + +A `:free` sibling is kept when its base model is reachable, mirroring the +gateway, which admits a row when any of its requestable ids passes and treats +anything unknown as a keep — "over-listing costs a 403 from the authoritative +gate, while hiding a row the gate would serve is unrecoverable from the client" +(`api/src/libs/catalog_policy.ts:74-78`). Prefer over-listing here too. + +**Test.** With a policy hiding model X: X is absent from each of the four +lists, and no surface makes more Nous requests than it does today. With no +policy, with credentials broken, or with an empty read, every list is byte-for- +byte what it is today. A model the Portal flags as free but the org hides stays +out; curated ordering survives filtering. + +## Fix 2 — audit the other readers of the pricing map + +**Problem.** `fetch_models_with_pricing` is shared, so any caller that treats +its keys as "the models that exist" inherits whatever authentication the first +caller happened to have. Fix 0 stops the *authentication* from leaking between +callers; this fix is about which callers may treat the map as a source of ids +at all. + +**Do.** Make the map a lookup *for* ids already in the list, never a source of +ids. Two consumers are already correct and should stay that way: +`partition_nous_models_by_tier` only looks up ids it was given, and the +`union_with_portal_*` pair only ever writes into the map — their id-widening +comes from the Portal endpoint (fix 2), not from the map. + +The one that is wrong is `agent/auxiliary_client.py:869-908` +(`_fast_model_from_catalog`), which iterates the map's keys directly as its +candidate list off an anonymous read. Reachable for nous on the titling path, +where it can select a policy-hidden model that then 403s at request time. + +**Test.** With credentials broken so the read falls back to anonymous, no +list grows. + +## Fix 3 — stop prefetching the nous catalog + +**Problem.** The background prefetch calls +`cached_provider_model_ids("nous", force_refresh=True)` +(`model_switch.py:2390`); nous is collected into it because +`_collect_authed_provider_slugs` treats any `auth.json` providers entry as +credentials regardless of `auth_type` (`model_switch.py:2519-2526`). Because +`force_refresh=True` skips the cache read and no nous surface reads the entry, +this is a live authenticated `/v1/models` round trip per picker open written to +a location nothing consults. + +**Do.** Exclude nous from the prefetch and delete the write-only entry. + +This replaces what an earlier draft proposed here — folding `org_id` into +`_credential_fingerprint` and shortening `_PROVIDER_MODELS_STALE_SERVE_MAX` +for nous (a single global constant, `models.py:4204`, with no per-provider +branching today). Both would have hardened a cache that, after fix 1, has no +nous readers to protect. If a future surface routes nous through +`cached_provider_model_ids` again, revisit the fingerprint then: it hashes +env-var values and `auth.json` mtime and carries no org signal +(`models.py:4277`), so two orgs on one machine can serve each other's list. + +**Test.** Opening the `/model` picker makes no Nous `/v1/models` request beyond +the one the displayed list is built from. + +## Fix 4 — the `policy_present` claim + +**Problem.** Under omission a blocked model simply vanishes, which reads as +"Hermes does not support this" rather than "your org disallows it". + +**Do.** Read the `policy_present` claim off the Nous OAuth access token and, +when it is `true`, show a single line stating that the org restricts which +models are available. No enumeration, no per-model marking. + +The claim rides the same JWT as `org_id` +(`access-token-issuer.ts:552,595`, `token_use: "access"`) — the token the +client already decodes — and `_info_from_valid_jwt` already retains every +claim in `raw_claims` (`nous_account.py:600-647`), so surfacing it is one +typed field on `NousPortalAccountInfo` and no new request. + +It is already widened to cover provider-only restrictions, not just model +allowlists (`nous-account-service/src/server/entitlement-snapshot.ts:478-480`). +Two NAS docs still describe it as allowlist-only and list the widening as +pending — they are stale; trust that expression. + +**Do not** enumerate the blocked set. Model policy is allowlist-only — +`denyModels` is a dead column (`nous-account-service/src/server/model-policy.ts:230`) +— so an org that allows five models blocks the entire rest of the catalog. +Graying hundreds of rows is a worse UI than omitting them. An earlier draft +proposed deriving the blocked set by diffing the anonymous and authenticated +reads and feeding it to `_prompt_model_selection`'s `unavailable_models`; that +is the wrong shape twice over, because that picker carries one +`unavailable_message` for the whole list and cannot say "free-tier-gated" and +"policy-hidden" at once. + +**Do not** report the absence of the claim as the absence of a policy. It is +tri-state: `true`, `false`, and absent, where absent means unknown — an older +mint, not an unrestricted org. The gateway rejects a corrupt (non-boolean) +claim outright rather than reading it as "no policy" +(`api/src/middleware/nas_jwt_auth.ts:179`). Show the line only on `true`. + +**Known bound:** the claim is stamped at mint time, so it goes stale until the +next token refresh — the line can lag a policy change by up to the access +token's lifetime. Acceptable, and worth stating rather than rediscovering. + +**Test.** With `policy_present` true the line shows; with it false or absent it +does not. + +--- + +## Order + +Fix 0 first: fix 1 is silently wrong without it, and it is the only piece +testable before NAS switches the feature on. Fix 4's claim gates fix 1, so the +two land together. Fix 1 is the correctness work — without it the policy is +bypassed on every surface a user picks from. Fix 2 keeps the pricing map from +becoming another way to widen a list. Fix 3 is a deletion that fix 1 makes +safe. + +Run tests with `scripts/run_tests.sh` — not bare `pytest`. diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index cb7f9cf6df..3516cdfe24 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -2565,7 +2565,12 @@ def _collect_authed_provider_slugs( slugs.append(_cp.slug) seen.add(_cp.slug.lower()) - return slugs + # Nous is deliberately excluded. Its picker branch builds from the curated + # list rather than cached_provider_model_ids, and nous cannot reach the + # api_key-only unified pathway, so a prefetched entry is written and never + # read — a live authenticated /v1/models round trip per picker open for + # nothing. + return [s for s in slugs if s != "nous"] def list_authenticated_providers( diff --git a/tests/hermes_cli/test_nous_policy_surfaces.py b/tests/hermes_cli/test_nous_policy_surfaces.py index d3103c9619..e279162b0c 100644 --- a/tests/hermes_cli/test_nous_policy_surfaces.py +++ b/tests/hermes_cli/test_nous_policy_surfaces.py @@ -201,3 +201,19 @@ class TestAuxiliaryFastModel: monkeypatch, catalog=["vendor/blocked", "vendor/allowed"] ) assert picked == "vendor/allowed" + + +class TestNousPrefetch: + """The nous disk-cache entry is write-only: its picker branch builds from + the curated list, so prefetching it is a round trip for nothing.""" + + def test_nous_is_not_collected_for_prefetch(self, monkeypatch): + import hermes_cli.auth as auth_mod + import hermes_cli.model_switch as ms + + monkeypatch.setattr( + auth_mod, "_load_auth_store", + lambda *a, **k: {"providers": {"nous": {"access_token": "tok"}}}, + ) + slugs = ms._collect_authed_provider_slugs({}, {"nous": list(CURATED)}, []) + assert "nous" not in slugs From c38d62aefe9f0a394fb417c598e9458fe2c193d9 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Thu, 27 Aug 2026 19:17:56 -0300 Subject: [PATCH 023/634] feat(nous): tell a governed org its model choice is restricted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gateway omits a policy-blocked model from `/v1/models` rather than marking it, so after the preceding commits a restricted model is simply absent from the pickers. That reads as "Hermes does not support this" instead of "your organization disallows it". Show one line when the org is governed, in the two flows where a user picks a model. It enumerates nothing: model policy is an allowlist, so an org admitting a handful of models blocks the whole rest of the catalog, and graying hundreds of rows would be a worse UI than omitting them. Driven by the `policy_present` claim, which is tri-state — the line shows only when it is explicitly true, because an absent claim means an older mint rather than an unrestricted org. The claim is stamped at mint time, so the line can lag a policy change by up to the access token's lifetime. Co-Authored-By: Claude Opus 5 (1M context) --- hermes_cli/auth.py | 5 ++++ hermes_cli/model_setup_flows.py | 5 ++++ hermes_cli/nous_account.py | 21 +++++++++++++++ tests/hermes_cli/test_nous_policy_filter.py | 27 +++++++++++++++++++ tests/hermes_cli/test_nous_policy_surfaces.py | 20 ++++++++++++++ 5 files changed, 78 insertions(+) diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index f97f91bca5..eebbc5609c 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -9438,6 +9438,11 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: ) _portal = auth_state.get("portal_base_url", "") if model_ids: + from hermes_cli.nous_account import nous_policy_notice + + _policy_notice = nous_policy_notice() + if _policy_notice: + print(_policy_notice) print(f"Showing {len(model_ids)} curated models — use \"Enter custom model name\" for others.") selected_model = _prompt_model_selection( model_ids, pricing=pricing, diff --git a/hermes_cli/model_setup_flows.py b/hermes_cli/model_setup_flows.py index 90075912a1..9489692159 100644 --- a/hermes_cli/model_setup_flows.py +++ b/hermes_cli/model_setup_flows.py @@ -581,6 +581,11 @@ def _model_flow_nous(config, current_model="", args=None): print(unavailable_message or f"Upgrade at {_url} to access paid models.") return + from hermes_cli.nous_account import nous_policy_notice + + _policy_notice = nous_policy_notice() + if _policy_notice: + print(_policy_notice) print( f'Showing {len(model_ids)} curated models — use "Enter custom model name" for others.' ) diff --git a/hermes_cli/nous_account.py b/hermes_cli/nous_account.py index 4c3bf0a51f..c63d090fc4 100644 --- a/hermes_cli/nous_account.py +++ b/hermes_cli/nous_account.py @@ -427,6 +427,27 @@ def nous_policy_present() -> Optional[bool]: return None +def nous_policy_notice() -> str: + """A one-line notice for an org that restricts model choice, else ``""``. + + Under the gateway's policy filter a blocked model is omitted rather than + marked, which reads as "Hermes does not support this" instead of "your org + disallows it". This says which it is without enumerating anything: model + policy is an allowlist, so an org that admits a handful of models blocks + the whole rest of the catalog, and listing those would be a worse UI than + omitting them. + + Silent unless the claim is explicitly true — absent means an older mint, + not an unrestricted org. + """ + if nous_policy_present() is not True: + return "" + return ( + "Your organization restricts which models are available — " + "models outside its policy are not listed." + ) + + def _fresh_account_info( *, state: dict[str, Any], diff --git a/tests/hermes_cli/test_nous_policy_filter.py b/tests/hermes_cli/test_nous_policy_filter.py index a225d560dc..edb7e6dc20 100644 --- a/tests/hermes_cli/test_nous_policy_filter.py +++ b/tests/hermes_cli/test_nous_policy_filter.py @@ -157,3 +157,30 @@ class TestNousPolicyPresent: def test_undecodable_token_is_unknown(self, monkeypatch): self._patch_token(monkeypatch, "not-a-jwt") assert nous_policy_present() is None + + +class TestNousPolicyNotice: + """A governed org is told its choice is restricted, rather than left to + read an omitted model as one Hermes does not support.""" + + def _patch(self, monkeypatch, present): + monkeypatch.setattr(account_mod, "nous_policy_present", lambda: present) + + def test_shows_a_line_for_a_governed_org(self, monkeypatch): + self._patch(monkeypatch, True) + assert "restricts which models" in account_mod.nous_policy_notice() + + @pytest.mark.parametrize("present", [False, None]) + def test_silent_otherwise(self, monkeypatch, present): + """Absent is an older mint, not an unrestricted org — either way there + is nothing truthful to say.""" + self._patch(monkeypatch, present) + assert account_mod.nous_policy_notice() == "" + + def test_names_no_models(self, monkeypatch): + """Policy is an allowlist, so the blocked set is most of the catalog; + the notice must not try to enumerate it.""" + self._patch(monkeypatch, True) + notice = account_mod.nous_policy_notice() + assert "/" not in notice, f"looks like it names a model: {notice}" + assert len(notice.splitlines()) == 1 diff --git a/tests/hermes_cli/test_nous_policy_surfaces.py b/tests/hermes_cli/test_nous_policy_surfaces.py index e279162b0c..e59974c511 100644 --- a/tests/hermes_cli/test_nous_policy_surfaces.py +++ b/tests/hermes_cli/test_nous_policy_surfaces.py @@ -217,3 +217,23 @@ class TestNousPrefetch: ) slugs = ms._collect_authed_provider_slugs({}, {"nous": list(CURATED)}, []) assert "nous" not in slugs + + +class TestPolicyNoticeIsShown: + """The notice reaches the two flows where a user picks a model.""" + + def test_login_prints_it(self, monkeypatch, tmp_path, policy, capsys): + import hermes_cli.nous_account as account_mod + + monkeypatch.setattr(account_mod, "nous_policy_present", lambda: True) + TestLoginNous()._run(monkeypatch, tmp_path) + assert "restricts which models" in capsys.readouterr().out + + def test_login_silent_for_an_ungoverned_org( + self, monkeypatch, tmp_path, no_policy, capsys + ): + import hermes_cli.nous_account as account_mod + + monkeypatch.setattr(account_mod, "nous_policy_present", lambda: False) + TestLoginNous()._run(monkeypatch, tmp_path) + assert "restricts which models" not in capsys.readouterr().out From a69a9c351dfa7cc37b804259b19c4f4a6a3a7e44 Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Fri, 28 Aug 2026 15:22:16 +1000 Subject: [PATCH 024/634] feat(telemetry): transmit the stable install_id as-is Product-owner decision, 2026-08-27: the analytical need is stable cross-window identity (retention curves, longitudinal install behaviour), which the rotating pseudonym destroyed by design. The feature has not shipped - zero consented users, zero production transmissions - so identity semantics can change without breaking any promise made to a user; existing (dev-only) consent windows carry forward unchanged. Removed in full rather than weakened in place: - shared_metrics_identity.py (salt generation/rotation, HMAC-SHA256 derivation, payload substitution) and its 19-test file. - The sender's derivation step. _freeze_identity keeps its validation role (unreadable/non-object/id-less payloads still reject rather than block the queue) and now records the raw install_id in sent_install_id; _body rewrites the payload's install_id from that frozen column, keeping byte-identical resends anchored to one recorded value. Consent surface updated in the same change: the setup wizard now states plainly that packages carry the stable profile-scoped install ID (a random UUID, no personal information, reset by deleting the shared-metrics directory). No consent was ever collected under the old wording in any shipped build. Docs A.2/A.3 rewritten as decision records rather than silently edited: A.2 records what is transmitted now and states the consequences plainly (indefinite cross-package correlation is the designed behaviour); A.3 records why rotation existed and why its removal was accepted. The main-body "must not reuse the persistent local identifier by default" escape hatch is exercised, not deleted: that paragraph required exactly this product decision, which has now been made. A.6's deletion note updated: install_id is now itself the lookup key, so a future delete-on-request needs only a service-side API, not a mapping. Tests: the two privacy assertions invert deliberately (test_the_stable_install_id_is_transmitted_as_is and the e2e wire variant); freezing/byte-identical-retry coverage unchanged. Staging E2E script now asserts transmitted == install_id. 258 targeted tests pass; ruff + footguns clean; both staging E2E harnesses green with the raw id observed on the wire (202s). --- docs/observability/relay-shared-metrics.md | 138 +++++++------- hermes_cli/observability/shared_metrics.py | 5 +- .../observability/shared_metrics_identity.py | 131 ------------- .../observability/shared_metrics_sender.py | 36 ++-- hermes_cli/setup.py | 10 +- scripts/e2e_shared_metrics_staging.py | 12 +- .../test_shared_metrics_identity.py | 179 ------------------ .../hermes_cli/test_shared_metrics_sender.py | 14 +- .../test_shared_metrics_sender_e2e.py | 7 +- 9 files changed, 119 insertions(+), 413 deletions(-) delete mode 100644 hermes_cli/observability/shared_metrics_identity.py delete mode 100644 tests/hermes_cli/test_shared_metrics_identity.py diff --git a/docs/observability/relay-shared-metrics.md b/docs/observability/relay-shared-metrics.md index cb3ea2197a..a736bf6edd 100644 --- a/docs/observability/relay-shared-metrics.md +++ b/docs/observability/relay-shared-metrics.md @@ -230,17 +230,18 @@ packages from that profile and can therefore link those local packages. Deleting `$HERMES_HOME/telemetry/shared_metrics` resets the identifier together with all aggregates and package files. -Remote delivery is opt-in and off by default. A remote exporter must not reuse -the persistent local identifier by default. It requires a separate product and -privacy decision covering consent, identity scope, rotation or keyed -pseudonymization, reset behavior, retention, and deletion. +Remote delivery is opt-in and off by default. Reusing the persistent local +identifier remotely required a separate product and privacy decision covering +consent, identity scope, reset behavior, retention, and deletion — that +decision has been made. > Those decisions are recorded in > [Appendix A](#appendix-a-remote-exporter-decisions-phase-2), and the exporter > implementing them has shipped. Collection alone still transmits nothing: the -> sender runs only when `telemetry.shared_metrics.send` is also true, and it -> transmits a rotating HMAC of the install identity rather than the identifier -> itself. +> sender runs only when `telemetry.shared_metrics.send` is also true. Each +> transmitted package carries the stable `install_id` as-is (product decision, +> 2026-08-27 — see A.2 for the record, including the superseded +> HMAC-pseudonym design). The install identity is scoped to one `HERMES_HOME`. To reset it, stop Hermes processes and remove `$HERMES_HOME/telemetry/shared_metrics`. This deliberately @@ -327,60 +328,66 @@ Local history can be up to 30 days old, and that data was collected under a promise that nothing is uploaded. Honouring consent forward-only costs at most 30 days of backlog we never had permission to send. -### A.2 Identity scope — the transmitted identifier is derived, not the local one +### A.2 Identity scope — the stable install_id is transmitted as-is -`install_id` is the persistent profile-scoped identifier described above. It is -**not transmitted**. Each package sent carries a derived value instead: +**Decision record.** The original design of this exporter (and revisions 1–8 +of this appendix) transmitted a keyed pseudonym instead of the identifier: +`HMAC-SHA256(key = locally-held rotating salt, message = install_id)`, with +the salt rotating every 30 days. On **2026-08-27**, before the feature +shipped (zero consented users, zero production transmissions), the product +owner decided the analytical need is a **stable cross-window identity** — +retention curves, longitudinal install behaviour — which rotation by design +destroys. The pseudonymization layer was removed in full rather than +weakened in place. -```text -transmitted_id = HMAC-SHA256(key = rotation_salt, message = install_id) -``` +What is transmitted now: -- `rotation_salt` is random, generated locally, and never leaves the machine. -- The derivation is one-way: the service cannot recover `install_id`. -- Within a rotation window, packages from one profile correlate — so distinct - installs remain countable, which is the primary analytical question. -- Across windows, they do not. +- Each package carries `install_id` verbatim: the persistent, profile-scoped + random UUID described above. +- It is generated locally (`uuid4`), contains no hardware, account, user, or + machine-derived information, and identifies a *profile*, not a person. +- It is stable until the user deletes the shared-metrics directory, which + regenerates it (see A.4). -This satisfies "must not reuse the persistent local identifier by default" -while keeping the data useful. Stripping the identifier entirely was rejected -because "how many installs are reporting" is the first question the data must -answer; sending `install_id` unchanged was rejected because it contradicts the -commitment made above. +Consequences stated plainly rather than papered over: -**Byte-identical resends still hold.** The derived value is computed **once**, -when the package is first prepared for sending, and stored alongside the -package (the derived id only — not a second copy of the payload, which is -recomputed deterministically from the stored package). A retry therefore -rebuilds identical bytes even if the salt rotated in between. The contract -requires this: resending a `package_id` with different content is undefined -behaviour. +- Packages from one profile correlate **indefinitely**, not per-window. + Long-term linkability of one install's daily envelope sequence is now the + designed behaviour, not a residue. +- The A.3 residue analysis of the old design (stable `resource` tuple + + contiguous periods bridging rotation windows) is moot — there is no window + boundary left to bridge. +- The setup wizard's consent language states this identity model explicitly; + it was updated in the same change that removed the derivation, so no + consent was ever collected under the old wording in any shipped build. -### A.3 Rotation +**Byte-identical resends still hold.** The transmitted id is recorded on the +row (`sent_install_id`) when the package is first prepared, and the wire body +is always rebuilt from that recorded value, so a retry rebuilds identical +bytes. The contract requires this: resending a `package_id` with different +content is undefined behaviour. (With a stable id the recorded copy is no +longer load-bearing against rotation — it remains as the audit column and as +cheap insurance against any future change to identity semantics.) -`rotation_salt` rotates on a fixed schedule (default: every 30 days, aligned to -local history retention). Rotation only affects packages prepared after it; -already-prepared packages keep their derived value so retries stay -byte-identical. +### A.3 Rotation — removed (decision record) -Rotation bounds long-term linkability without destroying short-term cohort -analysis. A profile is one identity for the length of a window, and an -unrelated identity after it. +Salt rotation was deleted together with the derivation (product decision, +2026-08-27). This section is retained as a record of what the earlier design +did and why the removal was accepted: -**What rotation does not bound.** The identifier changes; the rest of the -envelope does not. `resource` (`os_family`, `architecture`, `install_method`, -`hermes_version`) is stable and low-entropy, and `period_start` / -`period_end` are contiguous across a rotation boundary. For a common -configuration this is no help to an observer — measured against the 11 real -packages in a development outbox, every one shares the same -`arm64 / macos / git` tuple. For a **rare** configuration it is a plausible -re-identification aid: an unusual architecture or install method, combined -with an uninterrupted daily period sequence, can bridge two windows. The -claim this design makes is therefore "rotation raises the cost of long-term -correlation", not "rotation makes it impossible". Narrowing that residue -would mean coarsening `resource` or jittering period boundaries, and neither -is worth the analytical loss today — but it should be a conscious decision, -not an unexamined one. +- Rotation existed to bound long-term linkability: one identity per 30-day + window, unrelated identities across windows. +- The documented residue (see git history for the full analysis): the + envelope's stable, low-entropy `resource` tuple plus contiguous daily + periods could plausibly bridge windows for rare configurations anyway, so + the boundary was a cost-raiser, not a wall. +- The product need that killed it: cross-window continuity is precisely what + retention analysis requires. A boundary that mostly inconveniences honest + analysis while only raising costs for a determined correlator was judged + the wrong trade once stable identity became a requirement. + +There is no salt in the store, no rotation schedule, and no derived +identifier anywhere in the pipeline. ### A.4 Reset behavior @@ -388,11 +395,11 @@ Removing `$HERMES_HOME/telemetry/shared_metrics` still resets local identity, aggregates, and package files, exactly as documented above. Two honest qualifications now apply: -- Reset also discards `rotation_salt`, so subsequent packages derive a **new** - transmitted identity. Local reset does give a new remote identity. +- Reset regenerates `install_id`, so subsequent packages transmit a **new** + identity. Local reset does give a new remote identity. - Reset **cannot unsend**. Packages already transmitted remain in the ingest - service's storage under their derived identifier. There is no read-back or - delete API in the v1 contract. + service's storage under the identifier they were sent with. There is no + read-back or delete API in the v1 contract. Setting `send: false` stops transmission immediately: consent is re-read before every package, so a pass already in flight stops after the package it @@ -439,13 +446,13 @@ invent one. What a user can do: |---|---| | `send: false` | No further packages leave the machine | | `enabled: false` | Collection stops; existing local state remains | -| Remove `.../shared_metrics` | Local identity, aggregates, and files reset; future sends use a new derived identity | +| Remove `.../shared_metrics` | Local identity, aggregates, and files reset; future sends use a new install_id | | Delete already-sent data | Not self-service — requires an operator acting on the S3 bucket | -If a deletion-on-request obligation is ever taken on, it needs a lookup path -from a user to their derived identifiers. That is deliberately **not** built: -it would require retaining the mapping this design exists to avoid. Any such -change is a new product decision, not an implementation detail. +If a deletion-on-request obligation is ever taken on, the lookup path is now +direct: the user's `install_id` (readable from their local store) is the key +their data is stored under. Building the service-side delete API remains a +new product decision, not an implementation detail. ### A.7 What the outbox directory is @@ -464,7 +471,8 @@ state they were promised. Send state lives in new columns on the ### A.8 Scope note -The `install_id` field inside the package body is what gets replaced by the -derived value. No other payload field changes, nothing is added, and the -service treats the whole body as opaque. Payload schema evolution therefore -stays a sender-side concern, as before. +The `install_id` field inside the package body is transmitted as the +generator wrote it (rewritten from the row's frozen `sent_install_id`, which +records the same value). No other payload field changes, nothing is added, +and the service treats the whole body as opaque. Payload schema evolution +therefore stays a sender-side concern, as before. diff --git a/hermes_cli/observability/shared_metrics.py b/hermes_cli/observability/shared_metrics.py index ddf570b6f9..87094922d9 100644 --- a/hermes_cli/observability/shared_metrics.py +++ b/hermes_cli/observability/shared_metrics.py @@ -373,8 +373,9 @@ class SharedMetricsStore: # Earliest next attempt; enforces backoff across process restarts. ("next_attempt_at", "TEXT"), ("last_error", "TEXT"), - # The derived identifier actually transmitted, frozen on the first - # attempt so retries stay byte-identical across a salt rotation. + # The identifier actually transmitted, frozen on the first + # attempt so retries stay byte-identical. Since the 2026-08-27 + # product decision this is the stable install_id itself. # Only the ~36-byte id is stored: the body is recomputed from # payload_json, whose serialisation is deterministic. ("sent_install_id", "TEXT"), diff --git a/hermes_cli/observability/shared_metrics_identity.py b/hermes_cli/observability/shared_metrics_identity.py deleted file mode 100644 index b4f8cda8e3..0000000000 --- a/hermes_cli/observability/shared_metrics_identity.py +++ /dev/null @@ -1,131 +0,0 @@ -"""Keyed pseudonymization of the shared-metrics install identity. - -``install_id`` is a persistent, profile-scoped identifier. It is deliberately -NOT transmitted: ``docs/observability/relay-shared-metrics.md`` commits that a -remote exporter "must not reuse the persistent local identifier by default". - -Each transmitted package instead carries:: - - HMAC-SHA256(key=rotation_salt, message=install_id) - -where ``rotation_salt`` is generated locally, never leaves the machine, and -rotates on a fixed schedule. Within a rotation window the value is stable, so -distinct installs stay countable — the primary analytical question. Across -windows it changes, bounding long-term linkability. - -The derivation is one-way: the service cannot recover ``install_id`` from what -it receives. - -See Appendix A.2 and A.3 of the doc above for the decision record. -""" - -from __future__ import annotations - -import hashlib -import hmac -import secrets -import sqlite3 -from datetime import datetime, timedelta, timezone - -#: Salt lifetime. Matches local history retention so the two ages line up. -ROTATION_INTERVAL = timedelta(days=30) - -#: ``telemetry_state`` keys. The salt lives in the same store as install_id, so -#: deleting the shared-metrics directory resets both together — the documented -#: reset behaviour keeps working without a second cleanup path. -SALT_KEY = "send_rotation_salt" -SALT_ISSUED_AT_KEY = "send_rotation_salt_issued_at" - -_SALT_BYTES = 32 - - -def _isoformat(value: datetime) -> str: - return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") - - -def _parse(value: str | None) -> datetime | None: - if not value: - return None - try: - parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError: - return None - if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=timezone.utc) - return parsed.astimezone(timezone.utc) - - -def _read(connection: sqlite3.Connection, key: str) -> str | None: - row = connection.execute( - "SELECT value FROM telemetry_state WHERE key = ?", (key,) - ).fetchone() - if row is None: - return None - # sqlite3.Row and plain tuples both index by position. - return str(row[0]) - - -def _write(connection: sqlite3.Connection, key: str, value: str) -> None: - connection.execute( - """ - INSERT INTO telemetry_state(key, value) VALUES (?, ?) - ON CONFLICT(key) DO UPDATE SET value = excluded.value - """, - (key, value), - ) - - -def current_salt( - connection: sqlite3.Connection, - *, - now: datetime | None = None, -) -> str: - """Return the active salt, generating or rotating it when due. - - Must be called inside a write transaction: it can write to - ``telemetry_state``. - """ - moment = now or datetime.now(timezone.utc) - salt = _read(connection, SALT_KEY) - issued_at = _parse(_read(connection, SALT_ISSUED_AT_KEY)) - - fresh = ( - salt is not None - and issued_at is not None - # Strictly within the window. A future issued_at means the clock moved - # backwards (or the value was tampered with), so the recorded age - # cannot be trusted and we reissue rather than keep using a salt of - # unknown vintage. Reissuing is the safe direction: it shortens - # linkability, and already-prepared packages keep their frozen - # identifier so retries stay byte-identical. - and issued_at <= moment < issued_at + ROTATION_INTERVAL - ) - if fresh: - return str(salt) - - salt = secrets.token_hex(_SALT_BYTES) - _write(connection, SALT_KEY, salt) - _write(connection, SALT_ISSUED_AT_KEY, _isoformat(moment)) - return salt - - -def derive_install_id(install_id: str, salt: str) -> str: - """Return the transmitted identifier for ``install_id`` under ``salt``.""" - return hmac.new( - salt.encode("utf-8"), - install_id.encode("utf-8"), - hashlib.sha256, - ).hexdigest() - - -def substitute_install_id(payload: dict, derived: str) -> dict: - """Return ``payload`` with its ``install_id`` replaced by ``derived``. - - This is the ONLY field the exporter changes. Everything else is - transmitted exactly as the generator wrote it, so payload schema evolution - stays a sender-side concern. A shallow copy is enough — only a top-level - key is replaced — and the caller's dict is left untouched. - """ - updated = dict(payload) - updated["install_id"] = derived - return updated diff --git a/hermes_cli/observability/shared_metrics_sender.py b/hermes_cli/observability/shared_metrics_sender.py index 8e6f81c4b8..9418353c9b 100644 --- a/hermes_cli/observability/shared_metrics_sender.py +++ b/hermes_cli/observability/shared_metrics_sender.py @@ -39,12 +39,6 @@ from datetime import datetime, timedelta, timezone from hermes_cli.sqlite_util import write_txn -from .shared_metrics_identity import ( - current_salt, - derive_install_id, - substitute_install_id, -) - logger = logging.getLogger(__name__) #: Contract recommends timing out at 30s and treating a timeout as retryable. @@ -432,13 +426,17 @@ class SharedMetricsSender: payload_json, now: datetime, ) -> str | None: - """Derive and persist the transmitted id, or reject an unusable row. + """Record the transmitted id on the row, or reject an unusable one. - Returns None when the package can never be sent. Rejecting rather than - raising matters: an exception here rolls back the claim transaction - and blocks every healthy package behind this one. + The stable install_id is transmitted as-is (product decision, + 2026-08-27 — see the doc's A.2). What remains of "freezing" is the + validation and the audit column: ``sent_install_id`` records exactly + what the wire will carry, and rejecting unusable rows here rather + than raising matters because an exception rolls back the claim + transaction and blocks every healthy package behind this one. """ reason = None + install_id = None try: payload = json.loads(payload_json) except (TypeError, ValueError): @@ -467,24 +465,26 @@ class SharedMetricsSender: ) return None - salt = current_salt(connection, now=now) - derived = derive_install_id(payload["install_id"], salt) connection.execute( "UPDATE package_outbox SET sent_install_id = ? WHERE package_id = ?", - (derived, package_id), + (install_id, package_id), ) - return derived + return str(install_id) # -- transmission ------------------------------------------------------ - def _body(self, payload_json: str, derived: str) -> bytes: + def _body(self, payload_json: str, transmitted_id: str) -> bytes: """Rebuild the exact bytes to send. The payload is recomputed from the stored package rather than kept as - a second copy: json.dumps with these options is deterministic, and the - only mutable input (the derived id) is frozen in the row. + a second copy: json.dumps with these options is deterministic. The + install_id is written from the frozen ``sent_install_id`` column + rather than trusted implicitly, keeping "a resend is byte-identical" + anchored to one recorded value. """ - payload = substitute_install_id(json.loads(payload_json), derived) + payload = json.loads(payload_json) + payload = dict(payload) + payload["install_id"] = transmitted_id return json.dumps(payload, indent=2, sort_keys=True).encode("utf-8") def _mark( diff --git a/hermes_cli/setup.py b/hermes_cli/setup.py index 4772929b88..2ec08da8af 100644 --- a/hermes_cli/setup.py +++ b/hermes_cli/setup.py @@ -2464,10 +2464,12 @@ def setup_telemetry(config: dict): print_success("Local shared metrics enabled.") print_info("") print_info("Sending uploads each daily package to the Nous telemetry") - print_info("service. Your profile-scoped install ID is NOT sent: packages") - print_info("carry a rotating HMAC of it instead. Only packages from the") - print_info("day you opt in onwards are ever sent, and sending can be") - print_info("turned off again at any time.") + print_info("service. Packages carry your profile-scoped install ID, a") + print_info("stable random UUID that identifies this profile across days") + print_info("(it contains no personal information and is reset by deleting") + print_info("the shared-metrics directory). Only packages from the day you") + print_info("opt in onwards are ever sent, and sending can be turned off") + print_info("again at any time.") shared_metrics["send"] = prompt_yes_no( "Send shared metrics to Nous?", default=shared_metrics.get("send") is True, diff --git a/scripts/e2e_shared_metrics_staging.py b/scripts/e2e_shared_metrics_staging.py index e0c497a342..666c9e51e9 100644 --- a/scripts/e2e_shared_metrics_staging.py +++ b/scripts/e2e_shared_metrics_staging.py @@ -171,10 +171,12 @@ def main() -> int: print(f" last_error : {row[5]}") if row[1] != "sent": failures.append(f"{row[0]} is {row[1]}: {row[5]}") - if row[4] == real_install_id: - failures.append(f"{row[0]} LEAKED the real install_id") - if not row[4] or len(str(row[4])) != 64: - failures.append(f"{row[0]} has a malformed derived id") + # Product decision 2026-08-27: the stable install_id is transmitted + # as-is; the transmitted value must be exactly the local id. + if row[4] != real_install_id: + failures.append( + f"{row[0]} transmitted {row[4]!r}, expected the install_id" + ) print() if failures: @@ -183,7 +185,7 @@ def main() -> int: print(f" ✗ {failure}") return 1 - print("PASS: every package acknowledged 202 with a derived identifier.") + print("PASS: every package acknowledged 202 with the stable install_id.") print() print("Verify the objects in S3 with the package ids above:") print(" aws s3 ls --recursive " diff --git a/tests/hermes_cli/test_shared_metrics_identity.py b/tests/hermes_cli/test_shared_metrics_identity.py deleted file mode 100644 index 1ea1d95961..0000000000 --- a/tests/hermes_cli/test_shared_metrics_identity.py +++ /dev/null @@ -1,179 +0,0 @@ -"""Tests for keyed pseudonymization of the shared-metrics install identity. - -The load-bearing property: install_id must never be transmitted, and the -value that IS transmitted must stay stable for a package even across a salt -rotation, or a retry would change the body under an already-used package_id. -""" - -from __future__ import annotations - -import sqlite3 -from datetime import datetime, timedelta, timezone - -import pytest - -from hermes_cli.observability.shared_metrics_identity import ( - ROTATION_INTERVAL, - SALT_ISSUED_AT_KEY, - SALT_KEY, - current_salt, - derive_install_id, - substitute_install_id, -) - -INSTALL_ID = "12a73e97-4de9-4766-830d-9ca1192c0420" -T0 = datetime(2026, 8, 26, 12, 0, tzinfo=timezone.utc) - - -@pytest.fixture -def connection(): - conn = sqlite3.connect(":memory:") - conn.execute( - "CREATE TABLE telemetry_state (key TEXT PRIMARY KEY, value TEXT NOT NULL)" - ) - yield conn - conn.close() - - -class TestSaltLifecycle: - def test_first_call_generates_a_salt(self, connection): - salt = current_salt(connection, now=T0) - assert len(salt) == 64 # 32 bytes hex - assert int(salt, 16) >= 0 # valid hex - - def test_salt_is_stable_within_the_window(self, connection): - first = current_salt(connection, now=T0) - later = current_salt(connection, now=T0 + timedelta(days=29, hours=23)) - assert first == later - - def test_salt_rotates_after_the_interval(self, connection): - first = current_salt(connection, now=T0) - after = current_salt(connection, now=T0 + ROTATION_INTERVAL + timedelta(seconds=1)) - assert first != after - - def test_salt_is_persisted(self, connection): - salt = current_salt(connection, now=T0) - stored = connection.execute( - "SELECT value FROM telemetry_state WHERE key = ?", (SALT_KEY,) - ).fetchone()[0] - assert stored == salt - - def test_issued_at_is_recorded(self, connection): - current_salt(connection, now=T0) - stored = connection.execute( - "SELECT value FROM telemetry_state WHERE key = ?", (SALT_ISSUED_AT_KEY,) - ).fetchone()[0] - assert stored.startswith("2026-08-26T12:00") - - def test_two_installs_get_different_salts(self): - salts = set() - for _ in range(5): - conn = sqlite3.connect(":memory:") - conn.execute( - "CREATE TABLE telemetry_state (key TEXT PRIMARY KEY, value TEXT NOT NULL)" - ) - salts.add(current_salt(conn, now=T0)) - conn.close() - assert len(salts) == 5, "salts must be random per install, not derived" - - def test_clock_rollback_reissues_rather_than_trusting_the_stamp(self, connection): - """A future issued_at means the clock moved; the age is unknowable. - - Reissuing is the safe direction — it shortens linkability rather than - extending it, and packages already prepared keep their frozen id. - """ - first = current_salt(connection, now=T0) - rolled_back = current_salt(connection, now=T0 - timedelta(days=5)) - assert rolled_back != first - - def test_corrupt_issued_at_reissues_rather_than_crashing(self, connection): - current_salt(connection, now=T0) - connection.execute( - "UPDATE telemetry_state SET value = 'not-a-date' WHERE key = ?", - (SALT_ISSUED_AT_KEY,), - ) - assert current_salt(connection, now=T0) is not None - - -class TestDerivation: - def test_derivation_is_deterministic(self): - salt = "a" * 64 - assert derive_install_id(INSTALL_ID, salt) == derive_install_id(INSTALL_ID, salt) - - def test_derivation_hides_the_install_id(self): - derived = derive_install_id(INSTALL_ID, "a" * 64) - assert INSTALL_ID not in derived - assert derived != INSTALL_ID - - def test_different_salts_give_different_values(self): - assert derive_install_id(INSTALL_ID, "a" * 64) != derive_install_id( - INSTALL_ID, "b" * 64 - ) - - def test_different_installs_give_different_values(self): - salt = "a" * 64 - assert derive_install_id(INSTALL_ID, salt) != derive_install_id("other", salt) - - def test_output_shape_is_sha256_hex(self): - derived = derive_install_id(INSTALL_ID, "a" * 64) - assert len(derived) == 64 - int(derived, 16) - - -class TestSubstitution: - def _package(self): - return { - "schema_version": "hermes.shared_metrics.v2", - "package_id": "3a63d27e-f170-4d4c-8c4d-ebd80feac592", - "install_id": INSTALL_ID, - "generated_at": "2026-08-26T01:01:25.311956Z", - "period_start": "2026-08-26T00:00:00Z", - "period_end": "2026-08-27T00:00:00Z", - "resource": {"hermes_version": "0.20.5", "os_family": "macos"}, - "metrics": [{"name": "hermes.client.active", "type": "counter", "value": 1}], - } - - def test_install_id_is_replaced(self): - result = substitute_install_id(self._package(), "derived-value") - assert result["install_id"] == "derived-value" - - def test_no_other_field_changes(self): - original = self._package() - result = substitute_install_id(original, "derived-value") - for key in original: - if key != "install_id": - assert result[key] == original[key] - - def test_the_caller_dict_is_not_mutated(self): - original = self._package() - substitute_install_id(original, "derived-value") - assert original["install_id"] == INSTALL_ID - - def test_no_fields_are_added_or_removed(self): - original = self._package() - assert set(substitute_install_id(original, "x")) == set(original) - - def test_the_raw_install_id_never_survives_substitution(self): - import json - - body = json.dumps(substitute_install_id(self._package(), "derived-value")) - assert INSTALL_ID not in body - - -class TestRetryStability: - """The property that keeps retries contract-compliant.""" - - def test_a_frozen_derived_id_survives_a_rotation(self, connection): - salt_before = current_salt(connection, now=T0) - frozen = derive_install_id(INSTALL_ID, salt_before) - - # Time passes, the salt rotates, and the package is retried. - salt_after = current_salt(connection, now=T0 + ROTATION_INTERVAL + timedelta(days=1)) - assert salt_after != salt_before - - # Rebuilding from the FROZEN value reproduces identical bytes; deriving - # afresh would not. - assert substitute_install_id({"install_id": INSTALL_ID}, frozen) == { - "install_id": frozen - } - assert derive_install_id(INSTALL_ID, salt_after) != frozen diff --git a/tests/hermes_cli/test_shared_metrics_sender.py b/tests/hermes_cli/test_shared_metrics_sender.py index c6a7455fe2..cbaff4c9ea 100644 --- a/tests/hermes_cli/test_shared_metrics_sender.py +++ b/tests/hermes_cli/test_shared_metrics_sender.py @@ -374,15 +374,19 @@ class TestConsentGate: class TestIdentity: - def test_install_id_is_never_transmitted(self, store): + def test_the_stable_install_id_is_transmitted_as_is(self, store): + """Product decision 2026-08-27: no pseudonymization. + + The wire body carries the profile-scoped install_id verbatim. This + test is the deliberate inversion of the pre-decision assertion that + the raw id never crossed the wire. + """ _add_package(store, "pkg-1", "2026-08-26") transport = FakeTransport(FakeResponse(202)) _sender(store, transport).send_pending() - raw = transport.calls[0]["payload"].decode("utf-8") - assert INSTALL_ID not in raw - assert transport.bodies[0]["install_id"] != INSTALL_ID + assert transport.bodies[0]["install_id"] == INSTALL_ID - def test_derived_id_is_frozen_on_the_row(self, store): + def test_transmitted_id_is_frozen_on_the_row(self, store): _add_package(store, "pkg-1", "2026-08-26") transport = FakeTransport(FakeResponse(503), FakeResponse(202)) _sender(store, transport).send_pending() diff --git a/tests/hermes_cli/test_shared_metrics_sender_e2e.py b/tests/hermes_cli/test_shared_metrics_sender_e2e.py index 9ff8bf9caf..85be9b2388 100644 --- a/tests/hermes_cli/test_shared_metrics_sender_e2e.py +++ b/tests/hermes_cli/test_shared_metrics_sender_e2e.py @@ -171,12 +171,11 @@ class TestRealTransport: ).fetchone()[0] assert state == "sent" - def test_the_install_id_never_crosses_the_wire(self, store, server): + def test_the_stable_install_id_crosses_the_wire_as_is(self, store, server): + """Product decision 2026-08-27: the raw install_id is transmitted.""" _add(store, "pkg-1", metrics=40) _sender(store, server).send_pending() - body = json.dumps(Ingest.received[0]["body"]) - assert INSTALL_ID not in body - assert len(Ingest.received[0]["body"]["install_id"]) == 64 + assert Ingest.received[0]["body"]["install_id"] == INSTALL_ID def test_content_type_is_json(self, store, server): _add(store, "pkg-1") From 24ecc2a7693d71bf18f71a3bf13cde6de8e2768f Mon Sep 17 00:00:00 2001 From: Ben Barclay Date: Fri, 28 Aug 2026 17:03:22 +1000 Subject: [PATCH 025/634] docs(telemetry): update cli-config.yaml.example for the stable-id decision PR review (andrexibiza, post-a69a9c351d) caught the one operator-facing surface the identity change missed: the example config still promised the profile-scoped ID is NOT sent and described the 30-day rotating HMAC. Rewritten to state the stable install_id is transmitted as-is, matching the sender, wizard, and docs A.2. Swept the repo for further stale references: none remain (the HMAC text in relay-shared-metrics.md A.2/A.3 is the intentional decision record). --- cli-config.yaml.example | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 2fe2c5e610..0675156986 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -1794,11 +1794,11 @@ display: # When sending is on: # * only packages whose period starts on or after the day you opted in are # ever transmitted, so data collected beforehand stays on this machine; -# * the profile-scoped ID is NOT sent. Each package carries an HMAC of it, -# keyed by a local-only salt that rotates every 30 days, so installs stay -# countable without shipping a durable identifier. +# * each package carries the profile-scoped ID as-is. It is a random UUID +# with no hardware, account, or host-derived content, and deleting the +# shared-metrics directory resets it. # See docs/observability/relay-shared-metrics.md (Appendix A) for the full -# consent, identity, rotation, retention, and deletion decisions. +# consent, identity, retention, and deletion decisions. telemetry: shared_metrics: enabled: false From 117e7fef88e18cbc9fe120eeff5f0e3456370f69 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Fri, 28 Aug 2026 12:24:55 -0300 Subject: [PATCH 026/634] fix(nous): surface allowed models the curated list does not carry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An org allowlist can name a model the docs-hosted curated manifest has never heard of. Intersecting the curated list against the reachable set then produced an empty picker — "No models available for Nous Portal after filtering" — which is strictly worse than showing an unfiltered list, because the one model the org may actually use is the one that got dropped. When the reachable set is small enough to be a human-authored allowlist, append whatever it admits that the curated list is missing, after the curated entries so their order survives. Bounded by size, which is what separates the two kinds of policy: an allowlist is small, while a provider-only policy leaves the whole catalog reachable and appending it would bury the curated order. Past the cap the intersection stands alone and the picker's custom-model entry remains the way to reach anything omitted. Co-Authored-By: Claude Opus 5 (1M context) --- hermes_cli/models.py | 30 +++++++++++++++- tests/hermes_cli/test_nous_policy_filter.py | 38 ++++++++++++++++++++- 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/hermes_cli/models.py b/hermes_cli/models.py index c1f7d580f1..4e5f4507a6 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2649,6 +2649,13 @@ def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str] return set(pricing) or None +# Above this many reachable models, an allowed set is treated as catalog-wide +# rather than as an allowlist worth enumerating in a picker. NAS caps an +# allowlist at 512, but a set this large is indistinguishable from the full +# catalog for display purposes. +_NOUS_POLICY_APPEND_MAX = 64 + + def restrict_to_nous_policy( model_ids: list[str], allowed: Optional[set[str]] ) -> list[str]: @@ -2665,12 +2672,33 @@ def restrict_to_nous_policy( """ if not allowed: return list(model_ids) - return [ + kept = [ mid for mid in model_ids if mid in allowed or mid.split(":", 1)[0] in allowed ] + # An allowlist can admit models the curated manifest has never heard of, and + # intersecting alone would then leave the user with nothing to pick at all — + # strictly worse than the unfiltered list. When the reachable set is no + # larger than what would have been shown anyway, it IS the list: append + # whatever it admits that the curated list is missing. + # + # Bounded by size, which is what separates the two kinds of policy: a model + # allowlist is human-authored and small, while a provider-only policy leaves + # the whole catalog reachable. Appending several hundred alphabetical + # vendor-prefixed ids would bury the curated order — the regression the + # pickers' curated branch exists to avoid. Past the cap the intersection + # stands on its own, and the picker's custom-model entry remains the way to + # reach anything it omits. + if len(allowed) <= _NOUS_POLICY_APPEND_MAX: + covered: set[str] = set() + for mid in kept: + covered.add(mid) + covered.add(mid.split(":", 1)[0]) + kept.extend(sorted(a for a in allowed if a not in covered)) + return kept + def get_pricing_for_provider(provider: str, *, force_refresh: bool = False) -> dict[str, dict[str, str]]: """Return live pricing for providers that support it (openrouter, nous, ai-gateway, novita).""" diff --git a/tests/hermes_cli/test_nous_policy_filter.py b/tests/hermes_cli/test_nous_policy_filter.py index edb7e6dc20..7da078c77e 100644 --- a/tests/hermes_cli/test_nous_policy_filter.py +++ b/tests/hermes_cli/test_nous_policy_filter.py @@ -63,7 +63,11 @@ class TestRestrictToNousPolicy: ) == ["vendor/model:free"] def test_drops_a_free_sibling_whose_base_is_blocked(self): - assert restrict_to_nous_policy(["vendor/model:free"], {"other/model"}) == [] + """The blocked sibling goes; the model the org may actually use takes + its place rather than leaving the picker empty.""" + assert restrict_to_nous_policy(["vendor/model:free"], {"other/model"}) == [ + "other/model" + ] class TestNousPolicyAllowedIds: @@ -184,3 +188,35 @@ class TestNousPolicyNotice: notice = account_mod.nous_policy_notice() assert "/" not in notice, f"looks like it names a model: {notice}" assert len(notice.splitlines()) == 1 + + +class TestAllowlistOutsideTheCuratedList: + """An allowlist can name a model the curated manifest has never heard of. + + Intersecting alone leaves the picker empty in that case — strictly worse + than showing an unfiltered list, because the one model the org may use is + the one that got dropped. + """ + + def test_surfaces_an_allowed_model_the_curated_list_lacks(self): + assert restrict_to_nous_policy( + ["vendor/a", "vendor/b"], {"amazon/nova-2-lite-v1"} + ) == ["amazon/nova-2-lite-v1"] + + def test_keeps_curated_order_then_appends_the_rest(self): + kept = restrict_to_nous_policy( + ["z/curated", "a/curated"], {"z/curated", "a/curated", "new/model"} + ) + assert kept == ["z/curated", "a/curated", "new/model"] + + def test_does_not_append_a_free_sibling_already_covered(self): + assert restrict_to_nous_policy(["vendor/m:free"], {"vendor/m"}) == [ + "vendor/m:free" + ] + + def test_a_provider_only_policy_does_not_bury_the_curated_order(self): + """Such a policy leaves the whole catalog reachable; appending it would + drop hundreds of alphabetical ids into the picker.""" + curated = ["vendor/one", "vendor/two"] + catalog = {f"vendor/model-{i}" for i in range(300)} | set(curated) + assert restrict_to_nous_policy(curated, catalog) == curated From bafaac5e61aaa1a30a0db67db638972eb54cba5f Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Fri, 28 Aug 2026 12:26:30 -0300 Subject: [PATCH 027/634] docs(nous): correct the subtract-only claim in the policy plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plan stated the policy set should only ever subtract from a list. That is wrong when an allowlist names a model the curated manifest lacks, which empties the picker instead of narrowing it — the behaviour fixed in 117e7fef88. Co-Authored-By: Claude Opus 5 (1M context) --- docs/nous-org-model-policy.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/nous-org-model-policy.md b/docs/nous-org-model-policy.md index 174f65fec7..942c798507 100644 --- a/docs/nous-org-model-policy.md +++ b/docs/nous-org-model-policy.md @@ -135,8 +135,16 @@ entry the surface already populates, so no surface makes an extra request. **Do not** replace a list with the response's keys. Every surface shows the curated agentic list in curated order deliberately — the live catalog is a large alphabetical dump of vendor-prefixed models, and swapping it in is the -regression `model_switch.py:3070` records. Recommendations should be able to -*reveal* a newly launched model; the policy set should only ever subtract. +regression `model_switch.py:3070` records. + +**Do not** treat the policy set as subtract-only either. An allowlist can name +a model the curated manifest has never heard of, and intersecting alone then +empties the picker — strictly worse than an unfiltered list, because the one +model the org may use is the one dropped. When the reachable set is small +enough to be a human-authored allowlist, append what it admits that the +curated list lacks, after the curated entries so their order survives. Bound +it by size: a provider-only policy leaves the whole catalog reachable, and +appending that would bury the curated order. **Do not** narrow a list on evidence that cannot support it. `nous_policy_allowed_ids()` returns `None` — meaning "leave the list alone" — From 04647f15c88a7dfdaf0c9b060b85b519d0835d50 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Fri, 28 Aug 2026 12:39:29 -0300 Subject: [PATCH 028/634] fix(nous): only fall back to the reachable set when the overlap is empty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surfacing allowed models the curated list lacks was gated on the size of the reachable set alone. A jurisdiction or provider policy leaves few enough models to pass that cap, so it appended the remainder — pushing non-curated alphabetical ids into a picker that shows a curated order on purpose, and making the list long enough that the non-curses fallback's input prompt scrolled off screen and read as a hang. Gate on the intersection instead. The fallback exists for an allowlist that names nothing curated, which is the empty-overlap case; a policy that merely narrows the catalog keeps the curated overlap and needs no help. The size cap stays as a guard on that one path. Co-Authored-By: Claude Opus 5 (1M context) --- docs/nous-org-model-policy.md | 18 ++++++----- hermes_cli/models.py | 35 +++++++++------------ tests/hermes_cli/test_nous_policy_filter.py | 17 ++++++++-- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/docs/nous-org-model-policy.md b/docs/nous-org-model-policy.md index 942c798507..7b4027ba72 100644 --- a/docs/nous-org-model-policy.md +++ b/docs/nous-org-model-policy.md @@ -138,13 +138,17 @@ large alphabetical dump of vendor-prefixed models, and swapping it in is the regression `model_switch.py:3070` records. **Do not** treat the policy set as subtract-only either. An allowlist can name -a model the curated manifest has never heard of, and intersecting alone then -empties the picker — strictly worse than an unfiltered list, because the one -model the org may use is the one dropped. When the reachable set is small -enough to be a human-authored allowlist, append what it admits that the -curated list lacks, after the curated entries so their order survives. Bound -it by size: a provider-only policy leaves the whole catalog reachable, and -appending that would bury the curated order. +only models the curated manifest has never heard of, and intersecting alone +then empties the picker — strictly worse than an unfiltered list, because the +models the org may use are the ones dropped. Fall back to the reachable set +itself in exactly that case. + +Only when the intersection is empty, and only when the set is small enough to +be an allowlist rather than a whole catalog. A jurisdiction or provider policy +narrows the catalog without emptying the curated overlap; appending its +remainder pushes non-curated alphabetical ids into a picker that shows a +curated order on purpose. A size cap alone does not catch this — a region +filter can leave few enough models to pass it. **Do not** narrow a list on evidence that cannot support it. `nous_policy_allowed_ids()` returns `None` — meaning "leave the list alone" — diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 4e5f4507a6..e7754fa196 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2650,9 +2650,9 @@ def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str] # Above this many reachable models, an allowed set is treated as catalog-wide -# rather than as an allowlist worth enumerating in a picker. NAS caps an -# allowlist at 512, but a set this large is indistinguishable from the full -# catalog for display purposes. +# rather than as an allowlist worth showing in place of an empty picker. NAS +# caps an allowlist at 512, but a set this large is indistinguishable from the +# full catalog for display purposes. _NOUS_POLICY_APPEND_MAX = 64 @@ -2678,25 +2678,18 @@ def restrict_to_nous_policy( if mid in allowed or mid.split(":", 1)[0] in allowed ] - # An allowlist can admit models the curated manifest has never heard of, and - # intersecting alone would then leave the user with nothing to pick at all — - # strictly worse than the unfiltered list. When the reachable set is no - # larger than what would have been shown anyway, it IS the list: append - # whatever it admits that the curated list is missing. + # An allowlist can admit only models the curated manifest has never heard + # of, leaving nothing to intersect and an empty picker — strictly worse than + # the unfiltered list, because the models the org may actually use are the + # ones dropped. Fall back to the reachable set itself in exactly that case. # - # Bounded by size, which is what separates the two kinds of policy: a model - # allowlist is human-authored and small, while a provider-only policy leaves - # the whole catalog reachable. Appending several hundred alphabetical - # vendor-prefixed ids would bury the curated order — the regression the - # pickers' curated branch exists to avoid. Past the cap the intersection - # stands on its own, and the picker's custom-model entry remains the way to - # reach anything it omits. - if len(allowed) <= _NOUS_POLICY_APPEND_MAX: - covered: set[str] = set() - for mid in kept: - covered.add(mid) - covered.add(mid.split(":", 1)[0]) - kept.extend(sorted(a for a in allowed if a not in covered)) + # Only when the intersection is empty. A jurisdiction or provider policy + # narrows the catalog without emptying the curated overlap, and appending + # its remainder would push non-curated alphabetical ids into a picker that + # shows a curated order on purpose. Anything omitted is still reachable + # through the picker's custom-model entry. + if not kept and len(allowed) <= _NOUS_POLICY_APPEND_MAX: + return sorted(allowed) return kept diff --git a/tests/hermes_cli/test_nous_policy_filter.py b/tests/hermes_cli/test_nous_policy_filter.py index 7da078c77e..b91fe10148 100644 --- a/tests/hermes_cli/test_nous_policy_filter.py +++ b/tests/hermes_cli/test_nous_policy_filter.py @@ -203,11 +203,24 @@ class TestAllowlistOutsideTheCuratedList: ["vendor/a", "vendor/b"], {"amazon/nova-2-lite-v1"} ) == ["amazon/nova-2-lite-v1"] - def test_keeps_curated_order_then_appends_the_rest(self): + def test_does_not_append_when_the_curated_overlap_is_non_empty(self): + """A jurisdiction or provider policy narrows the catalog without + emptying the curated overlap. Appending its remainder would push + non-curated alphabetical ids into a deliberately curated order.""" kept = restrict_to_nous_policy( ["z/curated", "a/curated"], {"z/curated", "a/curated", "new/model"} ) - assert kept == ["z/curated", "a/curated", "new/model"] + assert kept == ["z/curated", "a/curated"] + + def test_jurisdiction_policy_never_grows_the_list(self): + """Regression: a region filter leaves few enough models to slip under + the size cap, so a size-only guard let it append.""" + curated = ["vendor/one", "vendor/two", "vendor/three"] + reachable = {"vendor/one", "vendor/two"} | {f"cn/model-{i}" for i in range(20)} + assert restrict_to_nous_policy(curated, reachable) == [ + "vendor/one", + "vendor/two", + ] def test_does_not_append_a_free_sibling_already_covered(self): assert restrict_to_nous_policy(["vendor/m:free"], {"vendor/m"}) == [ From da3c2435e2ad850469d8b2031a74585b850d2f94 Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Fri, 28 Aug 2026 13:34:27 -0300 Subject: [PATCH 029/634] fix(nous): only rescue an empty list where emptiness means "filtered out" The fallback also ran on unavailable_models, which is legitimately empty on a paid tier, filling the picker with the whole reachable set. Make it opt-in. --- docs/nous-org-model-policy.md | 17 ++++--- hermes_cli/auth.py | 4 +- hermes_cli/model_setup_flows.py | 4 +- hermes_cli/model_switch.py | 4 +- hermes_cli/models.py | 22 +++++---- hermes_cli/web_server.py | 4 +- tests/hermes_cli/test_nous_policy_filter.py | 49 ++++++++++++++++----- 7 files changed, 74 insertions(+), 30 deletions(-) diff --git a/docs/nous-org-model-policy.md b/docs/nous-org-model-policy.md index 7b4027ba72..8b1cbc4573 100644 --- a/docs/nous-org-model-policy.md +++ b/docs/nous-org-model-policy.md @@ -143,12 +143,17 @@ then empties the picker — strictly worse than an unfiltered list, because the models the org may use are the ones dropped. Fall back to the reachable set itself in exactly that case. -Only when the intersection is empty, and only when the set is small enough to -be an allowlist rather than a whole catalog. A jurisdiction or provider policy -narrows the catalog without emptying the curated overlap; appending its -remainder pushes non-curated alphabetical ids into a picker that shows a -curated order on purpose. A size cap alone does not catch this — a region -filter can leave few enough models to pass it. +Only when the intersection is empty, only when the set is small enough to be +an allowlist rather than a whole catalog, and **only for the list a user picks +from**. Callers opt in per list. Any list whose emptiness carries meaning must +not get the rescue: a paid-tier user's unavailable list is legitimately empty, +and rescuing it reads that as "nothing survived" and fills the picker's +unavailable block with the entire reachable set. + +A size cap alone does not make this safe — a jurisdiction filter can leave few +enough models to pass it — and neither does gating on an empty intersection, +because an intentionally empty input is indistinguishable from a fully filtered +one. The opt-in is what separates them. **Do not** narrow a list on evidence that cannot support it. `nous_policy_allowed_ids()` returns `None` — meaning "leave the list alone" — diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index eebbc5609c..c6a7330e29 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -9432,7 +9432,9 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: # unauthenticated, so neither knows what the org may reach. # Narrow both lists to the policy before they are shown. _policy_allowed = nous_policy_allowed_ids() - model_ids = restrict_to_nous_policy(model_ids, _policy_allowed) + model_ids = restrict_to_nous_policy( + model_ids, _policy_allowed, rescue_empty=True, + ) unavailable_models = restrict_to_nous_policy( unavailable_models, _policy_allowed, ) diff --git a/hermes_cli/model_setup_flows.py b/hermes_cli/model_setup_flows.py index 9489692159..c4b9dedfd7 100644 --- a/hermes_cli/model_setup_flows.py +++ b/hermes_cli/model_setup_flows.py @@ -565,7 +565,9 @@ def _model_flow_nous(config, current_model="", args=None): from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy _policy_allowed = nous_policy_allowed_ids() - model_ids = restrict_to_nous_policy(model_ids, _policy_allowed) + model_ids = restrict_to_nous_policy( + model_ids, _policy_allowed, rescue_empty=True, + ) unavailable_models = restrict_to_nous_policy(unavailable_models, _policy_allowed) if not model_ids and not unavailable_models: diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index 3516cdfe24..fe18cb93a6 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -3111,7 +3111,9 @@ def list_authenticated_providers( restrict_to_nous_policy as _nous_restrict, ) - model_ids = _nous_restrict(model_ids, _nous_policy()) + model_ids = _nous_restrict( + model_ids, _nous_policy(), rescue_empty=True, + ) except Exception: pass else: diff --git a/hermes_cli/models.py b/hermes_cli/models.py index e7754fa196..6662b9d3c8 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2657,7 +2657,10 @@ _NOUS_POLICY_APPEND_MAX = 64 def restrict_to_nous_policy( - model_ids: list[str], allowed: Optional[set[str]] + model_ids: list[str], + allowed: Optional[set[str]], + *, + rescue_empty: bool = False, ) -> list[str]: """*model_ids* narrowed to *allowed*, preserving the caller's order. @@ -2681,14 +2684,17 @@ def restrict_to_nous_policy( # An allowlist can admit only models the curated manifest has never heard # of, leaving nothing to intersect and an empty picker — strictly worse than # the unfiltered list, because the models the org may actually use are the - # ones dropped. Fall back to the reachable set itself in exactly that case. + # ones dropped. *rescue_empty* falls back to the reachable set in exactly + # that case, and callers opt in per list: it is meaningful for the list a + # user picks from, and wrong for any list whose emptiness carries meaning. + # An unavailable/gated list is legitimately empty, and rescuing it would + # read that as "nothing survived" and fill it with the whole reachable set. # - # Only when the intersection is empty. A jurisdiction or provider policy - # narrows the catalog without emptying the curated overlap, and appending - # its remainder would push non-curated alphabetical ids into a picker that - # shows a curated order on purpose. Anything omitted is still reachable - # through the picker's custom-model entry. - if not kept and len(allowed) <= _NOUS_POLICY_APPEND_MAX: + # Bounded, because a jurisdiction or provider policy narrows the catalog + # without shrinking it to an allowlist, and a large alphabetical dump buries + # the curated order the pickers show on purpose. Anything omitted stays + # reachable through the picker's custom-model entry. + if rescue_empty and not kept and len(allowed) <= _NOUS_POLICY_APPEND_MAX: return sorted(allowed) return kept diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 41577df7d4..97c5dfe9e2 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -7520,7 +7520,9 @@ def get_recommended_default_model(provider: str = ""): # Neither the curated list nor the Portal's recommendations know # what the org may reach, and this endpoint picks the model a user # lands on without choosing it. - model_ids = restrict_to_nous_policy(model_ids, nous_policy_allowed_ids()) + model_ids = restrict_to_nous_policy( + model_ids, nous_policy_allowed_ids(), rescue_empty=True, + ) model = pick_silent_default_model(model_ids, provider="nous") return {"provider": "nous", "model": model, "free_tier": bool(free_tier)} diff --git a/tests/hermes_cli/test_nous_policy_filter.py b/tests/hermes_cli/test_nous_policy_filter.py index b91fe10148..b488ca2367 100644 --- a/tests/hermes_cli/test_nous_policy_filter.py +++ b/tests/hermes_cli/test_nous_policy_filter.py @@ -63,11 +63,7 @@ class TestRestrictToNousPolicy: ) == ["vendor/model:free"] def test_drops_a_free_sibling_whose_base_is_blocked(self): - """The blocked sibling goes; the model the org may actually use takes - its place rather than leaving the picker empty.""" - assert restrict_to_nous_policy(["vendor/model:free"], {"other/model"}) == [ - "other/model" - ] + assert restrict_to_nous_policy(["vendor/model:free"], {"other/model"}) == [] class TestNousPolicyAllowedIds: @@ -200,7 +196,7 @@ class TestAllowlistOutsideTheCuratedList: def test_surfaces_an_allowed_model_the_curated_list_lacks(self): assert restrict_to_nous_policy( - ["vendor/a", "vendor/b"], {"amazon/nova-2-lite-v1"} + ["vendor/a", "vendor/b"], {"amazon/nova-2-lite-v1"}, rescue_empty=True ) == ["amazon/nova-2-lite-v1"] def test_does_not_append_when_the_curated_overlap_is_non_empty(self): @@ -208,7 +204,9 @@ class TestAllowlistOutsideTheCuratedList: emptying the curated overlap. Appending its remainder would push non-curated alphabetical ids into a deliberately curated order.""" kept = restrict_to_nous_policy( - ["z/curated", "a/curated"], {"z/curated", "a/curated", "new/model"} + ["z/curated", "a/curated"], + {"z/curated", "a/curated", "new/model"}, + rescue_empty=True, ) assert kept == ["z/curated", "a/curated"] @@ -217,19 +215,46 @@ class TestAllowlistOutsideTheCuratedList: the size cap, so a size-only guard let it append.""" curated = ["vendor/one", "vendor/two", "vendor/three"] reachable = {"vendor/one", "vendor/two"} | {f"cn/model-{i}" for i in range(20)} - assert restrict_to_nous_policy(curated, reachable) == [ + assert restrict_to_nous_policy(curated, reachable, rescue_empty=True) == [ "vendor/one", "vendor/two", ] def test_does_not_append_a_free_sibling_already_covered(self): - assert restrict_to_nous_policy(["vendor/m:free"], {"vendor/m"}) == [ - "vendor/m:free" - ] + assert restrict_to_nous_policy( + ["vendor/m:free"], {"vendor/m"}, rescue_empty=True + ) == ["vendor/m:free"] def test_a_provider_only_policy_does_not_bury_the_curated_order(self): """Such a policy leaves the whole catalog reachable; appending it would drop hundreds of alphabetical ids into the picker.""" curated = ["vendor/one", "vendor/two"] catalog = {f"vendor/model-{i}" for i in range(300)} | set(curated) - assert restrict_to_nous_policy(curated, catalog) == curated + assert restrict_to_nous_policy(curated, catalog, rescue_empty=True) == curated + + +class TestRescueIsOptIn: + """The empty-intersection rescue is meaningful only for the list a user + picks from. Any list whose emptiness carries meaning must not get it.""" + + def test_no_rescue_by_default(self): + assert restrict_to_nous_policy([], {"a/one", "b/two"}) == [] + + def test_rescue_only_when_asked(self): + assert restrict_to_nous_policy( + [], {"a/one"}, rescue_empty=True + ) == ["a/one"] + + def test_an_already_empty_unavailable_list_is_never_filled(self): + """Regression: a paid-tier user has no gated models, so the + unavailable list is legitimately empty. Rescuing it read that as + "nothing survived" and pushed the whole reachable set into the picker's + unavailable block.""" + reachable = {f"cn/model-{i}" for i in range(42)} + assert restrict_to_nous_policy([], reachable) == [] + + def test_rescue_does_not_resurrect_a_fully_blocked_list(self): + """A list whose every entry was blocked is a real filter result, not a + signal to show something else — unless the caller asked for the + rescue, which only the selectable list does.""" + assert restrict_to_nous_policy(["x/blocked"], {"y/allowed"}) == [] From a51df3864ea336c82ae40405eca19a8ce6e97fde Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Fri, 28 Aug 2026 15:39:01 -0300 Subject: [PATCH 030/634] refactor(nous): trim comments and drop an unused field --- docs/nous-org-model-policy.md | 286 ---------------------------------- 1 file changed, 286 deletions(-) delete mode 100644 docs/nous-org-model-policy.md diff --git a/docs/nous-org-model-policy.md b/docs/nous-org-model-policy.md deleted file mode 100644 index 8b1cbc4573..0000000000 --- a/docs/nous-org-model-policy.md +++ /dev/null @@ -1,286 +0,0 @@ -# Honouring the Nous org model policy in the pickers - -> **Audience:** Contributors touching Nous model selection -> **Source files:** `hermes_cli/auth.py` (`_login_nous`, `fetch_nous_models`, -> `_prompt_model_selection`), `hermes_cli/models.py` (`fetch_models_with_pricing`, -> `get_pricing_for_provider`, `union_with_portal_*`, `partition_nous_models_by_tier`), -> `hermes_cli/model_setup_flows.py` (`_model_flow_nous`), -> `hermes_cli/model_switch.py` (`list_authenticated_providers`), -> `hermes_cli/web_server.py` (`/api/model/recommended-default`), -> `hermes_cli/nous_account.py` (`_info_from_valid_jwt`) -> **Related:** Inference gateway PR #164 (filters `GET /v1/models` by org policy), -> NAS #941 (team admins restrict providers), NAS `openrouter-provider-map` -> (publishes the model→providers map the gateway filter needs) - -## What changed upstream - -A Nous team admin can restrict which models and which serving providers their -org may use. The inference gateway applies that policy to `GET /v1/models`, so -an authenticated catalog read returns only what the caller may actually reach. -Blocked models are **omitted** — the row is skipped, no marker field is added -(`api/src/handlers/models.ts:99-138`). An anonymous read is still allowed and -still returns the full catalog (`api/src/app.ts:309-313` — no auth middleware -on the route). - -Two things bound how urgent this is. - -**The gateway is authoritative and this is cosmetic.** Asking for a hidden -model is refused at request time with `403 model_blocked_by_org_policy` -(`api/src/middleware/model_entitlement_gate.ts:337-356`). The listing fails -open; the request gate fails closed. Nothing here is a security boundary — the -cost of a wrong list is a predictable 403, and the gateway PR states that -tradeoff deliberately. This document is only about the client showing the -right list. - -**It is inert today.** PR #164 is merged, but is switched off until NAS -publishes the policy fields and the provider map, and the admin surface sits -behind the `org-model-policy` Vercel flag. The `openrouter-provider-map` branch -is the publisher half (a daily cron writing `openrouter_model_providers` to the -entitlement Redis). Until that lands, every caller — anonymous and -authenticated — gets the same unfiltered list. **No change here is verifiable -end to end yet; every test mocks the filtered response.** - -## Where we stand - -Four surfaces list Nous models. **None of them is filtered.** - -| surface | builds its list from | filtered | -| --- | --- | --- | -| Login (`_login_nous`, `auth.py:9383`) | `get_curated_nous_model_ids()` ∪ Portal recommendations | no | -| `hermes model` (`_model_flow_nous`, `model_setup_flows.py:399`) | same | no | -| `/model` picker (`list_authenticated_providers`, `model_switch.py:3062`) | same | no | -| Dashboard onboarding (`web_server.py:7486`) | same | no | - -All four seed from the docs-hosted manifest and union the Portal's -`recommended-models` endpoint. Neither source is authenticated, so org policy -has no effect on any list a user picks from. - -`cached_provider_model_ids("nous")` — which *does* reach the authenticated -`fetch_nous_models` — is not consulted by any of them. The `/model` picker -handles nous in its own branch that deliberately bypasses it, and nous cannot -reach the generic pathway at `model_switch.py:2898` because line 2861 skips -every non-`api_key` provider. Its only caller for nous is the background -prefetch (`model_switch.py:2390`), which writes an entry nothing reads. - -Two things that are already fine, and should stay that way: - -- `nous` is **not** in `_MODELS_DEV_PREFERRED`, so no models.dev entries are - merged on top of the live list. -- The nous fallback ladder in `provider_model_ids` is a *chain* (live → - manifest → in-repo snapshot), not a merge, so a successful live fetch is - used exclusively. - ---- - -## Fix 0 — put auth state in the pricing cache key - -**This is a prerequisite for fix 1, and worth landing on its own merits.** - -**Problem.** `fetch_models_with_pricing` caches on the base URL alone, and the -cache check happens *above* the point where the `Authorization` header is built -(`models.py:2404`): - -```python -cache_key = (base_url or "").rstrip("/") -if not force_refresh: - cached = _cached_catalog(cache_key) - if cached is not None: - return cached -... -if api_key: - headers["Authorization"] = f"Bearer {api_key}" -``` - -`_pricing_cache` is process-lifetime with no expiry for a non-empty result -(`models.py:2231-2253`). So whichever read of a given base URL lands first — -authenticated or anonymous — answers every later read in that process, -whatever key it passes. An anonymous read landing first (the auxiliary-model -path in fix 2 is one) makes a later authenticated read return an unfiltered -list without touching the network. A fix built on this cache looks like it -works and does not. - -**Do.** Fold auth state into the cache key. Distinguishing authenticated from -anonymous is enough — the token value need not be in the key, and keeping it -out avoids hashing a secret. - -**Do** update `agent/credits_tracker.py:257`, which reaches into the private -`_pricing_cache` dict assuming one entry per base URL. - -**Test.** An anonymous read followed by an authenticated read of the same base -URL issues two requests and returns two different lists. Independently -testable today, unlike everything below. - -## Fix 1 — narrow each list to the org's policy - -**Problem.** All four surfaces build their list from -`get_curated_nous_model_ids()` unioned with the Portal's `recommended-models` -endpoint. Neither is authenticated, so org policy has no effect on the model a -user picks — which is the model they then use. The Portal endpoint compounds -it: it takes no auth and no parameters, returns one globally CDN-cached payload -for the whole platform, and is invalidated only by admin pricing edits — never -by a policy change. It can put a hidden model straight back into a list. There -is no policy-aware variant of it and no parameter that would make one. - -Each surface, however, already fetches `/v1/models`. -`get_pricing_for_provider("nous")` calls `fetch_models_with_pricing`, which -reads that endpoint and returns `{model_id: {...}}`, and already resolves -credentials (`_resolve_nous_pricing_credentials`), so it is already the -authenticated read. Its keys are the reachable set. - -**Do.** Use that set to *narrow* each list, keeping the curated order. -`nous_policy_allowed_ids()` obtains the set; `restrict_to_nous_policy()` -applies it. Both live in `models.py`, and the fetch reuses the pricing cache -entry the surface already populates, so no surface makes an extra request. - -**Do not** replace a list with the response's keys. Every surface shows the -curated agentic list in curated order deliberately — the live catalog is a -large alphabetical dump of vendor-prefixed models, and swapping it in is the -regression `model_switch.py:3070` records. - -**Do not** treat the policy set as subtract-only either. An allowlist can name -only models the curated manifest has never heard of, and intersecting alone -then empties the picker — strictly worse than an unfiltered list, because the -models the org may use are the ones dropped. Fall back to the reachable set -itself in exactly that case. - -Only when the intersection is empty, only when the set is small enough to be -an allowlist rather than a whole catalog, and **only for the list a user picks -from**. Callers opt in per list. Any list whose emptiness carries meaning must -not get the rescue: a paid-tier user's unavailable list is legitimately empty, -and rescuing it reads that as "nothing survived" and fills the picker's -unavailable block with the entire reachable set. - -A size cap alone does not make this safe — a jurisdiction filter can leave few -enough models to pass it — and neither does gating on an empty intersection, -because an intentionally empty input is indistinguishable from a fully filtered -one. The opt-in is what separates them. - -**Do not** narrow a list on evidence that cannot support it. -`nous_policy_allowed_ids()` returns `None` — meaning "leave the list alone" — -in three cases, and each matters: - -- **The org has no policy, or the token is too old to say.** Gated on the - `policy_present` claim (fix 4). For an unrestricted org — the common case — - filtering buys nothing and risks dropping a Portal recommendation the - gateway catalog has not caught up on yet. This keeps the change a no-op for - everyone the policy does not apply to. -- **Credential resolution failed**, so the read was anonymous and therefore - unfiltered. A full catalog must not be mistaken for a filtered one. A stated - degradation, not a silent one. -- **The read came back empty**, which is a fetch failure, not an org that may - reach nothing. - -A `:free` sibling is kept when its base model is reachable, mirroring the -gateway, which admits a row when any of its requestable ids passes and treats -anything unknown as a keep — "over-listing costs a 403 from the authoritative -gate, while hiding a row the gate would serve is unrecoverable from the client" -(`api/src/libs/catalog_policy.ts:74-78`). Prefer over-listing here too. - -**Test.** With a policy hiding model X: X is absent from each of the four -lists, and no surface makes more Nous requests than it does today. With no -policy, with credentials broken, or with an empty read, every list is byte-for- -byte what it is today. A model the Portal flags as free but the org hides stays -out; curated ordering survives filtering. - -## Fix 2 — audit the other readers of the pricing map - -**Problem.** `fetch_models_with_pricing` is shared, so any caller that treats -its keys as "the models that exist" inherits whatever authentication the first -caller happened to have. Fix 0 stops the *authentication* from leaking between -callers; this fix is about which callers may treat the map as a source of ids -at all. - -**Do.** Make the map a lookup *for* ids already in the list, never a source of -ids. Two consumers are already correct and should stay that way: -`partition_nous_models_by_tier` only looks up ids it was given, and the -`union_with_portal_*` pair only ever writes into the map — their id-widening -comes from the Portal endpoint (fix 2), not from the map. - -The one that is wrong is `agent/auxiliary_client.py:869-908` -(`_fast_model_from_catalog`), which iterates the map's keys directly as its -candidate list off an anonymous read. Reachable for nous on the titling path, -where it can select a policy-hidden model that then 403s at request time. - -**Test.** With credentials broken so the read falls back to anonymous, no -list grows. - -## Fix 3 — stop prefetching the nous catalog - -**Problem.** The background prefetch calls -`cached_provider_model_ids("nous", force_refresh=True)` -(`model_switch.py:2390`); nous is collected into it because -`_collect_authed_provider_slugs` treats any `auth.json` providers entry as -credentials regardless of `auth_type` (`model_switch.py:2519-2526`). Because -`force_refresh=True` skips the cache read and no nous surface reads the entry, -this is a live authenticated `/v1/models` round trip per picker open written to -a location nothing consults. - -**Do.** Exclude nous from the prefetch and delete the write-only entry. - -This replaces what an earlier draft proposed here — folding `org_id` into -`_credential_fingerprint` and shortening `_PROVIDER_MODELS_STALE_SERVE_MAX` -for nous (a single global constant, `models.py:4204`, with no per-provider -branching today). Both would have hardened a cache that, after fix 1, has no -nous readers to protect. If a future surface routes nous through -`cached_provider_model_ids` again, revisit the fingerprint then: it hashes -env-var values and `auth.json` mtime and carries no org signal -(`models.py:4277`), so two orgs on one machine can serve each other's list. - -**Test.** Opening the `/model` picker makes no Nous `/v1/models` request beyond -the one the displayed list is built from. - -## Fix 4 — the `policy_present` claim - -**Problem.** Under omission a blocked model simply vanishes, which reads as -"Hermes does not support this" rather than "your org disallows it". - -**Do.** Read the `policy_present` claim off the Nous OAuth access token and, -when it is `true`, show a single line stating that the org restricts which -models are available. No enumeration, no per-model marking. - -The claim rides the same JWT as `org_id` -(`access-token-issuer.ts:552,595`, `token_use: "access"`) — the token the -client already decodes — and `_info_from_valid_jwt` already retains every -claim in `raw_claims` (`nous_account.py:600-647`), so surfacing it is one -typed field on `NousPortalAccountInfo` and no new request. - -It is already widened to cover provider-only restrictions, not just model -allowlists (`nous-account-service/src/server/entitlement-snapshot.ts:478-480`). -Two NAS docs still describe it as allowlist-only and list the widening as -pending — they are stale; trust that expression. - -**Do not** enumerate the blocked set. Model policy is allowlist-only — -`denyModels` is a dead column (`nous-account-service/src/server/model-policy.ts:230`) -— so an org that allows five models blocks the entire rest of the catalog. -Graying hundreds of rows is a worse UI than omitting them. An earlier draft -proposed deriving the blocked set by diffing the anonymous and authenticated -reads and feeding it to `_prompt_model_selection`'s `unavailable_models`; that -is the wrong shape twice over, because that picker carries one -`unavailable_message` for the whole list and cannot say "free-tier-gated" and -"policy-hidden" at once. - -**Do not** report the absence of the claim as the absence of a policy. It is -tri-state: `true`, `false`, and absent, where absent means unknown — an older -mint, not an unrestricted org. The gateway rejects a corrupt (non-boolean) -claim outright rather than reading it as "no policy" -(`api/src/middleware/nas_jwt_auth.ts:179`). Show the line only on `true`. - -**Known bound:** the claim is stamped at mint time, so it goes stale until the -next token refresh — the line can lag a policy change by up to the access -token's lifetime. Acceptable, and worth stating rather than rediscovering. - -**Test.** With `policy_present` true the line shows; with it false or absent it -does not. - ---- - -## Order - -Fix 0 first: fix 1 is silently wrong without it, and it is the only piece -testable before NAS switches the feature on. Fix 4's claim gates fix 1, so the -two land together. Fix 1 is the correctness work — without it the policy is -bypassed on every surface a user picks from. Fix 2 keeps the pricing map from -becoming another way to widen a list. Fix 3 is a deletion that fix 1 makes -safe. - -Run tests with `scripts/run_tests.sh` — not bare `pytest`. From 4d482ed344bc0ab507841cfc5efc4cb6bc4cfd0e Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Fri, 28 Aug 2026 15:39:23 -0300 Subject: [PATCH 031/634] refactor(nous): trim comments and drop an unused field --- agent/auxiliary_client.py | 11 ++- agent/credits_tracker.py | 5 +- hermes_cli/auth.py | 5 +- hermes_cli/model_setup_flows.py | 5 +- hermes_cli/model_switch.py | 14 ++-- hermes_cli/models.py | 78 +++++++------------ hermes_cli/nous_account.py | 29 ++----- hermes_cli/web_server.py | 5 +- tests/hermes_cli/test_nous_policy_filter.py | 53 ++++--------- tests/hermes_cli/test_nous_policy_surfaces.py | 24 ++---- .../hermes_cli/test_pricing_cache_auth_key.py | 14 +--- 11 files changed, 76 insertions(+), 167 deletions(-) diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 534b42184a..e3c87068f1 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -885,10 +885,9 @@ def _fast_model_from_catalog(provider_id: str) -> str: logger.debug("No credentials for %s catalog", provider_id, exc_info=True) if not api_key and provider_id.strip().lower() == "nous": - # Nous is OAuth, so the api-key resolver above raises for it. An - # anonymous read returns the full catalog rather than the one the - # org may reach, and a model picked from it is refused at request - # time with model_blocked_by_org_policy. + # Nous is OAuth, so the resolver above raises for it. An anonymous + # read returns the full catalog, and a model picked from it is + # refused at request time by the org's policy. try: from hermes_cli.models import _resolve_nous_pricing_credentials @@ -913,8 +912,8 @@ def _fast_model_from_catalog(provider_id: str) -> str: ids = sorted((str(m) for m in catalog), key=_model_recency_key, reverse=True) if provider_id.strip().lower() == "nous": - # The catalog's keys are a source of ids here, so the policy has to - # narrow them the same way it narrows the pickers' lists. + # The catalog's keys are a source of ids here, so the policy narrows + # them as it does the pickers' lists. try: from hermes_cli.models import ( nous_policy_allowed_ids, diff --git a/agent/credits_tracker.py b/agent/credits_tracker.py index 2d0873c563..82e2d53caa 100644 --- a/agent/credits_tracker.py +++ b/agent/credits_tracker.py @@ -254,10 +254,7 @@ def is_free_tier_model(model: str, base_url: str = "") -> bool: try: from hermes_cli.models import _is_model_free, peek_cached_pricing - # The agent's Nous base_url is /v1-suffixed - # (https://inference-api.nousresearch.com/v1) but the catalog fetchers - # key on the pre-/v1 root, and on auth state besides; peek_cached_pricing - # owns both details. + # peek_cached_pricing owns the /v1-suffix and auth-state key details. pricing = peek_cached_pricing(base_url) if not pricing: return False diff --git a/hermes_cli/auth.py b/hermes_cli/auth.py index c6a7330e29..6d5bfb8ae1 100644 --- a/hermes_cli/auth.py +++ b/hermes_cli/auth.py @@ -9428,9 +9428,8 @@ def _login_nous(args, pconfig: ProviderConfig) -> None: model_ids, pricing = union_with_portal_paid_recommendations( model_ids, pricing, _portal_for_recs, ) - # The curated list and the Portal's recommendations are both - # unauthenticated, so neither knows what the org may reach. - # Narrow both lists to the policy before they are shown. + # Neither the curated list nor the Portal's recommendations + # know what the org may reach. _policy_allowed = nous_policy_allowed_ids() model_ids = restrict_to_nous_policy( model_ids, _policy_allowed, rescue_empty=True, diff --git a/hermes_cli/model_setup_flows.py b/hermes_cli/model_setup_flows.py index c4b9dedfd7..90c9dd38d2 100644 --- a/hermes_cli/model_setup_flows.py +++ b/hermes_cli/model_setup_flows.py @@ -559,9 +559,8 @@ def _model_flow_nous(config, current_model="", args=None): model_ids, pricing, _nous_portal_url, ) - # The curated list and the Portal's recommendations are both - # unauthenticated, so neither knows what the org may reach. Narrow both - # lists to the policy before they are shown. + # Neither the curated list nor the Portal's recommendations know what the + # org may reach. from hermes_cli.models import nous_policy_allowed_ids, restrict_to_nous_policy _policy_allowed = nous_policy_allowed_ids() diff --git a/hermes_cli/model_switch.py b/hermes_cli/model_switch.py index fe18cb93a6..fa7f4c928f 100644 --- a/hermes_cli/model_switch.py +++ b/hermes_cli/model_switch.py @@ -2565,11 +2565,9 @@ def _collect_authed_provider_slugs( slugs.append(_cp.slug) seen.add(_cp.slug.lower()) - # Nous is deliberately excluded. Its picker branch builds from the curated - # list rather than cached_provider_model_ids, and nous cannot reach the - # api_key-only unified pathway, so a prefetched entry is written and never - # read — a live authenticated /v1/models round trip per picker open for - # nothing. + # Nous excluded: its picker branch builds from the curated list and it + # cannot reach the api_key-only pathway, so a prefetched entry is written + # and never read. return [s for s in slugs if s != "nous"] @@ -3101,10 +3099,8 @@ def list_authenticated_providers( # curated list alone (still correct, just may lag newly # launched models, exactly like an offline CLI run). pass - # Both the curated list and the Portal's recommendations are - # unauthenticated, so neither knows what the org may reach. Narrow - # to the policy outside the try, so a failed recommendation fetch - # still yields a filtered curated list. + # Outside the try above, so a failed recommendation fetch still + # yields a policy-filtered curated list. try: from hermes_cli.models import ( nous_policy_allowed_ids as _nous_policy, diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 6662b9d3c8..9c81afb132 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2255,18 +2255,16 @@ def _cache_catalog( return result -# A governed endpoint answers an authenticated read with a policy-filtered -# catalog and an anonymous read with the full one, so auth state is part of the -# cache identity. NUL cannot appear in a URL, so the suffix cannot collide with -# a base URL that happens to end this way. +# NUL cannot appear in a URL, so this cannot collide with a real base URL. _PRICING_AUTH_KEY_SUFFIX = "\x00auth" def _pricing_cache_key(url_root: str, api_key: str | None) -> str: - """The ``_pricing_cache`` key for a read of *url_root*. + """Cache key for a read of *url_root*. - Only *whether* a key was supplied participates — never its value, so no - secret reaches the cache key. + A governed endpoint answers an authenticated read with a policy-filtered + catalog and an anonymous one with the full catalog, so the two cannot share + an entry. Only whether a key was supplied participates, never its value. """ return url_root + _PRICING_AUTH_KEY_SUFFIX if api_key else url_root @@ -2274,9 +2272,8 @@ def _pricing_cache_key(url_root: str, api_key: str | None) -> str: def peek_cached_pricing(base_url: str) -> dict[str, dict[str, Any]]: """Pricing already cached for *base_url*, or ``{}``. Never fetches. - Accepts a ``/v1``-suffixed URL as well as the pre-``/v1`` root the - catalog fetchers key on. Prefers the authenticated catalog, which is the - one scoped to the caller's org. + Accepts a ``/v1``-suffixed URL as well as the pre-``/v1`` root the fetchers + key on, and prefers the authenticated catalog. """ root = (base_url or "").rstrip("/") if root.endswith("/v1"): @@ -2607,23 +2604,13 @@ def _resolve_nous_pricing_credentials() -> tuple[str, str]: def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str]]: """The Nous model ids the caller's org may reach, or ``None`` to not filter. - The gateway filters ``GET /v1/models`` by the org's model policy for an - authenticated read, omitting blocked rows with no marker field, so the keys - of the authenticated pricing response are the reachable set. This reuses - that response rather than issuing a second round trip. + The gateway omits policy-blocked rows from an authenticated + ``GET /v1/models``, so that response's keys are the reachable set. - Returns ``None`` — meaning "leave the caller's list alone" — in three cases, - each of which would otherwise narrow a list on evidence that cannot support - it: - - * the org carries no policy, or the token is too old to say (see - :func:`~hermes_cli.nous_account.nous_policy_present`). Filtering an - unrestricted org's list buys nothing and risks dropping a model the - Portal recommends before the gateway catalog lists it. - * credential resolution failed, so the read is anonymous and therefore - unfiltered. A full catalog must not be mistaken for a policy-filtered one. - * the read came back empty, which is a fetch failure rather than an org - that may reach nothing. + ``None`` means "leave the caller's list alone", for the three states that + cannot support narrowing one: no policy (or a token too old to say), an + anonymous read whose catalog is unfiltered, and an empty read, which is a + fetch failure rather than an org that may reach nothing. """ try: from hermes_cli.nous_account import nous_policy_present @@ -2638,8 +2625,8 @@ def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str] return None # Same arguments as get_pricing_for_provider's nous branch, so a caller - # that also asks for pricing shares this cache entry instead of paying for - # a second request. + # asking for pricing too shares this entry instead of paying for a second + # request. pricing = fetch_models_with_pricing( api_key=api_key, base_url=base_url, @@ -2649,10 +2636,8 @@ def nous_policy_allowed_ids(*, force_refresh: bool = False) -> Optional[set[str] return set(pricing) or None -# Above this many reachable models, an allowed set is treated as catalog-wide -# rather than as an allowlist worth showing in place of an empty picker. NAS -# caps an allowlist at 512, but a set this large is indistinguishable from the -# full catalog for display purposes. +# Past this size an allowed set reads as a whole catalog rather than an +# allowlist, and is not worth showing in place of an empty picker. _NOUS_POLICY_APPEND_MAX = 64 @@ -2664,14 +2649,12 @@ def restrict_to_nous_policy( ) -> list[str]: """*model_ids* narrowed to *allowed*, preserving the caller's order. - A ``None`` or empty *allowed* leaves the list untouched — see - :func:`nous_policy_allowed_ids` for when that happens. + A ``None`` or empty *allowed* leaves the list untouched. - A ``:free`` sibling is kept when its base model is reachable. The gateway - admits a row when any of its requestable ids passes, and treats anything - unknown as a keep on the grounds that over-listing costs a 403 from the - authoritative gate while hiding a row the gate would serve is unrecoverable - from the client. This mirrors that. + A ``:free`` sibling is kept when its base model is reachable, mirroring the + gateway, which admits a row when any of its requestable ids passes. Prefer + over-listing: that costs a 403 from the authoritative gate, while hiding a + row the gate would serve is unrecoverable from the client. """ if not allowed: return list(model_ids) @@ -2681,19 +2664,10 @@ def restrict_to_nous_policy( if mid in allowed or mid.split(":", 1)[0] in allowed ] - # An allowlist can admit only models the curated manifest has never heard - # of, leaving nothing to intersect and an empty picker — strictly worse than - # the unfiltered list, because the models the org may actually use are the - # ones dropped. *rescue_empty* falls back to the reachable set in exactly - # that case, and callers opt in per list: it is meaningful for the list a - # user picks from, and wrong for any list whose emptiness carries meaning. - # An unavailable/gated list is legitimately empty, and rescuing it would - # read that as "nothing survived" and fill it with the whole reachable set. - # - # Bounded, because a jurisdiction or provider policy narrows the catalog - # without shrinking it to an allowlist, and a large alphabetical dump buries - # the curated order the pickers show on purpose. Anything omitted stays - # reachable through the picker's custom-model entry. + # An allowlist can name only models the curated manifest lacks, leaving an + # empty picker — worse than no filter, since the models the org may use are + # the ones dropped. Opt-in per list: an already-empty list (a paid tier's + # gated models) means "nothing to gate", not "nothing survived". if rescue_empty and not kept and len(allowed) <= _NOUS_POLICY_APPEND_MAX: return sorted(allowed) return kept diff --git a/hermes_cli/nous_account.py b/hermes_cli/nous_account.py index c63d090fc4..6eba71c831 100644 --- a/hermes_cli/nous_account.py +++ b/hermes_cli/nous_account.py @@ -99,7 +99,6 @@ class NousPortalAccountInfo: subscription: Optional[NousPortalSubscriptionInfo] = None paid_service_access: Optional[bool] = None paid_service_access_info: Optional[NousPaidServiceAccessInfo] = None - policy_present: Optional[bool] = None tool_access: Optional[NousToolAccessInfo] = None raw_claims: Optional[dict[str, Any]] = None raw_account: Optional[dict[str, Any]] = None @@ -400,17 +399,12 @@ def get_nous_portal_account_info( def nous_policy_present() -> Optional[bool]: """Whether the caller's org carries a restrictive model/provider policy. - Read from the ``policy_present`` claim on the Nous OAuth access token, so - this costs no request. ``/api/oauth/account`` does not carry the claim, - which is why this reads the token directly rather than going through - :func:`get_nous_portal_account_info`. + Reads the ``policy_present`` claim off the access token, so it costs no + request; ``/api/oauth/account`` does not carry it. Stamped at mint time, so + it goes stale until the next token refresh. - ``None`` means unknown — an older mint, an unreadable token, or a - non-boolean claim. Unknown is NOT "no policy": callers must not report the - absence of the claim as the absence of a restriction. - - The claim is stamped at mint time, so it goes stale until the next token - refresh. + ``None`` is unknown — an older mint or an unreadable claim — and must not be + reported as the absence of a policy. """ try: from hermes_cli.auth import get_provider_auth_state, _decode_jwt_claims @@ -430,15 +424,9 @@ def nous_policy_present() -> Optional[bool]: def nous_policy_notice() -> str: """A one-line notice for an org that restricts model choice, else ``""``. - Under the gateway's policy filter a blocked model is omitted rather than - marked, which reads as "Hermes does not support this" instead of "your org - disallows it". This says which it is without enumerating anything: model - policy is an allowlist, so an org that admits a handful of models blocks - the whole rest of the catalog, and listing those would be a worse UI than - omitting them. - - Silent unless the claim is explicitly true — absent means an older mint, - not an unrestricted org. + A blocked model is omitted rather than marked, which reads as "Hermes does + not support this". This says which it is without enumerating the blocked + set, which under an allowlist is most of the catalog. """ if nous_policy_present() is not True: return "" @@ -694,7 +682,6 @@ def _info_from_valid_jwt( expires_at=datetime.fromtimestamp(exp, tz=timezone.utc), paid_service_access=paid_access, paid_service_access_info=access_info, - policy_present=_coerce_bool(claims.get("policy_present")), tool_access=_tool_access_from_value(claims.get("tool_access")), raw_claims=dict(claims), ) diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 97c5dfe9e2..ec3aae53ec 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -7517,9 +7517,8 @@ def get_recommended_default_model(provider: str = ""): model_ids, pricing, portal_url ) - # Neither the curated list nor the Portal's recommendations know - # what the org may reach, and this endpoint picks the model a user - # lands on without choosing it. + # This endpoint picks the model a user lands on without choosing + # it, so an unreachable one here is worse than in a picker. model_ids = restrict_to_nous_policy( model_ids, nous_policy_allowed_ids(), rescue_empty=True, ) diff --git a/tests/hermes_cli/test_nous_policy_filter.py b/tests/hermes_cli/test_nous_policy_filter.py index b488ca2367..35a0de704e 100644 --- a/tests/hermes_cli/test_nous_policy_filter.py +++ b/tests/hermes_cli/test_nous_policy_filter.py @@ -1,10 +1,7 @@ """Narrowing the Nous model lists to an org's policy. -The inference gateway omits policy-blocked rows from an authenticated -``GET /v1/models`` with no marker field, so the keys of the authenticated -catalog read are the reachable set. These helpers turn that into a filter the -pickers can apply without a second round trip, and — just as importantly — -decline to filter when the evidence cannot support it. +The gateway omits policy-blocked rows from an authenticated ``GET /v1/models``, +so that response's keys are the reachable set. """ from __future__ import annotations @@ -44,15 +41,12 @@ class TestRestrictToNousPolicy: ) == ["a/one", "c/three"] def test_preserves_curated_order(self): - """The pickers show a curated order deliberately; filtering must not - reorder it into the catalog's alphabetical order.""" curated = ["z/last", "a/first", "m/middle"] allowed = {"a/first", "m/middle", "z/last"} assert restrict_to_nous_policy(curated, allowed) == curated def test_keeps_a_free_sibling_when_its_base_is_reachable(self): - """Portal free recommendations are ``:free`` ids; the gateway admits a - row when any of its requestable ids passes.""" + """Portal free recommendations are ``:free`` ids.""" assert restrict_to_nous_policy(["vendor/model:free"], {"vendor/model"}) == [ "vendor/model:free" ] @@ -115,8 +109,7 @@ class TestNousPolicyAllowedIds: assert calls == [] def test_declines_to_filter_on_an_anonymous_read(self, monkeypatch): - """An anonymous read returns the full catalog; treating it as the - policy-filtered set would silently widen the list to everything.""" + """An anonymous read returns the full, unfiltered catalog.""" self._patch(monkeypatch, policy_present=True, api_key="", pricing={"a/one": {}}) assert nous_policy_allowed_ids() is None @@ -146,7 +139,6 @@ class TestNousPolicyPresent: assert nous_policy_present() is None def test_non_boolean_claim_is_unknown(self, monkeypatch): - """The gateway refuses to read a corrupt claim as "no policy".""" self._patch_token(monkeypatch, _jwt({"policy_present": "yes"})) assert nous_policy_present() is None @@ -160,8 +152,6 @@ class TestNousPolicyPresent: class TestNousPolicyNotice: - """A governed org is told its choice is restricted, rather than left to - read an omitted model as one Hermes does not support.""" def _patch(self, monkeypatch, present): monkeypatch.setattr(account_mod, "nous_policy_present", lambda: present) @@ -172,14 +162,12 @@ class TestNousPolicyNotice: @pytest.mark.parametrize("present", [False, None]) def test_silent_otherwise(self, monkeypatch, present): - """Absent is an older mint, not an unrestricted org — either way there - is nothing truthful to say.""" + """Absent is an older mint, not an unrestricted org.""" self._patch(monkeypatch, present) assert account_mod.nous_policy_notice() == "" def test_names_no_models(self, monkeypatch): - """Policy is an allowlist, so the blocked set is most of the catalog; - the notice must not try to enumerate it.""" + """The blocked set is most of the catalog under an allowlist.""" self._patch(monkeypatch, True) notice = account_mod.nous_policy_notice() assert "/" not in notice, f"looks like it names a model: {notice}" @@ -187,12 +175,8 @@ class TestNousPolicyNotice: class TestAllowlistOutsideTheCuratedList: - """An allowlist can name a model the curated manifest has never heard of. - - Intersecting alone leaves the picker empty in that case — strictly worse - than showing an unfiltered list, because the one model the org may use is - the one that got dropped. - """ + """An allowlist can name only models the curated manifest lacks, which + intersecting alone turns into an empty picker.""" def test_surfaces_an_allowed_model_the_curated_list_lacks(self): assert restrict_to_nous_policy( @@ -200,9 +184,6 @@ class TestAllowlistOutsideTheCuratedList: ) == ["amazon/nova-2-lite-v1"] def test_does_not_append_when_the_curated_overlap_is_non_empty(self): - """A jurisdiction or provider policy narrows the catalog without - emptying the curated overlap. Appending its remainder would push - non-curated alphabetical ids into a deliberately curated order.""" kept = restrict_to_nous_policy( ["z/curated", "a/curated"], {"z/curated", "a/curated", "new/model"}, @@ -211,8 +192,8 @@ class TestAllowlistOutsideTheCuratedList: assert kept == ["z/curated", "a/curated"] def test_jurisdiction_policy_never_grows_the_list(self): - """Regression: a region filter leaves few enough models to slip under - the size cap, so a size-only guard let it append.""" + """A region filter can slip under the size cap, so the cap alone is not + enough of a guard.""" curated = ["vendor/one", "vendor/two", "vendor/three"] reachable = {"vendor/one", "vendor/two"} | {f"cn/model-{i}" for i in range(20)} assert restrict_to_nous_policy(curated, reachable, rescue_empty=True) == [ @@ -226,16 +207,13 @@ class TestAllowlistOutsideTheCuratedList: ) == ["vendor/m:free"] def test_a_provider_only_policy_does_not_bury_the_curated_order(self): - """Such a policy leaves the whole catalog reachable; appending it would - drop hundreds of alphabetical ids into the picker.""" curated = ["vendor/one", "vendor/two"] catalog = {f"vendor/model-{i}" for i in range(300)} | set(curated) assert restrict_to_nous_policy(curated, catalog, rescue_empty=True) == curated class TestRescueIsOptIn: - """The empty-intersection rescue is meaningful only for the list a user - picks from. Any list whose emptiness carries meaning must not get it.""" + """The rescue is meaningful only for the list a user picks from.""" def test_no_rescue_by_default(self): assert restrict_to_nous_policy([], {"a/one", "b/two"}) == [] @@ -246,15 +224,10 @@ class TestRescueIsOptIn: ) == ["a/one"] def test_an_already_empty_unavailable_list_is_never_filled(self): - """Regression: a paid-tier user has no gated models, so the - unavailable list is legitimately empty. Rescuing it read that as - "nothing survived" and pushed the whole reachable set into the picker's - unavailable block.""" + """A paid tier has no gated models, so this list is legitimately + empty — not a filter result to rescue.""" reachable = {f"cn/model-{i}" for i in range(42)} assert restrict_to_nous_policy([], reachable) == [] def test_rescue_does_not_resurrect_a_fully_blocked_list(self): - """A list whose every entry was blocked is a real filter result, not a - signal to show something else — unless the caller asked for the - rescue, which only the selectable list does.""" assert restrict_to_nous_policy(["x/blocked"], {"y/allowed"}) == [] diff --git a/tests/hermes_cli/test_nous_policy_surfaces.py b/tests/hermes_cli/test_nous_policy_surfaces.py index e59974c511..1fd43725d3 100644 --- a/tests/hermes_cli/test_nous_policy_surfaces.py +++ b/tests/hermes_cli/test_nous_policy_surfaces.py @@ -1,9 +1,7 @@ """Every Nous model list is narrowed to the org's policy before it is shown. -Four surfaces build a Nous list from the curated manifest unioned with the -Portal's ``recommended-models`` endpoint. Neither source is authenticated, so -without this filter an org's hidden model is offered to the user and then -refused at request time with ``model_blocked_by_org_policy``. +Four surfaces build their list from the curated manifest unioned with the +Portal's ``recommended-models`` endpoint; neither source is authenticated. """ from __future__ import annotations @@ -32,7 +30,6 @@ def no_policy(monkeypatch): class TestLoginNous: - """``_login_nous`` — the model picked at login is the model then used.""" def _run(self, monkeypatch, tmp_path): import hermes_cli.auth as auth_mod @@ -119,8 +116,7 @@ class TestModelSwitchPicker: assert set(CURATED) <= set(row["models"]) def test_filter_survives_a_failed_recommendation_fetch(self, monkeypatch, policy): - """The filter sits outside the try that wraps the Portal union, so a - Portal outage still yields a policy-filtered curated list.""" + """The filter sits outside the try wrapping the Portal union.""" def _boom(_p): raise RuntimeError("portal down") @@ -132,8 +128,7 @@ class TestModelSwitchPicker: class TestRecommendedDefaultEndpoint: - """``GET /api/model/recommended-default`` picks a model the user never sees - chosen, so an unreachable one there is worse than in a picker.""" + """This endpoint picks a model the user never sees chosen.""" def _call(self, monkeypatch): import hermes_cli.auth as auth_mod @@ -163,8 +158,7 @@ class TestRecommendedDefaultEndpoint: class TestAuxiliaryFastModel: - """``_fast_model_from_catalog`` treats the catalog's keys as a source of - ids, so an anonymous read there can select a model the gateway refuses.""" + """``_fast_model_from_catalog`` uses the catalog's keys as a source of ids.""" def _pick(self, monkeypatch, *, catalog): import agent.auxiliary_client as aux @@ -184,8 +178,7 @@ class TestAuxiliaryFastModel: return picked, seen def test_reads_the_catalog_with_nous_oauth_credentials(self, monkeypatch, no_policy): - """The api-key resolver raises for OAuth providers; without a fallback - the read goes out anonymous and returns the unfiltered catalog.""" + """The api-key resolver raises for OAuth providers.""" _, seen = self._pick(monkeypatch, catalog=["vendor/haiku-fast"]) assert seen["api_key"] == "sk-nous" @@ -204,8 +197,8 @@ class TestAuxiliaryFastModel: class TestNousPrefetch: - """The nous disk-cache entry is write-only: its picker branch builds from - the curated list, so prefetching it is a round trip for nothing.""" + """The nous disk-cache entry is write-only, so prefetching it is a round + trip for nothing.""" def test_nous_is_not_collected_for_prefetch(self, monkeypatch): import hermes_cli.auth as auth_mod @@ -220,7 +213,6 @@ class TestNousPrefetch: class TestPolicyNoticeIsShown: - """The notice reaches the two flows where a user picks a model.""" def test_login_prints_it(self, monkeypatch, tmp_path, policy, capsys): import hermes_cli.nous_account as account_mod diff --git a/tests/hermes_cli/test_pricing_cache_auth_key.py b/tests/hermes_cli/test_pricing_cache_auth_key.py index a1ae120365..d67c8d6e3a 100644 --- a/tests/hermes_cli/test_pricing_cache_auth_key.py +++ b/tests/hermes_cli/test_pricing_cache_auth_key.py @@ -1,10 +1,8 @@ """``_pricing_cache`` keys on auth state, not just the base URL. -A governed endpoint (Nous ``/v1/models`` filtered by an org's model policy) -answers an authenticated read with a narrower catalog than an anonymous one. -Keyed on the base URL alone, whichever read landed first in a process answered -every later one — so an authenticated caller could be handed the full, -unfiltered catalog without a request going out. +Nous ``/v1/models`` answers an authenticated read with a policy-filtered +catalog and an anonymous one with the full catalog, so the two must not share +a cache entry. """ from __future__ import annotations @@ -60,8 +58,6 @@ def catalog(monkeypatch): def test_authenticated_read_is_not_answered_by_an_anonymous_one(catalog): - """The bug: an anonymous read landing first must not answer the next - authenticated read out of cache.""" anon = fetch_models_with_pricing(api_key="", base_url=BASE) authed = fetch_models_with_pricing(api_key="sk-test", base_url=BASE) @@ -72,7 +68,6 @@ def test_authenticated_read_is_not_answered_by_an_anonymous_one(catalog): def test_anonymous_read_is_not_answered_by_an_authenticated_one(catalog): - """And the reverse direction, so neither entry can shadow the other.""" authed = fetch_models_with_pricing(api_key="sk-test", base_url=BASE) anon = fetch_models_with_pricing(api_key="", base_url=BASE) @@ -108,12 +103,11 @@ class TestPeekCachedPricing: assert peek_cached_pricing(BASE) == {} def test_accepts_a_v1_suffixed_url(self, catalog): - """The agent holds a /v1-suffixed base URL; the fetchers key on the root.""" + """The agent holds a /v1-suffixed base URL; fetchers key on the root.""" fetch_models_with_pricing(api_key="sk-test", base_url=BASE) assert sorted(peek_cached_pricing(BASE + "/v1")) == sorted(_FILTERED) def test_prefers_the_authenticated_catalog(self, catalog): - """It is the one scoped to the caller's org.""" fetch_models_with_pricing(api_key="", base_url=BASE) fetch_models_with_pricing(api_key="sk-test", base_url=BASE) assert sorted(peek_cached_pricing(BASE)) == sorted(_FILTERED) From 705a10850d7fcf3db04f6a83432ab41ff8755aba Mon Sep 17 00:00:00 2001 From: Mariano Nicolini Date: Fri, 28 Aug 2026 17:00:26 -0300 Subject: [PATCH 032/634] refactor(nous): trim comments and drop unused code --- hermes_cli/models.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 9c81afb132..e2bd3a6682 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -2259,16 +2259,6 @@ def _cache_catalog( _PRICING_AUTH_KEY_SUFFIX = "\x00auth" -def _pricing_cache_key(url_root: str, api_key: str | None) -> str: - """Cache key for a read of *url_root*. - - A governed endpoint answers an authenticated read with a policy-filtered - catalog and an anonymous one with the full catalog, so the two cannot share - an entry. Only whether a key was supplied participates, never its value. - """ - return url_root + _PRICING_AUTH_KEY_SUFFIX if api_key else url_root - - def peek_cached_pricing(base_url: str) -> dict[str, dict[str, Any]]: """Pricing already cached for *base_url*, or ``{}``. Never fetches. @@ -2434,7 +2424,11 @@ def fetch_models_with_pricing( ``original``. """ url_root = (base_url or "").rstrip("/") - cache_key = _pricing_cache_key(url_root, api_key) + # A governed endpoint answers an authenticated read with a policy-filtered + # catalog and an anonymous one with the full catalog, so the two cannot + # share an entry. Only whether a key was supplied participates, never its + # value. + cache_key = url_root + _PRICING_AUTH_KEY_SUFFIX if api_key else url_root if not force_refresh: cached = _cached_catalog(cache_key) if cached is not None: From ff3835a630deb1f03054806d91ae5712b76f16d1 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 30 Aug 2026 18:03:45 -0700 Subject: [PATCH 033/634] fix(gateway): compact the live codex thread instead of no-op mirror rewrites (#73503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the codex_app_server runtime the model's real working context is the app-server's server-side thread: CodexAppServerSession is constructed with no history and each turn submits only the new user message (agent/codex_runtime.py), so Hermes' transcript is a mirror that is never replayed into a thread. Every out-of-turn compression call site (gateway session hygiene, gateway /compress) built a DETACHED agent whose _codex_session was None, so the codex route bailed at its "no active codex thread" guard and returned the transcript unchanged ("compressed 150 -> 150 msgs") — and hygiene's finally-clause then evicted the cached live agent, destroying the only real context: the next turn spawned an empty thread while Hermes still mirrored a full history. Fix, per the documented compression.codex_app_server_auto contract: * Session hygiene now routes codex_app_server sessions to run_codex_hygiene_compaction(): in 'hermes' mode it compacts the LIVE cached agent's thread via thread/compact/start (through the existing codex route in _compress_context) and KEEPS that agent cached; 'native' and 'off' skip cleanly with no eviction and no local fallback. A wedged compaction records the persistent failure cooldown; success resets the hygiene failure streak. * Gateway /compress detects the codex_app_server runtime before building a temporary compression agent and compacts the live thread with force=True instead (a manual compress is an explicit user decision in every mode). No live thread -> honest "nothing to compact" reply instead of a mirror rewrite plus eviction. * No mode ever runs the local transcript compressor on this runtime: rewriting the mirror cannot shrink the thread, so the #73715-style local fallback (including its force=True leak into native/off) is deliberately not adopted. Diagnosis of the mode-gate/no-thread deadlock builds on PR #73715. Closes #73503 Co-authored-by: webtecnica --- gateway/run.py | 175 ++++++++- gateway/slash_commands.py | 78 ++++ .../gateway/test_codex_hygiene_compaction.py | 338 ++++++++++++++++++ 3 files changed, 590 insertions(+), 1 deletion(-) create mode 100644 tests/gateway/test_codex_hygiene_compaction.py diff --git a/gateway/run.py b/gateway/run.py index 2a6a54f96c..a7a6191bf7 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -314,6 +314,135 @@ def _hygiene_compression_timeout_message( ) +async def run_codex_hygiene_compaction( + gateway, + session_key: str, + session_id: str, + *, + auto_mode: str, + history: list, + approx_tokens: int, + timeout_seconds: float, + failure_cooldown_seconds: float = 300.0, +) -> str: + """Session hygiene for ``codex_app_server`` sessions (#73503). + + On this runtime the model's real working context is the app-server's + server-side thread, not Hermes' transcript: ``CodexAppServerSession`` is + constructed with no history and each turn submits only the new user + message (agent/codex_runtime.py), so the persisted transcript is a mirror + that is never replayed into a thread. Two consequences drive this path: + + * Rewriting the local mirror (the detached hygiene agent's normal + compression) shrinks nothing the model actually carries — it was a + permanent no-op ("compressed 150 -> 150 msgs"). + * Evicting the cached live agent afterwards destroys the only real + context: the next turn spawns an EMPTY thread and the model starts + blank while Hermes still mirrors a full history (abrupt amnesia — the + user-facing damage documented on #73503). + + So hygiene must compact the LIVE cached agent's thread via the + app-server's own ``thread/compact/start`` (through + ``_compress_context_via_codex_app_server``) and KEEP that agent cached. + Never build a detached compressor and never evict here. + + Mode contract (``compression.codex_app_server_auto``): only ``hermes`` + lets Hermes' threshold initiate app-server compaction; ``native`` leaves + the schedule to codex itself and ``off`` disables Hermes-initiated + automatic compaction entirely — both return without touching the thread + or the transcript, and neither may fall back to the local compressor. + + Returns an outcome tag for logging/tests: ``compacted``, + ``skipped:`` or ``failed:``. + """ + mode = str(auto_mode or "native").lower() + if mode not in {"native", "hermes", "off"}: + mode = "native" + if mode != "hermes": + # native: the app-server compacts on its own schedule; off: the + # operator disabled Hermes-initiated automatic compaction. A local + # transcript fallback is wrong in EVERY mode here (it cannot shrink + # the thread), so both modes are a clean skip — crucially without + # the detached-compressor path's cache eviction. + return f"skipped:mode={mode}" + + agent = None + lock = getattr(gateway, "_agent_cache_lock", None) + cache = getattr(gateway, "_agent_cache", None) + if cache is not None: + try: + if lock: + with lock: + entry = cache.get(session_key) + else: + entry = cache.get(session_key) + except Exception: + entry = None + agent = entry[0] if isinstance(entry, tuple) and entry else entry + if agent is None or agent is _AGENT_PENDING_SENTINEL: + # No live agent → no live thread → nothing real to compact. The + # mirror-only rewrite the detached path would perform is exactly the + # no-op this function exists to remove, so skip honestly instead. + return "skipped:no-cached-agent" + if getattr(agent, "_codex_session", None) is None: + return "skipped:no-live-thread" + + loop = asyncio.get_running_loop() + compressor = getattr(agent, "context_compressor", None) + count_before = getattr(compressor, "compression_count", 0) + try: + await asyncio.wait_for( + loop.run_in_executor( + None, + lambda: agent._compress_context( + history, + "", + approx_tokens=approx_tokens, + ), + ), + timeout=max(float(timeout_seconds), 1.0), + ) + except asyncio.TimeoutError: + # The executor thread keeps running (compact_thread has its own RPC + # timeouts); brake per-turn retries so a wedged app-server does not + # re-trigger a compaction attempt on every message. + if failure_cooldown_seconds >= 0: + _record_hygiene_cooldown( + gateway, + session_id, + failure_cooldown_seconds, + "codex app-server thread compaction timed out", + ) + logger.warning( + "Session hygiene: codex app-server thread compaction for " + "session %s timed out after %.1fs; continuing without compaction", + session_id, + timeout_seconds, + ) + return "failed:timeout" + except Exception as exc: + logger.warning( + "Session hygiene: codex app-server thread compaction for " + "session %s failed: %s", + session_id, + exc, + ) + return f"failed:{exc}" + + count_after = getattr(compressor, "compression_count", 0) + if count_after > count_before: + # A native compaction boundary was recorded on the live agent + # (thread compacted server-side; transcript intentionally NOT + # rewritten — state.db records the boundary, the mirror stays + # intact and the agent stays cached). + _reset_hygiene_failure_streak(gateway, session_key) + return "compacted" + # compress_context returned without recording a boundary: an internal + # skip (its own failure cooldown) or a compaction error — the codex + # route already persisted its own failure cooldown in that case. + return "failed:no-boundary" + + def _record_hygiene_cooldown( gateway, session_id: str, @@ -20512,7 +20641,51 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew session_key=session_key, user_config=_hyg_data if isinstance(_hyg_data, dict) else None, ) - if _hyg_runtime.get("api_key"): + _hyg_api_mode = str( + _hyg_runtime.get("api_mode") or "" + ).lower() + if _hyg_api_mode == "codex_app_server": + # codex app-server runtime: the model's real + # context is the app-server's server-side thread, + # not the transcript mirror. The detached-agent + # block below could only rewrite the mirror (a + # guaranteed no-op for the thread) and its + # finally-clause eviction would destroy the live + # thread — the next turn then starts blank + # (#73503). Route to the live cached agent's + # thread/compact/start instead and KEEP it cached. + _hyg_codex_auto = "native" + _hyg_comp_cfg = ( + _hyg_data.get("compression") + if isinstance(_hyg_data, dict) + else None + ) + if isinstance(_hyg_comp_cfg, dict): + _hyg_codex_auto = str( + _hyg_comp_cfg.get( + "codex_app_server_auto", "native" + ) + or "native" + ) + _hyg_codex_outcome = await run_codex_hygiene_compaction( + self, + session_key, + session_entry.session_id, + auto_mode=_hyg_codex_auto, + history=history, + approx_tokens=_approx_tokens, + timeout_seconds=_hyg_total_ceiling_seconds, + failure_cooldown_seconds=_hyg_failure_cooldown_seconds, + ) + logger.info( + "Session hygiene (codex app-server): %s " + "(session=%s, mode=%s, ~%s tokens)", + _hyg_codex_outcome, + session_entry.session_id, + _hyg_codex_auto, + f"{_approx_tokens:,}", + ) + elif _hyg_runtime.get("api_key"): # Pass the FULL transcript (tool results included). # Filtering to user/assistant-only starved the # compressor: tool results are usually the bulk of diff --git a/gateway/slash_commands.py b/gateway/slash_commands.py index bd86f0d96d..2bf7e55797 100644 --- a/gateway/slash_commands.py +++ b/gateway/slash_commands.py @@ -4449,6 +4449,67 @@ class GatewaySlashCommandsMixin: with _profile_runtime_scope(profile_home): return await self._handle_compress_command_inner(event) + async def _compress_codex_app_server_session( + self, session_key: str, session_id: str + ) -> str: + """Manual /compress for codex_app_server sessions (#73503). + + Compacts the LIVE cached agent's app-server thread via + ``thread/compact/start`` (through ``_compress_context``'s codex route + with ``force=True``, which bypasses the automatic-mode gate in every + ``compression.codex_app_server_auto`` mode — a manual /compress is an + explicit user decision) and keeps that agent cached, so the compacted + thread is what the next turn continues from. Never builds a temporary + compression agent and never rewrites the transcript mirror: neither + can shrink the server-side thread that is the model's real context. + """ + agent = None + lock = getattr(self, "_agent_cache_lock", None) + cache = getattr(self, "_agent_cache", None) + if cache is not None: + if lock: + with lock: + entry = cache.get(session_key) + else: + entry = cache.get(session_key) + agent = entry[0] if isinstance(entry, tuple) and entry else entry + from gateway.run import _AGENT_PENDING_SENTINEL + + if ( + agent is None + or agent is _AGENT_PENDING_SENTINEL + or getattr(agent, "_codex_session", None) is None + ): + return ( + "🗜️ Nothing to compact: this session runs on the Codex " + "app-server runtime, whose context lives in a Codex-owned " + "thread that only exists while the agent is active. Send a " + "message first, then /compress — or /reset to start fresh." + ) + + compressor = getattr(agent, "context_compressor", None) + count_before = getattr(compressor, "compression_count", 0) + try: + await self._run_in_executor_with_context( + lambda: agent._compress_context( + [], "", force=True, + ) + ) + except Exception as exc: + return t("gateway.compress.failed", error=exc) + count_after = getattr(compressor, "compression_count", 0) + if count_after > count_before: + return ( + "🗜️ Codex app-server thread compacted (thread/compact). " + "The transcript mirror is unchanged by design — the " + "app-server now carries the compacted context." + ) + return ( + "⚠️ Codex app-server compaction did not complete — the thread " + "is unchanged. Check the app-server logs, retry /compress, or " + "/reset for a clean session." + ) + async def _handle_compress_command_inner(self, event: MessageEvent) -> str: """Handle /compress command -- manually compress conversation context. @@ -4538,6 +4599,23 @@ class GatewaySlashCommandsMixin: source=source, session_key=session_key, ) + if str(runtime_kwargs.get("api_mode") or "").lower() == "codex_app_server": + # codex app-server runtime (#73503): the model's working + # context is the app-server's server-side thread, owned by the + # LIVE cached agent (agent/codex_runtime.py — one + # CodexAppServerSession per AIAgent, spawned lazily on first + # turn). A temporary compression agent has no thread, so the + # codex route in _compress_context_via_codex_app_server bailed + # at its "no active codex thread" guard, the transcript came + # back unchanged, and the finally-clause eviction below then + # destroyed the only real context. Compact the live agent's + # thread via thread/compact/start instead — and KEEP the agent + # cached so the compacted thread survives to the next turn. + # No local transcript fallback in any mode: rewriting the + # mirror cannot shrink the thread. + return await self._compress_codex_app_server_session( + session_key, session_entry.session_id + ) if not runtime_kwargs.get("api_key"): return t("gateway.compress.no_provider") diff --git a/tests/gateway/test_codex_hygiene_compaction.py b/tests/gateway/test_codex_hygiene_compaction.py new file mode 100644 index 0000000000..71dd907f3b --- /dev/null +++ b/tests/gateway/test_codex_hygiene_compaction.py @@ -0,0 +1,338 @@ +"""Regression tests for #73503 — codex_app_server compression must not be a no-op. + +On the codex_app_server runtime the model's real working context is the +app-server's server-side thread (CodexAppServerSession is constructed with no +history and each turn submits only the new user message), so: + +* the old detached-agent hygiene path could only rewrite the transcript + mirror — a guaranteed no-op ("compressed 150 -> 150 msgs") — and then + evicted the live cached agent, destroying the thread that held the only + real context; +* the fix routes hygiene and manual /compress to the LIVE cached agent's + ``thread/compact/start`` (asserted here at the compact_thread RPC-stub + boundary) and keeps that agent cached; +* ``compression.codex_app_server_auto`` semantics hold: only ``hermes`` + lets Hermes' threshold start a compaction; ``native``/``off`` skip + cleanly, and no mode ever runs the local transcript compressor. +""" + +import asyncio +from types import SimpleNamespace + +import pytest + +from agent.conversation_compression import compress_context +from agent.transports.codex_app_server_session import TurnResult +from gateway.run import run_codex_hygiene_compaction + + +class FakeCodexSession: + """RPC-stub boundary: stands in for the app-server thread client.""" + + def __init__(self, result=None): + self.result = result or TurnResult( + thread_id="thread-1", turn_id="compact-1", compacted=True + ) + self.compact_calls = 0 + self.closed = False + + def compact_thread(self): + self.compact_calls += 1 + return self.result + + def close(self): + self.closed = True + + +class LiveCodexAgent: + """Minimal live cached agent whose _compress_context is the REAL routing. + + The forwarder mirrors run_agent.AIAgent._compress_context: it calls the + module-level compress_context(), so these tests exercise the genuine + codex route (mode gate, cooldown, compact_thread RPC) rather than a stub + of the code under test. + """ + + def __init__(self, mode="hermes", session=None): + self.api_mode = "codex_app_server" + self.codex_app_server_auto_compaction = mode + self.session_id = "sess-1" + self.platform = "telegram" + self._cached_system_prompt = "cached prompt" + self._codex_session = session if session is not None else FakeCodexSession() + self.context_compressor = SimpleNamespace( + compression_count=0, + last_compression_rough_tokens=0, + last_prompt_tokens=100, + last_completion_tokens=10, + awaiting_real_usage_after_compression=False, + ) + self.local_compress_calls = 0 + self.warnings = [] + + # -- surface used by compress_context -------------------------------- + def _touch_activity(self, *a, **k): + pass + + def _emit_status(self, m): + pass + + def _emit_warning(self, m): + self.warnings.append(m) + + def _build_system_prompt(self, s): + return "built prompt" + + def _compress_context(self, messages, system_message, **kwargs): + return compress_context(self, messages, system_message, **kwargs) + + +def _history(n=150): + return [ + {"role": "user" if i % 2 == 0 else "assistant", "content": f"m{i}" * 50} + for i in range(n) + ] + + +def _gateway(tmp_path, session_key="tg:123", agent=None): + from hermes_state import SessionDB + + db = SessionDB(db_path=tmp_path / "state.db") + gw = SimpleNamespace( + _agent_cache={} if agent is None else {session_key: (agent, 0.0)}, + _agent_cache_lock=None, + _session_db=db, + ) + return gw, db + + +# --------------------------------------------------------------------------- +# Core no-op regression: hermes mode + live thread => thread/compact runs +# --------------------------------------------------------------------------- + +def test_hermes_mode_compacts_live_thread_at_rpc_boundary(tmp_path): + agent = LiveCodexAgent(mode="hermes") + key = "tg:123" + gw, db = _gateway(tmp_path, key, agent) + # Pre-arm a persisted failure streak so success provably resets it + # through the REAL SessionDB (hygiene path involves the DB). + assert db.increment_hygiene_failure_streak(key) == 1 + + outcome = asyncio.run( + run_codex_hygiene_compaction( + gw, + key, + agent.session_id, + auto_mode="hermes", + history=_history(), + approx_tokens=345_000, + timeout_seconds=30.0, + ) + ) + + assert outcome == "compacted" + # The no-op is gone: the thread genuinely shrank via the app-server's own + # mechanism — asserted at the RPC-stub boundary. + assert agent._codex_session.compact_calls == 1 + # A compaction boundary was recorded on the live agent's compressor. + assert agent.context_compressor.compression_count == 1 + # The live agent is STILL cached — hygiene must not evict the thread owner. + assert gw._agent_cache[key][0] is agent + # Success reset the persisted hygiene failure streak. + assert db.increment_hygiene_failure_streak(key) == 1 # was cleared to 0 + + +def test_hermes_mode_without_cached_agent_skips_without_local_compression(tmp_path): + # Detached case: no live agent → no thread → nothing real to compact. + # The old behavior "compressed" the mirror (a no-op) and then evicted; + # the new behavior is an honest skip with the transcript untouched. + gw, _ = _gateway(tmp_path, agent=None) + history = _history() + before = [dict(m) for m in history] + + outcome = asyncio.run( + run_codex_hygiene_compaction( + gw, + "tg:123", + "sess-1", + auto_mode="hermes", + history=history, + approx_tokens=345_000, + timeout_seconds=5.0, + ) + ) + + assert outcome == "skipped:no-cached-agent" + assert history == before + + +# --------------------------------------------------------------------------- +# Mode semantics: native / off never touch the thread nor run local fallback +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("mode", ["native", "off"]) +def test_native_and_off_modes_skip_cleanly(tmp_path, mode): + agent = LiveCodexAgent(mode=mode) + key = "tg:123" + gw, _ = _gateway(tmp_path, key, agent) + + outcome = asyncio.run( + run_codex_hygiene_compaction( + gw, + key, + agent.session_id, + auto_mode=mode, + history=_history(), + approx_tokens=345_000, + timeout_seconds=5.0, + ) + ) + + assert outcome == f"skipped:mode={mode}" + # off must not silently compress; native leaves scheduling to codex. + assert agent._codex_session.compact_calls == 0 + assert agent.context_compressor.compression_count == 0 + # And the agent stays cached in every skip path. + assert gw._agent_cache[key][0] is agent + + +def test_unknown_mode_falls_back_to_native_semantics(tmp_path): + agent = LiveCodexAgent(mode="banana") + gw, _ = _gateway(tmp_path, "tg:123", agent) + outcome = asyncio.run( + run_codex_hygiene_compaction( + gw, + "tg:123", + agent.session_id, + auto_mode="banana", + history=_history(), + approx_tokens=345_000, + timeout_seconds=5.0, + ) + ) + assert outcome == "skipped:mode=native" + assert agent._codex_session.compact_calls == 0 + + +# --------------------------------------------------------------------------- +# force=True (manual /compress) must never violate the "no local fallback" +# contract: it compacts the THREAD in every mode, and never rewrites the +# transcript mirror. +# --------------------------------------------------------------------------- + +@pytest.mark.parametrize("mode", ["native", "hermes", "off"]) +def test_force_compacts_thread_never_local_fallback(mode): + agent = LiveCodexAgent(mode=mode) + messages = _history() + before = [dict(m) for m in messages] + + returned, prompt = compress_context( + agent, messages, "system", approx_tokens=345_000, force=True + ) + + # Thread compaction ran (manual force is an explicit user decision)... + assert agent._codex_session.compact_calls == 1 + # ...and the local transcript mirror was NOT rewritten in any mode. + assert returned is messages + assert returned == before + assert prompt == "cached prompt" + + +@pytest.mark.parametrize("mode", ["native", "off"]) +def test_force_without_live_thread_does_not_run_local_compressor(mode): + # The #73715 branch let force=True fall through to the local Hermes + # compressor in native/off when no thread existed. That is wrong on this + # runtime in every mode: rewriting the mirror cannot shrink the thread. + agent = LiveCodexAgent(mode=mode, session=None) + agent._codex_session = None + messages = _history() + before = [dict(m) for m in messages] + + returned, _ = compress_context( + agent, messages, "system", approx_tokens=345_000, force=True + ) + + assert returned is messages + assert returned == before + assert agent.context_compressor.compression_count == 0 + + +# --------------------------------------------------------------------------- +# Failure handling: a wedged compaction records a DB cooldown (real SessionDB) +# --------------------------------------------------------------------------- + +def test_timeout_records_persistent_cooldown(tmp_path): + class HangingSession(FakeCodexSession): + def compact_thread(self): + self.compact_calls += 1 + import time + + time.sleep(3.0) + return self.result + + agent = LiveCodexAgent(mode="hermes", session=HangingSession()) + key = "tg:123" + gw, db = _gateway(tmp_path, key, agent) + db.create_session(agent.session_id, "gateway") + + outcome = asyncio.run( + run_codex_hygiene_compaction( + gw, + key, + agent.session_id, + auto_mode="hermes", + history=_history(), + approx_tokens=345_000, + timeout_seconds=1.0, + failure_cooldown_seconds=300.0, + ) + ) + + assert outcome == "failed:timeout" + state = db.get_compression_failure_cooldown(agent.session_id) + assert state is not None and state.get("remaining_seconds", 0) > 0 + # Even on failure the live agent stays cached — its thread still holds + # the only real context. + assert gw._agent_cache[key][0] is agent + + +# --------------------------------------------------------------------------- +# Manual /compress gateway surface +# --------------------------------------------------------------------------- + +def _slash_host(agent, session_key="tg:123"): + from gateway.slash_commands import GatewaySlashCommandsMixin + + host = SimpleNamespace( + _agent_cache={session_key: (agent, 0.0)} if agent is not None else {}, + _agent_cache_lock=None, + ) + + async def _run_in_executor_with_context(fn): + return await asyncio.get_running_loop().run_in_executor(None, fn) + + host._run_in_executor_with_context = _run_in_executor_with_context + host._compress_codex_app_server_session = ( + GatewaySlashCommandsMixin._compress_codex_app_server_session.__get__(host) + ) + return host + + +def test_manual_compress_routes_to_live_thread(): + agent = LiveCodexAgent(mode="off") # even 'off': manual is a user decision + host = _slash_host(agent) + + reply = asyncio.run( + host._compress_codex_app_server_session("tg:123", agent.session_id) + ) + + assert agent._codex_session.compact_calls == 1 + assert "compacted" in reply + + +def test_manual_compress_without_live_thread_reports_honestly(): + host = _slash_host(None) + reply = asyncio.run( + host._compress_codex_app_server_session("tg:123", "sess-1") + ) + assert "Nothing to compact" in reply From e730deedd143b1fb63017b1e226ebd2bdf88e451 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 30 Aug 2026 20:10:25 -0700 Subject: [PATCH 034/634] =?UTF-8?q?feat(bot-mode):=20Group=20Chats=20survi?= =?UTF-8?q?ve=20the=20authority=20gateway=20dying=20=E2=80=94=20log=20repl?= =?UTF-8?q?ication=20and=20fenced=20takeover?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every participant gateway can now keep a durable copy of a hosted room's ordered log and continue the room when its authority host is gone: - gateway/hosted_room_replicas.py: replica store in root state.db. ingest_page() persists authority-stamped groups.log pages idempotently, refusing sequence gaps and authority-epoch regressions. promote_replica() continues the room locally at epoch+1 with a lineage-proving authority.claimed event; the stale owner is fenced everywhere the claim replicates. demote_room() lets a returning stale authority fence itself (authority.lost) upon observing a newer epoch, killing split-brain writes. - tui_gateway/methods_groups.py: groups.replicate / groups.replica_state / groups.promote / groups.demote RPC surface. Promotion requires confirm=true — storage decides HOW takeover is atomic and provable, the caller (user action now, lease/quorum driver later) decides WHEN it is safe, matching the boundary blessed on #97681. Validation: 20 new tests incl. a full failover round-trip (A hosts, B replicates incrementally, A dies, B promotes with complete history, A returns demoted and fenced); 69 total across the hosted-rooms area; E2E with two real gateway stores and real install identities. --- gateway/hosted_room_replicas.py | 570 ++++++++++++++++++ tests/gateway/test_hosted_room_replicas.py | 296 +++++++++ .../test_groups_replication_methods.py | 173 ++++++ tui_gateway/methods_groups.py | 102 ++++ 4 files changed, 1141 insertions(+) create mode 100644 gateway/hosted_room_replicas.py create mode 100644 tests/gateway/test_hosted_room_replicas.py create mode 100644 tests/tui_gateway/test_groups_replication_methods.py diff --git a/gateway/hosted_room_replicas.py b/gateway/hosted_room_replicas.py new file mode 100644 index 0000000000..26b87080ed --- /dev/null +++ b/gateway/hosted_room_replicas.py @@ -0,0 +1,570 @@ +"""Replica store and takeover primitives for hosted Group Chat rooms. + +The authority gateway owns a room's ordered log in ``gateway/hosted_rooms.py``. +This module gives every OTHER participant gateway a durable local copy of that +log, and the fenced primitives to continue the room when the authority host +dies: + +- ``ingest_page()`` persists replay pages (``groups.log`` output, which carries + the room's authority stamp) idempotently, refusing sequence gaps and + authority-epoch regressions. +- ``promote_replica()`` instantiates the replicated log as a locally-owned + hosted room at ``epoch + 1`` with a lineage-proving ``authority.claimed`` + event, so a surviving participant can resume the room. +- ``demote_room()`` fences a returning stale authority: presented with proof of + a newer epoch, the local room records ``authority.lost`` and stops being + authoritative. + +Storage primitives only: none of these decide *when* takeover is safe. The +caller (an explicit user action today; a lease/quorum driver later) must +establish that the previous owner can no longer commit before promoting. +""" + +from __future__ import annotations + +import json +import sqlite3 +import time +from pathlib import Path +from typing import Any + +from gateway.hosted_rooms import ( + MAX_ACTOR_ID_CHARS, + MAX_EVENT_JSON_BYTES, + MAX_ROOM_ID_CHARS, + HostedRoomError, + RoomConflictError, + _canonical_json, + _connect, + _transaction, + _validate_identifier, + _validate_members, + _validate_room_name, + local_authority_gateway_id, +) + +MAX_REPLICA_ROOMS = 256 +MAX_REPLICA_EVENT_BYTES = 256 * 1024 * 1024 + + +class ReplicaError(HostedRoomError): + """Base class for invalid or conflicting replica operations.""" + + +class ReplicaGapError(ReplicaError): + """A page does not start at the replica's next expected sequence.""" + + +class ReplicaEpochRegressionError(ReplicaError): + """A page or demotion carries an older authority epoch than stored.""" + + +def _initialize_replica_schema(conn: sqlite3.Connection) -> None: + conn.execute( + """CREATE TABLE IF NOT EXISTS hosted_room_replicas ( + room_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + members_json TEXT NOT NULL, + authority_gateway_id TEXT NOT NULL, + authority_epoch INTEGER NOT NULL CHECK (authority_epoch >= 1), + last_seq INTEGER NOT NULL DEFAULT 0 CHECK (last_seq >= 0), + latest_seq INTEGER NOT NULL DEFAULT 0, + event_bytes INTEGER NOT NULL DEFAULT 0, + created_at REAL NOT NULL, + updated_at REAL NOT NULL + )""" + ) + conn.execute( + """CREATE TABLE IF NOT EXISTS hosted_room_replica_events ( + room_id TEXT NOT NULL, + seq INTEGER NOT NULL CHECK (seq >= 1), + event_id TEXT NOT NULL, + kind TEXT NOT NULL, + actor_json TEXT NOT NULL, + authority_epoch INTEGER, + payload_json TEXT NOT NULL, + created_at REAL NOT NULL, + PRIMARY KEY (room_id, seq) + )""" + ) + + +def _replica_transaction(db_path: Path | str): + return _transaction(db_path, immediate=True) + + +def _ensure_schema(db_path: Path | str) -> None: + conn = _connect(db_path) + try: + with conn: + _initialize_replica_schema(conn) + finally: + conn.close() + + +def _event_bytes(event: dict[str, Any]) -> int: + return ( + len(str(event["event_id"]).encode("utf-8")) + + len(str(event["kind"]).encode("utf-8")) + + len( + json.dumps( + event["actor"], ensure_ascii=False, separators=(",", ":") + ).encode("utf-8") + ) + + len( + json.dumps( + event["payload"], ensure_ascii=False, separators=(",", ":") + ).encode("utf-8") + ) + ) + + +def _validate_page(page: Any) -> tuple[list[dict[str, Any]], dict[str, Any]]: + if not isinstance(page, dict): + raise ReplicaError("page must be an object") + events = page.get("events") + authority = page.get("authority") + if not isinstance(events, list): + raise ReplicaError("page.events must be a list") + if not isinstance(authority, dict): + raise ReplicaError("page.authority is required for replication") + gateway_id = _validate_identifier( + authority.get("gateway_id"), + label="page.authority.gateway_id", + max_chars=MAX_ACTOR_ID_CHARS, + ) + epoch = authority.get("epoch") + if isinstance(epoch, bool) or not isinstance(epoch, int) or epoch < 1: + raise ReplicaError("page.authority.epoch must be a positive integer") + previous_seq: int | None = None + for event in events: + if not isinstance(event, dict): + raise ReplicaError("page events must be objects") + seq = event.get("seq") + if isinstance(seq, bool) or not isinstance(seq, int) or seq < 1: + raise ReplicaError("event.seq must be a positive integer") + if previous_seq is not None and seq != previous_seq + 1: + raise ReplicaGapError("page events must be contiguous") + previous_seq = seq + for field in ("event_id", "kind"): + if not isinstance(event.get(field), str) or not event[field]: + raise ReplicaError(f"event.{field} must be a non-empty string") + if not isinstance(event.get("actor"), dict): + raise ReplicaError("event.actor must be an object") + if "payload" not in event: + raise ReplicaError("event.payload is required") + return events, {"gateway_id": gateway_id, "epoch": epoch} + + +def ingest_page( + db_path: Path | str, + *, + room_id: Any, + room_name: Any, + members: Any, + page: Any, + now: float | None = None, +) -> dict[str, Any]: + """Persist one replay page for ``room_id``; idempotent, gap- and + epoch-regression-safe. + + ``page`` is the verbatim result of the authority's ``groups.log`` call + (``read_events()``), whose ``authority`` stamp proves lineage. + """ + room_id = _validate_identifier( + room_id, label="room_id", max_chars=MAX_ROOM_ID_CHARS + ) + room_name = _validate_room_name(room_name) + _, members_json = _validate_members(members) + events, authority = _validate_page(page) + now = time.time() if now is None else float(now) + _ensure_schema(db_path) + + with _replica_transaction(db_path) as conn: + _initialize_replica_schema(conn) + row = conn.execute( + """SELECT authority_gateway_id, authority_epoch, last_seq, + latest_seq, event_bytes + FROM hosted_room_replicas WHERE room_id=?""", + (room_id,), + ).fetchone() + if row is None: + count = conn.execute( + "SELECT COUNT(*) FROM hosted_room_replicas" + ).fetchone()[0] + if int(count) >= MAX_REPLICA_ROOMS: + raise ReplicaError("replica room capacity exhausted") + stored_epoch = 0 + last_seq = 0 + stored_bytes = 0 + else: + stored_epoch = int(row["authority_epoch"]) + last_seq = int(row["last_seq"]) + stored_bytes = int(row["event_bytes"]) + + if authority["epoch"] < stored_epoch: + raise ReplicaEpochRegressionError( + "page authority epoch is older than the stored replica epoch" + ) + + new_events = [e for e in events if int(e["seq"]) > last_seq] + if new_events and int(new_events[0]["seq"]) != last_seq + 1: + raise ReplicaGapError( + "page skips sequences the replica has not stored" + ) + added_bytes = 0 + for event in new_events: + size = _event_bytes(event) + if stored_bytes + added_bytes + size > MAX_REPLICA_EVENT_BYTES: + raise ReplicaError("replica event storage exhausted") + actor_json = _canonical_json( + event["actor"], label="actor", max_bytes=4 * 1024 + ) + payload_json = _canonical_json( + event["payload"], label="payload", max_bytes=MAX_EVENT_JSON_BYTES + ) + epoch_value = event.get("authority_epoch") + conn.execute( + """INSERT INTO hosted_room_replica_events + (room_id, seq, event_id, kind, actor_json, authority_epoch, + payload_json, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", + ( + room_id, + int(event["seq"]), + event["event_id"], + event["kind"], + actor_json, + epoch_value, + payload_json, + float(event.get("created_at") or now), + ), + ) + added_bytes += size + new_last = int(new_events[-1]["seq"]) if new_events else last_seq + latest_seq = page.get("latest_seq") + if isinstance(latest_seq, bool) or not isinstance(latest_seq, int): + latest_seq = new_last + if row is None: + conn.execute( + """INSERT INTO hosted_room_replicas + (room_id, name, members_json, authority_gateway_id, + authority_epoch, last_seq, latest_seq, event_bytes, + created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + ( + room_id, + room_name, + members_json, + authority["gateway_id"], + authority["epoch"], + new_last, + max(latest_seq, new_last), + added_bytes, + now, + now, + ), + ) + else: + conn.execute( + """UPDATE hosted_room_replicas + SET name=?, members_json=?, authority_gateway_id=?, + authority_epoch=?, last_seq=?, latest_seq=?, + event_bytes=event_bytes+?, updated_at=? + WHERE room_id=?""", + ( + room_name, + members_json, + authority["gateway_id"], + authority["epoch"], + new_last, + max(latest_seq, new_last), + added_bytes, + now, + room_id, + ), + ) + return { + "room_id": room_id, + "stored_seq": new_last, + "ingested": len(new_events), + "authority": authority, + "caught_up": new_last >= max(latest_seq, new_last), + } + + +def replica_state(db_path: Path | str, *, room_id: Any) -> dict[str, Any]: + """Return the stored replica's coverage and authority lineage.""" + room_id = _validate_identifier( + room_id, label="room_id", max_chars=MAX_ROOM_ID_CHARS + ) + _ensure_schema(db_path) + with _replica_transaction(db_path) as conn: + _initialize_replica_schema(conn) + row = conn.execute( + """SELECT room_id, name, members_json, authority_gateway_id, + authority_epoch, last_seq, latest_seq, event_bytes, + created_at, updated_at + FROM hosted_room_replicas WHERE room_id=?""", + (room_id,), + ).fetchone() + if row is None: + raise ReplicaError("replica not found") + return { + "room_id": row["room_id"], + "name": row["name"], + "members": json.loads(row["members_json"]), + "authority": { + "gateway_id": row["authority_gateway_id"], + "epoch": int(row["authority_epoch"]), + }, + "last_seq": int(row["last_seq"]), + "latest_seq": int(row["latest_seq"]), + "event_bytes": int(row["event_bytes"]), + "created_at": float(row["created_at"]), + "updated_at": float(row["updated_at"]), + } + + +def promote_replica( + db_path: Path | str, + *, + room_id: Any, + reason: Any = "authority-unreachable", + now: float | None = None, +) -> dict[str, Any]: + """Continue a replicated room on THIS gateway at ``epoch + 1``. + + Copies the replica's log into the authoritative store, appends a lineage- + proving ``authority.claimed`` event, and returns the new room state. The + old authority is fenced everywhere the claim replicates: its epoch is now + stale and every fenced primitive rejects it. + + The caller decides that takeover is safe (the previous owner can no longer + commit). This primitive only makes the takeover atomic and provable. + """ + room_id = _validate_identifier( + room_id, label="room_id", max_chars=MAX_ROOM_ID_CHARS + ) + if not isinstance(reason, str) or not reason or len(reason) > 200: + raise ReplicaError("reason must be a non-empty string of at most 200 chars") + now = time.time() if now is None else float(now) + local_gateway = local_authority_gateway_id() + _ensure_schema(db_path) + + with _replica_transaction(db_path) as conn: + _initialize_replica_schema(conn) + replica = conn.execute( + """SELECT room_id, name, members_json, authority_gateway_id, + authority_epoch, last_seq, event_bytes + FROM hosted_room_replicas WHERE room_id=?""", + (room_id,), + ).fetchone() + if replica is None: + raise ReplicaError("replica not found") + if replica["authority_gateway_id"] == local_gateway: + raise ReplicaError("this gateway already holds the room authority") + if conn.execute( + "SELECT 1 FROM hosted_rooms WHERE room_id=?", (room_id,) + ).fetchone(): + raise RoomConflictError( + "room_id already exists in the local authoritative store" + ) + if conn.execute( + "SELECT 1 FROM hosted_room_retired_ids WHERE room_id=?", + (room_id,), + ).fetchone(): + raise RoomConflictError("room_id belongs to a disbanded room") + + previous_gateway = str(replica["authority_gateway_id"]) + previous_epoch = int(replica["authority_epoch"]) + target_epoch = previous_epoch + 1 + last_seq = int(replica["last_seq"]) + claim_seq = last_seq + 1 + claim_event_id = f"system:authority-claimed:{target_epoch}" + claim_actor_json = _canonical_json( + {"kind": "system", "id": "authority-control"}, + label="actor", + max_bytes=4 * 1024, + ) + claim_payload_json = _canonical_json( + { + "previous_gateway_id": previous_gateway, + "authority_gateway_id": local_gateway, + "authority_epoch": target_epoch, + "promoted_from_replica": True, + "reason": reason, + }, + label="payload", + max_bytes=MAX_EVENT_JSON_BYTES, + ) + claim_bytes = ( + len(claim_event_id.encode("utf-8")) + + len(b"authority.claimed") + + len(claim_actor_json.encode("utf-8")) + + len(claim_payload_json.encode("utf-8")) + ) + + conn.execute( + """INSERT INTO hosted_rooms + (room_id, name, members_json, authority_gateway_id, + authority_epoch, next_seq, event_bytes, revision, + created_at, updated_at, disbanded_at) + VALUES (?, ?, ?, ?, ?, ?, ?, 1, ?, ?, NULL)""", + ( + room_id, + replica["name"], + replica["members_json"], + local_gateway, + target_epoch, + claim_seq + 1, + int(replica["event_bytes"]) + claim_bytes, + now, + now, + ), + ) + conn.execute( + """INSERT INTO hosted_room_events + (room_id, seq, event_id, kind, actor_json, authority_epoch, + payload_json, created_at) + SELECT room_id, seq, event_id, kind, actor_json, + authority_epoch, payload_json, created_at + FROM hosted_room_replica_events WHERE room_id=?""", + (room_id,), + ) + conn.execute( + """INSERT INTO hosted_room_events + (room_id, seq, event_id, kind, actor_json, authority_epoch, + payload_json, created_at) + VALUES (?, ?, ?, 'authority.claimed', ?, ?, ?, ?)""", + ( + room_id, + claim_seq, + claim_event_id, + claim_actor_json, + target_epoch, + claim_payload_json, + now, + ), + ) + conn.execute( + "DELETE FROM hosted_room_replica_events WHERE room_id=?", (room_id,) + ) + conn.execute( + "DELETE FROM hosted_room_replicas WHERE room_id=?", (room_id,) + ) + return { + "room_id": room_id, + "authority_gateway_id": local_gateway, + "authority_epoch": target_epoch, + "previous_gateway_id": previous_gateway, + "previous_epoch": previous_epoch, + "claim_seq": claim_seq, + "latest_seq": claim_seq, + } + + +def demote_room( + db_path: Path | str, + *, + room_id: Any, + observed_gateway_id: Any, + observed_epoch: Any, + now: float | None = None, +) -> dict[str, Any]: + """Fence THIS gateway's stale room authority against a proven newer epoch. + + Called when a returning gateway observes (via a replicated + ``authority.claimed`` event or a transport rejection) that another gateway + now owns the room at a higher epoch. Appends ``authority.lost`` and adopts + the observed lineage so no further local sends can be committed at the + stale epoch. Idempotent for repeated observations of the same lineage. + """ + room_id = _validate_identifier( + room_id, label="room_id", max_chars=MAX_ROOM_ID_CHARS + ) + observed_gateway_id = _validate_identifier( + observed_gateway_id, + label="observed_gateway_id", + max_chars=MAX_ACTOR_ID_CHARS, + ) + if ( + isinstance(observed_epoch, bool) + or not isinstance(observed_epoch, int) + or observed_epoch < 1 + ): + raise ReplicaError("observed_epoch must be a positive integer") + now = time.time() if now is None else float(now) + local_gateway = local_authority_gateway_id() + + with _replica_transaction(db_path) as conn: + row = conn.execute( + """SELECT authority_gateway_id, authority_epoch, next_seq + FROM hosted_rooms WHERE room_id=? AND disbanded_at IS NULL""", + (room_id,), + ).fetchone() + if row is None: + raise ReplicaError("room not found in the local authoritative store") + current_gateway = str(row["authority_gateway_id"]) + current_epoch = int(row["authority_epoch"]) + if ( + current_gateway == observed_gateway_id + and current_epoch == observed_epoch + ): + return { + "room_id": room_id, + "authority_gateway_id": current_gateway, + "authority_epoch": current_epoch, + "idempotent": True, + } + if observed_epoch <= current_epoch: + raise ReplicaEpochRegressionError( + "observed epoch does not supersede the stored authority" + ) + if current_gateway != local_gateway: + raise ReplicaError( + "room is not locally authoritative; nothing to demote" + ) + seq = int(row["next_seq"]) + lost_actor_json = _canonical_json( + {"kind": "system", "id": "authority-control"}, + label="actor", + max_bytes=4 * 1024, + ) + lost_payload_json = _canonical_json( + { + "previous_gateway_id": current_gateway, + "authority_gateway_id": observed_gateway_id, + "authority_epoch": observed_epoch, + }, + label="payload", + max_bytes=MAX_EVENT_JSON_BYTES, + ) + conn.execute( + """INSERT INTO hosted_room_events + (room_id, seq, event_id, kind, actor_json, authority_epoch, + payload_json, created_at) + VALUES (?, ?, ?, 'authority.lost', ?, ?, ?, ?)""", + ( + room_id, + seq, + f"system:authority-lost:{observed_epoch}", + lost_actor_json, + observed_epoch, + lost_payload_json, + now, + ), + ) + conn.execute( + """UPDATE hosted_rooms + SET authority_gateway_id=?, authority_epoch=?, + next_seq=next_seq+1, revision=revision+1, updated_at=? + WHERE room_id=?""", + (observed_gateway_id, observed_epoch, now, room_id), + ) + return { + "room_id": room_id, + "authority_gateway_id": observed_gateway_id, + "authority_epoch": observed_epoch, + "idempotent": False, + } diff --git a/tests/gateway/test_hosted_room_replicas.py b/tests/gateway/test_hosted_room_replicas.py new file mode 100644 index 0000000000..cfe5501683 --- /dev/null +++ b/tests/gateway/test_hosted_room_replicas.py @@ -0,0 +1,296 @@ +"""Tests for gateway/hosted_room_replicas.py — replica ingest, promotion, and +stale-authority demotion for hosted Group Chat rooms.""" + +import json + +import pytest + +import gateway.hosted_room_replicas as replicas +import gateway.hosted_rooms as rooms + +USER = {"kind": "user", "id": "tek"} +MEMBERS = [{"kind": "bot", "id": "planner"}, {"kind": "bot", "id": "coder"}] + +AUTH_A = "install:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +AUTH_B = "install:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + + +def _authority_db(tmp_path, name="authority.db"): + return tmp_path / name + + +def _replica_db(tmp_path, name="replica.db"): + return tmp_path / name + + +def _seed_room(db, *, gateway_id=AUTH_A, n_events=3, room_id="room-1"): + rooms.create_room( + db, + room_id=room_id, + name="Field Room", + members=MEMBERS, + authority_gateway_id=gateway_id, + ) + for index in range(n_events): + rooms.append_event( + db, + room_id=room_id, + event_id=f"e{index}", + kind="message.user", + actor=USER, + payload={"text": f"msg {index} 😀"}, + authority_gateway_id=gateway_id, + authority_epoch=1, + ) + return rooms.read_events(db, room_id=room_id, since_seq=0, limit=100) + + +def test_ingest_page_persists_events_and_lineage(tmp_path): + page = _seed_room(_authority_db(tmp_path)) + rdb = _replica_db(tmp_path) + result = replicas.ingest_page( + rdb, room_id="room-1", room_name="Field Room", members=MEMBERS, page=page + ) + assert result["ingested"] == 3 + assert result["stored_seq"] == 3 + assert result["caught_up"] is True + state = replicas.replica_state(rdb, room_id="room-1") + assert state["last_seq"] == 3 + assert state["authority"] == page["authority"] + assert state["members"] == MEMBERS + + +def test_ingest_page_is_idempotent(tmp_path): + page = _seed_room(_authority_db(tmp_path)) + rdb = _replica_db(tmp_path) + replicas.ingest_page( + rdb, room_id="room-1", room_name="Field Room", members=MEMBERS, page=page + ) + again = replicas.ingest_page( + rdb, room_id="room-1", room_name="Field Room", members=MEMBERS, page=page + ) + assert again["ingested"] == 0 + assert again["stored_seq"] == 3 + + +def test_ingest_rejects_sequence_gap(tmp_path): + adb = _authority_db(tmp_path) + _seed_room(adb, n_events=5) + later = rooms.read_events(adb, room_id="room-1", since_seq=2, limit=100) + rdb = _replica_db(tmp_path) + with pytest.raises(replicas.ReplicaGapError): + replicas.ingest_page( + rdb, + room_id="room-1", + room_name="Field Room", + members=MEMBERS, + page=later, + ) + + +def test_ingest_rejects_epoch_regression(tmp_path): + page = _seed_room(_authority_db(tmp_path)) + rdb = _replica_db(tmp_path) + newer = json.loads(json.dumps(page)) + newer["authority"]["epoch"] = 3 + replicas.ingest_page( + rdb, room_id="room-1", room_name="Field Room", members=MEMBERS, page=newer + ) + stale = json.loads(json.dumps(page)) + stale["authority"]["epoch"] = 2 + with pytest.raises(replicas.ReplicaEpochRegressionError): + replicas.ingest_page( + rdb, + room_id="room-1", + room_name="Field Room", + members=MEMBERS, + page=stale, + ) + + +def test_ingest_requires_authority_stamp(tmp_path): + page = _seed_room(_authority_db(tmp_path)) + page.pop("authority") + with pytest.raises(replicas.ReplicaError): + replicas.ingest_page( + _replica_db(tmp_path), + room_id="room-1", + room_name="Field Room", + members=MEMBERS, + page=page, + ) + + +def test_promote_replica_continues_room_at_next_epoch(tmp_path, monkeypatch): + page = _seed_room(_authority_db(tmp_path)) + rdb = _replica_db(tmp_path) + replicas.ingest_page( + rdb, room_id="room-1", room_name="Field Room", members=MEMBERS, page=page + ) + monkeypatch.setattr(replicas, "local_authority_gateway_id", lambda: AUTH_B) + + promoted = replicas.promote_replica(rdb, room_id="room-1") + assert promoted["authority_gateway_id"] == AUTH_B + assert promoted["authority_epoch"] == 2 + assert promoted["previous_gateway_id"] == AUTH_A + assert promoted["claim_seq"] == 4 + + # The room is now locally authoritative with the full history + claim. + replay = rooms.read_events(rdb, room_id="room-1", since_seq=0, limit=100) + assert [e["seq"] for e in replay["events"]] == [1, 2, 3, 4] + claim = replay["events"][-1] + assert claim["kind"] == "authority.claimed" + assert claim["payload"]["previous_gateway_id"] == AUTH_A + assert claim["payload"]["authority_epoch"] == 2 + assert replay["authority"] == {"gateway_id": AUTH_B, "epoch": 2} + + # New work continues under the new epoch. + rooms.append_event( + rdb, + room_id="room-1", + event_id="post-takeover", + kind="message.user", + actor=USER, + payload={"text": "continuing"}, + authority_gateway_id=AUTH_B, + authority_epoch=2, + ) + + # The old authority's identity/epoch is fenced out. + with pytest.raises(rooms.HostedRoomError): + rooms.append_event( + rdb, + room_id="room-1", + event_id="stale-write", + kind="message.user", + actor=USER, + payload={"text": "stale"}, + authority_gateway_id=AUTH_A, + authority_epoch=1, + ) + + # Replica bookkeeping is consumed by promotion. + with pytest.raises(replicas.ReplicaError): + replicas.replica_state(rdb, room_id="room-1") + + +def test_promote_refuses_when_room_exists_locally(tmp_path, monkeypatch): + db = _authority_db(tmp_path) + page = _seed_room(db) + # Same DB also holds a replica row for the same id — conflict must win. + replicas.ingest_page( + db, room_id="room-1", room_name="Field Room", members=MEMBERS, page=page + ) + monkeypatch.setattr(replicas, "local_authority_gateway_id", lambda: AUTH_B) + with pytest.raises(rooms.RoomConflictError): + replicas.promote_replica(db, room_id="room-1") + + +def test_promote_refuses_when_already_authority(tmp_path, monkeypatch): + page = _seed_room(_authority_db(tmp_path)) + rdb = _replica_db(tmp_path) + replicas.ingest_page( + rdb, room_id="room-1", room_name="Field Room", members=MEMBERS, page=page + ) + monkeypatch.setattr(replicas, "local_authority_gateway_id", lambda: AUTH_A) + with pytest.raises(replicas.ReplicaError): + replicas.promote_replica(rdb, room_id="room-1") + + +def test_demote_fences_stale_local_authority(tmp_path, monkeypatch): + adb = _authority_db(tmp_path) + _seed_room(adb) + monkeypatch.setattr(replicas, "local_authority_gateway_id", lambda: AUTH_A) + + result = replicas.demote_room( + adb, room_id="room-1", observed_gateway_id=AUTH_B, observed_epoch=2 + ) + assert result["idempotent"] is False + assert result["authority_gateway_id"] == AUTH_B + assert result["authority_epoch"] == 2 + + replay = rooms.read_events(adb, room_id="room-1", since_seq=0, limit=100) + lost = replay["events"][-1] + assert lost["kind"] == "authority.lost" + assert lost["payload"]["authority_gateway_id"] == AUTH_B + assert replay["authority"] == {"gateway_id": AUTH_B, "epoch": 2} + + # Local sends at the stale identity/epoch are now rejected. + with pytest.raises(rooms.HostedRoomError): + rooms.append_event( + adb, + room_id="room-1", + event_id="after-demote", + kind="message.user", + actor=USER, + payload={"text": "stale"}, + authority_gateway_id=AUTH_A, + authority_epoch=1, + ) + + # Repeating the same observation is idempotent. + again = replicas.demote_room( + adb, room_id="room-1", observed_gateway_id=AUTH_B, observed_epoch=2 + ) + assert again["idempotent"] is True + + +def test_demote_rejects_non_superseding_epoch(tmp_path, monkeypatch): + adb = _authority_db(tmp_path) + _seed_room(adb) + monkeypatch.setattr(replicas, "local_authority_gateway_id", lambda: AUTH_A) + with pytest.raises(replicas.ReplicaEpochRegressionError): + replicas.demote_room( + adb, room_id="room-1", observed_gateway_id=AUTH_B, observed_epoch=1 + ) + + +def test_full_failover_round_trip(tmp_path, monkeypatch): + """Authority A hosts, replica B follows, A dies, B promotes, A returns + and is fenced + demoted; the room's history survives intact throughout.""" + adb = _authority_db(tmp_path) + rdb = _replica_db(tmp_path) + page = _seed_room(adb, n_events=4) + replicas.ingest_page( + rdb, room_id="room-1", room_name="Field Room", members=MEMBERS, page=page + ) + + # A "dies"; B takes over. + monkeypatch.setattr(replicas, "local_authority_gateway_id", lambda: AUTH_B) + promoted = replicas.promote_replica(rdb, room_id="room-1") + rooms.append_event( + rdb, + room_id="room-1", + event_id="b-work", + kind="message.user", + actor=USER, + payload={"text": "work continues on B"}, + authority_gateway_id=AUTH_B, + authority_epoch=promoted["authority_epoch"], + ) + + # A comes back, observes B's claim, and fences itself. + monkeypatch.setattr(replicas, "local_authority_gateway_id", lambda: AUTH_A) + replicas.demote_room( + adb, + room_id="room-1", + observed_gateway_id=AUTH_B, + observed_epoch=promoted["authority_epoch"], + ) + with pytest.raises(rooms.HostedRoomError): + rooms.append_event( + adb, + room_id="room-1", + event_id="a-stale", + kind="message.user", + actor=USER, + payload={"text": "split brain attempt"}, + authority_gateway_id=AUTH_A, + authority_epoch=1, + ) + + # B's room holds the complete history: 4 original + claim + new work. + replay = rooms.read_events(rdb, room_id="room-1", since_seq=0, limit=100) + kinds = [e["kind"] for e in replay["events"]] + assert kinds == ["message.user"] * 4 + ["authority.claimed", "message.user"] + assert replay["authority"]["gateway_id"] == AUTH_B diff --git a/tests/tui_gateway/test_groups_replication_methods.py b/tests/tui_gateway/test_groups_replication_methods.py new file mode 100644 index 0000000000..9101a8cbbe --- /dev/null +++ b/tests/tui_gateway/test_groups_replication_methods.py @@ -0,0 +1,173 @@ +"""Tests for the ``groups.replicate`` / ``groups.promote`` / ``groups.demote`` +JSON-RPC surface — cross-gateway room durability.""" + +from __future__ import annotations + +import pytest + +import tui_gateway.server as srv + +MEMBERS = [{"kind": "bot", "id": "planner"}] + + +@pytest.fixture +def home(tmp_path, monkeypatch): + path = tmp_path / ".hermes" + path.mkdir() + monkeypatch.setenv("HERMES_HOME", str(path)) + return path + + +def _result(envelope): + assert "error" not in envelope, envelope + return envelope["result"] + + +def _error(envelope): + assert "error" in envelope, envelope + return envelope["error"] + + +def _authority_page(tmp_path, gateway_id="install:" + "a" * 32, n=3): + """Build a real room + log on a SEPARATE 'remote authority' DB and return + its replay page, as a replicating client would fetch via groups.log.""" + from gateway import hosted_rooms as rooms + + db = tmp_path / "remote-authority.db" + rooms.create_room( + db, + room_id="room-1", + name="Field Room", + members=MEMBERS, + authority_gateway_id=gateway_id, + ) + for index in range(n): + rooms.append_event( + db, + room_id="room-1", + event_id=f"e{index}", + kind="message.user", + actor={"kind": "user", "id": "tek"}, + payload={"text": f"msg {index}"}, + authority_gateway_id=gateway_id, + authority_epoch=1, + ) + return rooms.read_events(db, room_id="room-1", since_seq=0, limit=100) + + +def test_capabilities_advertise_replication(home): + result = _result(srv._methods["groups.capabilities"](1, {})) + assert "log_replication" in result["features"] + assert "authority_takeover" in result["features"] + for name in ( + "groups.replicate", + "groups.replica_state", + "groups.promote", + "groups.demote", + ): + assert name in result["methods"] + assert name in srv._LONG_HANDLERS + + +def test_replicate_then_state_roundtrip(home, tmp_path): + page = _authority_page(tmp_path) + result = _result( + srv._methods["groups.replicate"]( + 1, + { + "room_id": "room-1", + "room_name": "Field Room", + "members": MEMBERS, + "page": page, + }, + ) + ) + assert result["ingested"] == 3 + state = _result(srv._methods["groups.replica_state"](2, {"room_id": "room-1"})) + assert state["last_seq"] == 3 + assert state["authority"] == page["authority"] + + +def test_promote_requires_confirm_and_takes_over(home, tmp_path): + page = _authority_page(tmp_path) + _result( + srv._methods["groups.replicate"]( + 1, + { + "room_id": "room-1", + "room_name": "Field Room", + "members": MEMBERS, + "page": page, + }, + ) + ) + + refused = _error(srv._methods["groups.promote"](2, {"room_id": "room-1"})) + assert refused["code"] == 4118 + + promoted = _result( + srv._methods["groups.promote"](3, {"room_id": "room-1", "confirm": True}) + ) + assert promoted["authority_epoch"] == 2 + assert promoted["previous_gateway_id"] == page["authority"]["gateway_id"] + + # The room is now hosted locally with full history + claim event. + log = _result(srv._methods["groups.log"](4, {"room_id": "room-1"})) + kinds = [event["kind"] for event in log["events"]] + assert kinds == ["message.user"] * 3 + ["authority.claimed"] + assert log["authority"]["epoch"] == 2 + + +def test_demote_fences_local_room_against_newer_epoch(home): + from gateway.hosted_rooms import local_authority_gateway_id + + _result( + srv._methods["groups.create"]( + 1, + {"room_id": "room-1", "name": "Local room", "members": MEMBERS}, + ) + ) + observed_gateway = "install:" + "b" * 32 + result = _result( + srv._methods["groups.demote"]( + 2, + { + "room_id": "room-1", + "observed_gateway_id": observed_gateway, + "observed_epoch": 2, + }, + ) + ) + assert result["idempotent"] is False + assert result["authority_gateway_id"] == observed_gateway + + # Local sends at the stale authority now fail. + envelope = srv._methods["groups.send"]( + 3, + { + "room_id": "room-1", + "event_id": "stale-send", + "actor": {"kind": "user", "id": "tek"}, + "payload": {"text": "should fence"}, + }, + ) + assert "error" in envelope + assert local_authority_gateway_id() != observed_gateway + + +def test_replicate_rejects_gapped_page(home, tmp_path): + from gateway import hosted_rooms as rooms + + _authority_page(tmp_path, n=5) + db = tmp_path / "remote-authority.db" + gapped = rooms.read_events(db, room_id="room-1", since_seq=2, limit=100) + envelope = srv._methods["groups.replicate"]( + 1, + { + "room_id": "room-1", + "room_name": "Field Room", + "members": MEMBERS, + "page": gapped, + }, + ) + assert _error(envelope)["code"] == 4116 diff --git a/tui_gateway/methods_groups.py b/tui_gateway/methods_groups.py index c2b6ecb4e9..9c014d797c 100644 --- a/tui_gateway/methods_groups.py +++ b/tui_gateway/methods_groups.py @@ -19,6 +19,10 @@ LONG_HANDLERS = frozenset({ "groups.send", "groups.log", "groups.disband", + "groups.replicate", + "groups.replica_state", + "groups.promote", + "groups.demote", }) @@ -46,6 +50,8 @@ def _(rid, params: dict) -> dict: "replayable_disband", "typed_events", "actor_identity", + "log_replication", + "authority_takeover", ], "methods": [ "groups.capabilities", @@ -55,6 +61,10 @@ def _(rid, params: dict) -> dict: "groups.send", "groups.log", "groups.disband", + "groups.replicate", + "groups.replica_state", + "groups.promote", + "groups.demote", ], "max_log_limit": MAX_LOG_LIMIT, }, @@ -262,5 +272,97 @@ def _(rid, params: dict) -> dict: return _err(rid, 5113, str(exc)) +@method("groups.replicate") +def _(rid, params: dict) -> dict: + """Persist one authority-stamped replay page into the local replica store. + + ``page`` is the verbatim ``groups.log`` result read from the room's + authority gateway; ingest is idempotent and refuses sequence gaps and + authority-epoch regressions. + """ + from gateway.hosted_room_replicas import ReplicaError, ingest_page + from gateway.hosted_rooms import default_db_path + + try: + result = ingest_page( + default_db_path(), + room_id=params.get("room_id"), + room_name=params.get("room_name"), + members=params.get("members"), + page=params.get("page"), + ) + return _ok(rid, result) + except ReplicaError as exc: + return _err(rid, 4116, str(exc)) + except Exception as exc: + return _err(rid, 5116, str(exc)) + + +@method("groups.replica_state") +def _(rid, params: dict) -> dict: + """Report the local replica's coverage and authority lineage.""" + from gateway.hosted_room_replicas import ReplicaError, replica_state + from gateway.hosted_rooms import default_db_path + + try: + return _ok(rid, replica_state(default_db_path(), room_id=params.get("room_id"))) + except ReplicaError as exc: + return _err(rid, 4117, str(exc)) + except Exception as exc: + return _err(rid, 5117, str(exc)) + + +@method("groups.promote") +def _(rid, params: dict) -> dict: + """Continue a replicated room on THIS gateway at ``epoch + 1``. + + Requires ``confirm: true`` — the caller asserts the previous authority can + no longer commit (explicit user action; a lease/quorum driver later). + """ + from gateway.hosted_room_replicas import ReplicaError, promote_replica + from gateway.hosted_rooms import HostedRoomError, default_db_path + + if params.get("confirm") is not True: + return _err( + rid, + 4118, + "promotion requires confirm=true acknowledging the previous " + "authority can no longer commit", + ) + try: + result = promote_replica( + default_db_path(), + room_id=params.get("room_id"), + reason=params.get("reason", "authority-unreachable"), + ) + return _ok(rid, result) + except ReplicaError as exc: + return _err(rid, 4118, str(exc)) + except HostedRoomError as exc: + return _err(rid, 4118, str(exc)) + except Exception as exc: + return _err(rid, 5118, str(exc)) + + +@method("groups.demote") +def _(rid, params: dict) -> dict: + """Fence this gateway's stale room authority against a proven newer epoch.""" + from gateway.hosted_room_replicas import ReplicaError, demote_room + from gateway.hosted_rooms import default_db_path + + try: + result = demote_room( + default_db_path(), + room_id=params.get("room_id"), + observed_gateway_id=params.get("observed_gateway_id"), + observed_epoch=params.get("observed_epoch"), + ) + return _ok(rid, result) + except ReplicaError as exc: + return _err(rid, 4119, str(exc)) + except Exception as exc: + return _err(rid, 5119, str(exc)) + + def register(server) -> None: _registry.install(server) From 5a134383fe4e8b03c191ac688c14b32cfe038699 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 30 Aug 2026 20:16:28 -0700 Subject: [PATCH 035/634] fix: failed subagents now surface a clean error to the user (CLI + gateway) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A delegate_task child that died (provider 404/400, timeout, crash) previously vanished silently: the child's conversation loop returns failed=True with the error summary in final_response, which the classifier treated as usable output -> status 'completed'. And even correctly-failed children only reached the parent MODEL — platforms with tool_progress off (Telegram/Slack defaults) never showed the human anything. - delegate_tool: result.failed now forces status 'failed' (with the error carried on the entry); new shared format_subagent_failure_line() renders one clean human-readable line (traceback -> exception message, length-capped); CLI tree + batch ✗ lines now include the reason. - gateway TurnRunner.progress_callback: subagent.complete events with a terminal failure status deliver that line via _deliver_platform_notice BEFORE all progress-queue gates; tool_progress_callback is now always attached (body gates each event class itself). - tests: failed-flag classification regression + notice rendering suite. - docs: Failure Visibility section in delegation docs. --- gateway/run.py | 45 ++++- tests/gateway/test_subagent_failure_notice.py | 171 ++++++++++++++++++ tests/tools/test_delegate.py | 29 +++ tools/delegate_tool.py | 89 +++++++++ website/docs/guides/delegation-patterns.md | 2 + .../docs/user-guide/features/delegation.md | 10 + 6 files changed, 337 insertions(+), 9 deletions(-) create mode 100644 tests/gateway/test_subagent_failure_notice.py diff --git a/gateway/run.py b/gateway/run.py index a7a6191bf7..7aa6cf7930 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -4590,6 +4590,36 @@ class TurnRunner: def progress_callback(self, event_type: str, tool_name: str = None, preview: str = None, args: dict = None, **kwargs): """Callback invoked by agent on tool lifecycle events.""" ctx = self._ctx + # Failed subagent → one clean user-facing notice. Handled FIRST, + # before every progress-queue gate: platforms that keep + # tool_progress off (Telegram, Slack, ...) must still hear about a + # delegation that died — a silently-vanishing subagent looks like + # the agent just dropped the task (community report, Aug 2026). + # Success/interrupt completions stay quiet; only terminal failure + # statuses render, via the same notice rail as credit warnings. + if event_type == "subagent.complete": + _sub_status = kwargs.get("status") + try: + from tools.delegate_tool import ( + SUBAGENT_FAILURE_STATUSES, + format_subagent_failure_line, + ) + if _sub_status in SUBAGENT_FAILURE_STATUSES and ctx._run_still_current(): + _line = format_subagent_failure_line( + kwargs.get("goal"), + _sub_status, + error=kwargs.get("summary") or preview, + duration_seconds=kwargs.get("duration_seconds"), + ) + safe_schedule_threadsafe( + self._runner._deliver_platform_notice(ctx.source, _line), + ctx._loop_for_step, + logger=logger, + log_message="subagent failure notice scheduling error", + ) + except Exception: + logger.debug("subagent failure notice failed", exc_info=True) + return # Live status line (Slack's assistant status): stash the current # tool phrase on the adapter; the _keep_typing refresh renders it # within a couple of seconds. Handled before every other gate @@ -6108,15 +6138,12 @@ class TurnRunner: # who set thinking_progress:true but kept tool_progress:off got a # None callback — so _thinking scratch bubbles never relayed even # though the progress queue was created for them. - agent.tool_progress_callback = ( - ctx.progress_callback - if ( - ctx.needs_progress_queue - or ctx.log_mode_enabled - or ctx._live_status_adapter is not None - ) - else None - ) + # Always attached (previously gated to None when no progress surface + # was active): the callback body gates each event class itself, and + # subagent-failure notices must fire even on platforms with + # tool_progress/thinking off — the None gate was exactly why a dead + # subagent vanished silently there. + agent.tool_progress_callback = ctx.progress_callback # Compose ID-bearing lifecycle consumers: Discord's one-time voice # ack and Slack's native task cards both ride the authoritative # start callback, so neither has to infer identity from tool names. diff --git a/tests/gateway/test_subagent_failure_notice.py b/tests/gateway/test_subagent_failure_notice.py new file mode 100644 index 0000000000..d5115bb7e5 --- /dev/null +++ b/tests/gateway/test_subagent_failure_notice.py @@ -0,0 +1,171 @@ +"""Subagent failures surface as one clean user-facing notice. + +Covers the Aug 2026 community report: a delegate_task child that dies +(provider 404, timeout, crash) previously vanished silently on platforms +with tool_progress off — the parent model saw the error but the human never +did. Now: + +- ``tools.delegate_tool.format_subagent_failure_line`` renders one clean, + human-readable line (no tracebacks/JSON walls). +- ``TurnRunner.progress_callback`` intercepts ``subagent.complete`` events + with a terminal failure status FIRST (before every progress-queue gate) + and delivers the line via ``_deliver_platform_notice``. +""" + +import asyncio +from unittest.mock import MagicMock + +import pytest + +from gateway.turn_context import TurnContext +from tools.delegate_tool import ( + SUBAGENT_FAILURE_STATUSES, + _clean_error_text, + format_subagent_failure_line, +) + + +class TestCleanErrorText: + def test_single_line_passthrough(self): + assert _clean_error_text("Error code: 404 - model not found") == ( + "Error code: 404 - model not found" + ) + + def test_traceback_takes_last_line(self): + tb = ( + "Traceback (most recent call last):\n" + ' File "x.py", line 1, in \n' + " raise RuntimeError('boom')\n" + "RuntimeError: boom" + ) + assert _clean_error_text(tb) == "RuntimeError: boom" + + def test_multiline_non_traceback_takes_first_line(self): + assert _clean_error_text("first line\nsecond line") == "first line" + + def test_caps_length(self): + out = _clean_error_text("x" * 500, max_chars=100) + assert len(out) == 100 + assert out.endswith("...") + + def test_empty_and_none(self): + assert _clean_error_text("") == "" + assert _clean_error_text(None) == "" + assert _clean_error_text(" \n ") == "" + + +class TestFormatSubagentFailureLine: + def test_failed_with_goal_error_duration(self): + line = format_subagent_failure_line( + "research competitor pricing", + "failed", + error="Error code: 404 - model not found", + duration_seconds=12.4, + ) + assert line.startswith("⚠️ Subagent failed") + assert '"research competitor pricing"' in line + assert "404" in line + assert "(after 12s)" in line + + def test_timeout_verb(self): + line = format_subagent_failure_line("do a thing", "timeout") + assert "timed out" in line + + def test_long_goal_truncated(self): + line = format_subagent_failure_line("g" * 200, "failed") + assert "g" * 57 + "..." in line + assert "g" * 61 not in line + + def test_no_goal_no_error(self): + line = format_subagent_failure_line(None, "error") + assert line == "⚠️ Subagent failed" + + def test_multiline_goal_flattened(self): + line = format_subagent_failure_line("a\nb", "failed") + assert "\n" not in line + + def test_failure_statuses_frozen(self): + assert SUBAGENT_FAILURE_STATUSES == {"failed", "error", "timeout"} + + +def _make_runner_and_captured(monkeypatch, run_still_current=True): + """TurnRunner with a stub gateway runner; captures scheduled notices.""" + from gateway import run as run_mod + + captured: list[str] = [] + + class _StubGatewayRunner: + def _adapter_for_source(self, source): + return None + + async def _deliver_platform_notice(self, source, content): + captured.append(content) + + def _fake_schedule(coro, loop, logger=None, log_message=None): + asyncio.run(coro) + + monkeypatch.setattr(run_mod, "safe_schedule_threadsafe", _fake_schedule) + + ctx = TurnContext( + source=MagicMock(), + _run_still_current=lambda: run_still_current, + progress_queue=None, + _loop_for_step=None, + ) + return run_mod.TurnRunner(_StubGatewayRunner(), ctx), captured + + +class TestGatewayFailureNotice: + @pytest.mark.parametrize("status", sorted(SUBAGENT_FAILURE_STATUSES)) + def test_failure_statuses_deliver_notice(self, monkeypatch, status): + runner, captured = _make_runner_and_captured(monkeypatch) + runner.progress_callback( + "subagent.complete", + preview="Error code: 404 - model not found", + status=status, + goal="scan the repo", + duration_seconds=8.0, + ) + assert len(captured) == 1 + assert "Subagent" in captured[0] + assert "404" in captured[0] + assert '"scan the repo"' in captured[0] + + @pytest.mark.parametrize("status", ["completed", "interrupted", None]) + def test_non_failure_statuses_stay_silent(self, monkeypatch, status): + runner, captured = _make_runner_and_captured(monkeypatch) + runner.progress_callback( + "subagent.complete", preview="all done", status=status, goal="g" + ) + assert captured == [] + + def test_stale_run_stays_silent(self, monkeypatch): + runner, captured = _make_runner_and_captured( + monkeypatch, run_still_current=False + ) + runner.progress_callback( + "subagent.complete", preview="boom", status="failed", goal="g" + ) + assert captured == [] + + def test_fires_without_progress_queue(self, monkeypatch): + """The notice must not depend on tool_progress being enabled — + progress_queue=None is exactly the Telegram/Slack default where the + silent-failure report came from.""" + runner, captured = _make_runner_and_captured(monkeypatch) + assert runner._ctx.progress_queue is None + runner.progress_callback( + "subagent.complete", preview="err", status="error", goal="g" + ) + assert len(captured) == 1 + + def test_summary_preferred_over_preview(self, monkeypatch): + runner, captured = _make_runner_and_captured(monkeypatch) + runner.progress_callback( + "subagent.complete", + preview="short preview", + status="failed", + goal="g", + summary="the real error detail", + ) + assert "the real error detail" in captured[0] diff --git a/tests/tools/test_delegate.py b/tests/tools/test_delegate.py index f7c07316ee..230e4ced40 100644 --- a/tests/tools/test_delegate.py +++ b/tests/tools/test_delegate.py @@ -675,6 +675,35 @@ class TestDelegateObservability(unittest.TestCase): result = json.loads(delegate_task(goal="Test empty sentinel", parent_agent=parent)) self.assertEqual(result["results"][0]["status"], "failed") + def test_failed_flag_marks_status_failed(self): + """Regression (Aug 2026 community report): a child whose conversation + loop aborts on a non-retryable HTTP error (404/400, billing wall) + returns failed=True with the ERROR SUMMARY in final_response. That + summary is not usable output — without checking `failed`, the entry + was classified 'completed' and no surface ever showed a failure.""" + parent = _make_mock_parent(depth=0) + + with patch("run_agent.AIAgent") as MockAgent: + mock_child = MagicMock() + mock_child.model = "totally/nonexistent-model" + mock_child.session_prompt_tokens = 0 + mock_child.session_completion_tokens = 0 + mock_child.run_conversation.return_value = { + "final_response": "HTTP 404: model not found", + "completed": False, + "failed": True, + "error": "HTTP 404: model not found", + "interrupted": False, + "api_calls": 1, + "messages": [], + } + MockAgent.return_value = mock_child + + result = json.loads(delegate_task(goal="Test failed flag", parent_agent=parent)) + entry = result["results"][0] + self.assertEqual(entry["status"], "failed") + self.assertIn("404", entry["error"]) + class TestSubagentCostRollup(unittest.TestCase): """Port of Kilo-Org/kilocode#9448 — parent's session_estimated_cost_usd diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py index 342cf62847..6cd04d5ccb 100644 --- a/tools/delegate_tool.py +++ b/tools/delegate_tool.py @@ -162,6 +162,63 @@ _RECENT_SUBAGENTS_CAP = 200 _recent_subagents: Dict[str, Dict[str, Any]] = {} +# Terminal child statuses that mean "the subagent did NOT deliver a usable +# result". Shared by the CLI spinner echo, the gateway failure notice, and +# the parent-facing failure summary so every surface agrees on what counts +# as a failure. +SUBAGENT_FAILURE_STATUSES = frozenset({"failed", "error", "timeout"}) + + +def _clean_error_text(error: Any, max_chars: int = 200) -> str: + """Reduce an arbitrary error payload to one clean human-readable line. + + Provider/SDK errors routinely arrive as multi-line tracebacks or JSON + walls. For a chat-facing notice we want the single most informative + line: the exception message (last line of a traceback) or the first + non-empty line otherwise, hard-capped in length. + """ + text = str(error or "").strip() + if not text: + return "" + lines = [ln.strip() for ln in text.splitlines() if ln.strip()] + if not lines: + return "" + # A traceback's last line is the actual exception message. + line = lines[-1] if lines[0].startswith("Traceback") else lines[0] + if len(line) > max_chars: + line = line[: max_chars - 3] + "..." + return line + + +def format_subagent_failure_line( + goal: Optional[str], + status: Optional[str], + error: Any = None, + duration_seconds: Any = None, +) -> str: + """One clean, human-readable line describing a failed subagent. + + Rendered directly to the user (CLI spinner echo, gateway platform + notice) — no JSON, no traceback, no internal field names. Example: + + ⚠️ Subagent failed — "research competitor pricing": Error code: 404 — + model not found (after 12s) + """ + goal_label = (goal or "").strip().replace("\n", " ") + if len(goal_label) > 60: + goal_label = goal_label[:57] + "..." + verb = "timed out" if status == "timeout" else "failed" + line = f"⚠️ Subagent {verb}" + if goal_label: + line += f' — "{goal_label}"' + err = _clean_error_text(error) + if err: + line += f": {err}" + if isinstance(duration_seconds, (int, float)) and duration_seconds > 0: + line += f" (after {round(duration_seconds)}s)" + return line + + def get_subagent_attribution(task_id: Optional[str]) -> Optional[Dict[str, Any]]: """Resolve a process task_id to its originating delegation, if any. @@ -1460,6 +1517,21 @@ def _build_child_progress_callback( return if event_type == "subagent.complete": + # Failed child: echo one clean reason line into the CLI tree so + # the human sees WHY, not just a vanished branch. Gateway-side + # rendering happens in TurnRunner.progress_callback off the + # relayed event below. + if spinner and kwargs.get("status") in SUBAGENT_FAILURE_STATUSES: + _fail_line = format_subagent_failure_line( + goal_label, + kwargs.get("status"), + error=kwargs.get("summary") or preview, + duration_seconds=kwargs.get("duration_seconds"), + ) + try: + spinner.print_above(f" {prefix}├─ {_fail_line}") + except Exception as e: + logger.debug("Spinner print_above failed: %s", e) _relay("subagent.complete", preview=preview, **kwargs) return @@ -3092,6 +3164,14 @@ def _run_single_child( if interrupted: status = "interrupted" + elif result.get("failed"): + # The child's conversation loop aborted (non-retryable HTTP + # error, retries exhausted, billing wall). final_response holds + # the error summary in this shape, NOT usable output — without + # this branch a provider 404/400 was classified "completed" with + # the error text as its summary, so no surface ever saw a + # failure (community report, Aug 2026). + status = "failed" elif summary and not _empty_sentinel: # A summary means the subagent produced usable output. # exit_reason ("completed" vs "max_iterations") already @@ -4106,6 +4186,15 @@ def delegate_task( icon = "✓" if status == "completed" else "✗" remaining = n_tasks - completed_count completion_line = f"{icon} [{idx+1}/{n_tasks}] {label} ({dur}s)" + # Failed/errored/timed-out children: say WHY on the + # same line, cleaned to one short human-readable + # fragment — a bare ✗ reads as "silently dropped". + if status in SUBAGENT_FAILURE_STATUSES: + _err_line = _clean_error_text( + entry.get("error"), max_chars=120 + ) + if _err_line: + completion_line += f" — {_err_line}" if spinner_ref: try: spinner_ref.print_above(completion_line) diff --git a/website/docs/guides/delegation-patterns.md b/website/docs/guides/delegation-patterns.md index 9f16bb3499..d0f5bffb27 100644 --- a/website/docs/guides/delegation-patterns.md +++ b/website/docs/guides/delegation-patterns.md @@ -235,6 +235,8 @@ delegation: **Check results.** Subagent summaries are just that — summaries. If a subagent says "fixed the bug and tests pass," verify by running the tests yourself or reading the diff. +**Failures are surfaced.** A subagent that dies (provider error, timeout, crash) is reported with a clean one-line notice — `⚠️ Subagent failed — "your goal": ` — in the CLI delegation tree and as a chat notice on gateway platforms, even when tool progress is turned off. The parent agent also receives the full error in the tool result. + --- *For the complete delegation reference — all parameters, ACP integration, and advanced configuration — see [Subagent Delegation](/user-guide/features/delegation).* diff --git a/website/docs/user-guide/features/delegation.md b/website/docs/user-guide/features/delegation.md index 52db42d895..95c7f94985 100644 --- a/website/docs/user-guide/features/delegation.md +++ b/website/docs/user-guide/features/delegation.md @@ -274,6 +274,16 @@ stopwatch kill from other failures without parsing text: `timeout_seconds` first request, `after_llm_calls` otherwise). All three are `null` on non-timeout errors. +## Failure Visibility + +A subagent that fails — non-retryable provider error (404/400), timeout, crash, or no usable output — is never silent: + +- **CLI**: the delegation tree prints a one-line reason: `⚠️ Subagent failed — "your goal": HTTP 404: model not found (after 12s)`. Batch runs append the reason to the per-task `✗` completion line. +- **Gateway platforms** (Telegram, Discord, Slack, ...): the same clean line is delivered as a standalone chat notice, **even when `tool_progress` is off** for that platform. +- **Parent agent**: the tool result entry carries `status: "failed"` plus the full `error` text, so the model can react (retry, re-route, report). + +Error text is reduced to the single most informative line (the exception message, not a traceback wall) and capped in length. + :::tip Diagnostic dump on zero-call timeout With a hard cap configured, if a subagent times out having made **zero** API calls (usually: provider unreachable, auth failure, or tool-schema rejection), `delegate_task` writes a structured diagnostic to `~/.hermes/logs/subagent-timeout--.log` containing the subagent's config snapshot, credential-resolution trace, any early error messages, and stack traces for **all** live threads (not just the child's own) — a child parked waiting on a nested helper thread is indistinguishable from a slow provider without the full picture. ::: From 452f6b7de2a1c3c1752c773f1d8b2110052693bf Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 30 Aug 2026 18:15:37 -0700 Subject: [PATCH 036/634] fix(compression): route-aware stale-thinking charge parity between compaction trigger and tail walks (#84371) The preflight trigger charged reasoning/reasoning_content on every assistant message while the tail-budget walks charged newest-turn-only (#73624), so reasoning-heavy codex_responses sessions fired compaction forever while the walk protected everything (middle_window_tokens=0, no_progress every turn, each attempt a full aux summarization). Wire truth: the codex_responses input builder never ships the text thinking keys (encrypted codex_reasoning_items carry the chain and were already charged unconditionally by both sides), so the trigger overcounted reality; echo-back chat-completions families (DeepSeek/Kimi/MiMo thinking mode) replay stored reasoning_content on every turn, so there the walk undercounted. New single wire-truth predicate message_sanitization.stale_thinking_reaches_wire() now drives BOTH sides: trigger estimates exclude stale thinking on non-echo routes; tail/prune walks charge it on echo routes. Also: reasoning/reasoning_content double-count fixed in both estimators (wire ships at most one; +53% overcount vs provider prompt_tokens per issue comment), and the commit-layer no_progress path now arms the structural no-op backoff so an unchanged-transcript compaction cannot re-fire every turn (defense in depth; overlaps the #96775 re-entry class). --- agent/context_compressor.py | 57 +++- agent/conversation_compression.py | 20 ++ agent/conversation_loop.py | 12 +- agent/message_sanitization.py | 29 ++ agent/model_metadata.py | 79 ++++- agent/turn_context.py | 20 ++ tests/agent/test_estimator_parity_84371.py | 330 +++++++++++++++++++++ 7 files changed, 541 insertions(+), 6 deletions(-) create mode 100644 tests/agent/test_estimator_parity_84371.py diff --git a/agent/context_compressor.py b/agent/context_compressor.py index 2ae3dc75f2..c092e78fb6 100644 --- a/agent/context_compressor.py +++ b/agent/context_compressor.py @@ -1531,7 +1531,18 @@ def _estimate_msg_budget_tokens(msg: dict, charge_stale_thinking: bool = True) - tokens += _serialized_length_for_budget(msg.get(key)) // _CHARS_PER_TOKEN if not charge_stale_thinking: return tokens + # The wire ships at most ONE of the generic thinking keys: every request + # build pops ``reasoning`` after (optionally) promoting it into + # ``reasoning_content`` (``apply_reasoning_content_policy``), and a + # non-empty stored ``reasoning_content`` always displaces it. Charging + # both keys double-counted the same thinking text on echo-back providers + # that persist it under both (#84371 comment: +53% vs real + # prompt_tokens). Mirror the wire: reasoning_content wins when present. + _rc = msg.get("reasoning_content") + _skip_reasoning_dup = isinstance(_rc, str) and bool(_rc.strip()) for key in _NEWEST_TURN_ONLY_BUDGET_KEYS: + if key == "reasoning" and _skip_reasoning_dup: + continue tokens += _serialized_length_for_budget(msg.get(key)) // _CHARS_PER_TOKEN # reasoning_details: charge only the thinking TEXT, never the signed / # base64 envelope (#73298 second site; mirrors the preflight estimator's @@ -3984,11 +3995,16 @@ class ContextCompressor(ContextEngine): # Same newest-turn-only thinking charge as the tail-cut walk # (#73624) — this boundary decides which tool results stay # prunable, and overcharging stale thinking shrinks that window. + # Echo-back routes charge every turn (#84371 estimator parity). _newest_asst_idx = _last_assistant_index(result) + _charge_all_thinking = self._stale_thinking_on_wire() for i in range(len(result) - 1, -1, -1): msg = result[i] msg_tokens = _estimate_msg_budget_tokens( - msg, charge_stale_thinking=(i == _newest_asst_idx) + msg, + charge_stale_thinking=( + _charge_all_thinking or i == _newest_asst_idx + ), ) if accumulated + msg_tokens > protect_tail_tokens and (len(result) - i) >= min_protect: boundary = i @@ -6553,6 +6569,30 @@ This compaction should PRIORITISE preserving all information related to the focu idx += 1 return idx + def _stale_thinking_on_wire(self) -> bool: + """Whether the active route replays stale thinking text (#84371). + + The tail-budget walks and the preflight trigger must charge the SAME + stale-thinking policy or a reasoning-heavy session can look + over-threshold to one and fully tail-protected to the other — the + infinite ineffective compaction loop. Echo-back chat-completions + families (DeepSeek/Kimi/MiMo thinking mode) replay stored + ``reasoning_content`` on EVERY assistant turn, so the walk must + charge it everywhere; codex_responses and strict providers never + ship the text keys, so newest-turn-only stands (#73624). + """ + try: + from agent.message_sanitization import stale_thinking_reaches_wire + + return stale_thinking_reaches_wire( + getattr(self, "api_mode", "") or "", + getattr(self, "provider", "") or "", + getattr(self, "model", "") or "", + getattr(self, "base_url", "") or "", + ) + except Exception: + return False + def _find_tail_cut_by_tokens( self, messages: List[Dict[str, Any]], head_end: int, token_budget: int | None = None, @@ -6598,12 +6638,20 @@ This compaction should PRIORITISE preserving all information related to the focu # fields any transport still replays (#73624) — every older turn's # reasoning/reasoning_content is stripped or padded at send time, # so charging it here spends tail budget on bytes that never ship. + # Exception: echo-back providers (DeepSeek/Kimi/MiMo thinking mode + # on chat_completions) replay stale thinking on EVERY turn — charge + # it everywhere so this walk agrees with the preflight trigger + # (#84371 estimator parity). _newest_asst_idx = _last_assistant_index(messages) + _charge_all_thinking = self._stale_thinking_on_wire() for i in range(n - 1, head_end - 1, -1): msg = messages[i] msg_tokens = _estimate_msg_budget_tokens( - msg, charge_stale_thinking=(i == _newest_asst_idx) + msg, + charge_stale_thinking=( + _charge_all_thinking or i == _newest_asst_idx + ), ) # Stop once we exceed the soft ceiling (unless we haven't hit min_tail yet) if accumulated + msg_tokens > soft_ceiling and (n - i) >= min_tail: @@ -6631,7 +6679,10 @@ This compaction should PRIORITISE preserving all information related to the focu for j in range(n - 1, head_end - 1, -1): raw_msg = messages[j] raw_tok = _estimate_msg_budget_tokens( - raw_msg, charge_stale_thinking=(j == _newest_asst_idx) + raw_msg, + charge_stale_thinking=( + _charge_all_thinking or j == _newest_asst_idx + ), ) if raw_accumulated + raw_tok > raw_budget and (n - j) >= min_tail: cut_idx = j diff --git a/agent/conversation_compression.py b/agent/conversation_compression.py index 2d9b17fa80..47ff195ead 100644 --- a/agent/conversation_compression.py +++ b/agent/conversation_compression.py @@ -4075,6 +4075,26 @@ def compress_context( "Compression made no progress (session=%s) — skipping boundary rewrite.", agent.session_id or "none", ) + # Dead-loop breaker (#84371): a fired compaction that returns the + # transcript UNCHANGED will fail identically next turn unless the + # transcript changes — yet this path recorded telemetry only, so + # auto-compress re-fired every turn, each attempt burning a full + # aux summarization (6+/10min in the wild). Arm the transient + # structural backoff so the next attempts are deferred; any + # successful boundary lifts it, and manual /compress overrides it. + try: + _no_progress_recorder = getattr( + agent.context_compressor, "_record_structural_no_op", None + ) + if callable(_no_progress_recorder): + _no_progress_recorder( + "compaction returned the transcript unchanged " + "(no_progress)" + ) + except Exception: + logger.debug( + "no-progress backoff arm failed", exc_info=True + ) _existing_sp = getattr(agent, "_cached_system_prompt", None) if not _existing_sp: _existing_sp = agent._build_system_prompt(system_message) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index f41a1a5684..f2b496a99a 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -2672,7 +2672,17 @@ def run_conversation( # messages walk inside estimate_request_tokens_rough. Tools added # separately (compression needs them: 50+ tools = 20-30K tokens). # total_chars is a rough (~) proxy — verbose log + hook metric only. - approx_tokens = estimate_messages_tokens_rough(api_messages) + # Charge stale thinking only when the active route actually replays + # it (#84371): on codex_responses the text keys never ship (the + # encrypted item sidecars — charged unconditionally — carry the + # chain), so counting them here re-created the trigger/tail-walk + # disagreement that dead-looped compaction. + from agent.turn_context import _agent_stale_thinking_on_wire + + approx_tokens = estimate_messages_tokens_rough( + api_messages, + charge_stale_thinking=_agent_stale_thinking_on_wire(agent), + ) # Route-aware pressure: when the upcoming request is eligible for # native Responses compaction the transport will checkpoint-prune # the payload before sending — the generic durable-history figure diff --git a/agent/message_sanitization.py b/agent/message_sanitization.py index 6a4c4cbbbd..d7b374a500 100644 --- a/agent/message_sanitization.py +++ b/agent/message_sanitization.py @@ -624,6 +624,7 @@ __all__ = [ "reasoning_echo_family", "matches_reasoning_echo_family", "needs_reasoning_echo", + "stale_thinking_reaches_wire", "apply_reasoning_content_policy", "reapply_reasoning_echo", ] @@ -893,6 +894,34 @@ def needs_reasoning_echo(provider: Any, model: Any, base_url: Any) -> bool: return reasoning_echo_family(provider, model, base_url) is not None +def stale_thinking_reaches_wire( + api_mode: Any, provider: Any, model: Any, base_url: Any +) -> bool: + """True when stale assistant ``reasoning``/``reasoning_content`` text is + actually replayed on the wire for the active route. + + This is the single wire-truth predicate the compaction TRIGGER estimator + and the tail-budget walks must share (#84371): when they disagree, a + reasoning-heavy session can simultaneously look over-threshold to + preflight and fully tail-protected to the walk — an infinite ineffective + compaction loop. + + * ``codex_responses``: the Responses input builder + (``_chat_messages_to_responses_input``) never reads the text keys — + reasoning continuity rides the encrypted ``codex_reasoning_items`` + sidecar, which both estimators already charge unconditionally. Stale + thinking TEXT never ships → ``False``. + * chat-completions echo-back families (DeepSeek/Kimi/MiMo thinking + mode): ``apply_reasoning_content_policy`` replays the stored + ``reasoning_content`` verbatim on EVERY assistant turn → ``True``. + * everything else: stripped or one-space-padded at send time (#73624) + → ``False``. + """ + if (api_mode or "") == "codex_responses": + return False + return needs_reasoning_echo(provider, model, base_url) + + def apply_reasoning_content_policy( source_msg: dict, api_msg: dict, needs_thinking_pad: bool ) -> None: diff --git a/agent/model_metadata.py b/agent/model_metadata.py index f0ef0011d6..2783c73458 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -3544,7 +3544,9 @@ def estimate_tokens_rough(text: str) -> int: return dense + ((sparse + 3) // 4) -def estimate_messages_tokens_rough(messages: List[Dict[str, Any]]) -> int: +def estimate_messages_tokens_rough( + messages: List[Dict[str, Any]], *, charge_stale_thinking: bool = True, +) -> int: """Rough token estimate for a message list (pre-flight only). Image parts (base64 PNG/JPEG) are counted as a flat ~1500 tokens per @@ -3552,6 +3554,19 @@ def estimate_messages_tokens_rough(messages: List[Dict[str, Any]]) -> int: character length. Without this, a single ~1MB screenshot would be estimated at ~250K tokens and trigger premature context compression. + ``charge_stale_thinking`` mirrors the tail-budget walk's policy + (``context_compressor._estimate_msg_budget_tokens``, #73624): generic + thinking text (``reasoning`` / ``reasoning_content``) rides the wire for + at most the NEWEST assistant turn on routes that do not echo stale + reasoning back (Codex Responses ships encrypted ``codex_reasoning_items`` + instead of the text keys; strict chat-completions providers strip or + one-space-pad the field). Passing ``False`` excludes those keys on every + assistant turn but the newest, so the compaction TRIGGER sees the same + size class as the tail-protection walk — the disagreement made + reasoning-heavy codex_responses sessions fire preflight forever while the + walk found nothing to compact (#84371 dead loop). Default ``True`` + preserves the conservative full charge for callers without route context. + Per-message results are memoized (see ``_estimate_message_tokens_cached``) keyed on a deep *identity fingerprint* of the message, so re-walking a long history every iteration only pays for messages whose object graph @@ -3559,12 +3574,50 @@ def estimate_messages_tokens_rough(messages: List[Dict[str, Any]]) -> int: leaf objects and structure, hence an identical estimate. """ _IMAGE_TOKEN_COST = 1500 + if not charge_stale_thinking: + messages = _strip_stale_thinking_for_estimate(messages) total = 0 for msg in messages: total += _estimate_message_tokens_cached(msg, _IMAGE_TOKEN_COST) return total +# Generic thinking-text keys replayed for at most the newest assistant turn +# on non-echo routes — must stay in lockstep with +# ``context_compressor._NEWEST_TURN_ONLY_BUDGET_KEYS``. +_STALE_THINKING_ESTIMATE_KEYS = ("reasoning", "reasoning_content") + + +def _strip_stale_thinking_for_estimate( + messages: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """Copy of ``messages`` with stale thinking keys removed (newest kept). + + Shallow stripped copies share the original value objects, so the + per-message memo still hits for the stripped shape on subsequent walks. + """ + newest = -1 + for i in range(len(messages) - 1, -1, -1): + m = messages[i] + if isinstance(m, dict) and m.get("role") == "assistant": + newest = i + break + out: List[Dict[str, Any]] = [] + for i, m in enumerate(messages): + if ( + i != newest + and isinstance(m, dict) + and m.get("role") == "assistant" + and any(m.get(k) for k in _STALE_THINKING_ESTIMATE_KEYS) + ): + m = { + k: v for k, v in m.items() + if k not in _STALE_THINKING_ESTIMATE_KEYS + } + out.append(m) + return out + + # --- Per-message token-estimate memo ------------------------------------- # # ``estimate_messages_tokens_rough`` is called on the full history every @@ -3692,10 +3745,24 @@ def _wire_message_shadow(msg: Dict[str, Any]) -> Dict[str, Any]: and bool(sidecar) and msg.get("role") in ("user", "assistant") ) + # The internal ``reasoning`` key never ships: every request build pops it + # after (optionally) promoting it into ``reasoning_content`` (see + # ``apply_reasoning_content_policy`` / conversation_loop's api_messages + # build). When a message carries BOTH keys — the normal shape on + # reasoning-echo providers, which pin ``reasoning_content`` at creation + # time while ``reasoning`` holds the same text for trajectory storage — + # counting both charged the same thinking twice and inflated the rough + # estimate by up to +53% against provider-reported prompt_tokens + # (#84371 comment data, llama.cpp/Qwen). Keep ``reasoning`` only as the + # promotion proxy when no ``reasoning_content`` exists to displace it. + _rc = msg.get("reasoning_content") + drop_reasoning_dup = isinstance(_rc, str) and bool(_rc.strip()) shadow: Dict[str, Any] = {} for k, v in msg.items(): if k in ("_anthropic_content_blocks", "reasoning_details") or k in PERSISTENCE_ONLY_MESSAGE_FIELDS: continue + if k == "reasoning" and drop_reasoning_dup: + continue if k == "api_content": # Always popped before the request is built; only counted when it # actually replaces ``content``. @@ -3750,6 +3817,7 @@ def estimate_request_tokens_rough( *, system_prompt: str = "", tools: Optional[List[Dict[str, Any]]] = None, + charge_stale_thinking: bool = True, ) -> int: """Rough token estimate for a full chat-completions request. @@ -3758,12 +3826,19 @@ def estimate_request_tokens_rough( tools enabled, schemas alone can add 20-30K tokens — a significant blind spot when only counting messages. Image content is counted at a flat per-image cost (see estimate_messages_tokens_rough). + + ``charge_stale_thinking`` is forwarded to + ``estimate_messages_tokens_rough`` — pass ``False`` when the active + route provably strips stale assistant thinking at send time (see + ``message_sanitization.stale_thinking_reaches_wire``, #84371). """ total = 0 if system_prompt: total += estimate_tokens_rough(system_prompt) if messages: - total += estimate_messages_tokens_rough(messages) + total += estimate_messages_tokens_rough( + messages, charge_stale_thinking=charge_stale_thinking + ) if tools: total += _estimate_tools_tokens_rough(tools) return total diff --git a/agent/turn_context.py b/agent/turn_context.py index 8959386bd4..698f630b69 100644 --- a/agent/turn_context.py +++ b/agent/turn_context.py @@ -99,9 +99,29 @@ def _preflight_request_tokens( messages, system_prompt=system_prompt or "", tools=tools, + charge_stale_thinking=_agent_stale_thinking_on_wire(agent), ) +def _agent_stale_thinking_on_wire(agent: Any) -> bool: + """Whether the agent's active route replays stale thinking text (#84371). + + Route facts unavailable (test doubles, partially-built agents) default to + ``True`` — the conservative full charge. + """ + try: + from agent.message_sanitization import stale_thinking_reaches_wire + + return stale_thinking_reaches_wire( + getattr(agent, "api_mode", "") or "", + getattr(agent, "provider", "") or "", + getattr(agent, "model", "") or "", + getattr(agent, "base_url", "") or "", + ) + except Exception: + return True + + def compose_user_api_content( content: Any, ext_prefetch_cache: str, diff --git a/tests/agent/test_estimator_parity_84371.py b/tests/agent/test_estimator_parity_84371.py new file mode 100644 index 0000000000..3b015646bf --- /dev/null +++ b/tests/agent/test_estimator_parity_84371.py @@ -0,0 +1,330 @@ +"""Regression tests for #84371 — compaction dead-loop on reasoning-heavy +codex_responses sessions. + +Root cause: the compaction TRIGGER (``estimate_messages_tokens_rough`` / +``estimate_request_tokens_rough``) charged stale ``reasoning`` / +``reasoning_content`` on EVERY assistant message, while the tail-protection +walk (``_find_tail_cut_by_tokens``) charged them on the newest turn only +(#73624). On a session where most tokens live in stale reasoning replay the +trigger fires above threshold while the walk protects everything — +``middle_window_tokens == 0`` / "insufficient progress" — and the same +compaction re-fires every turn, each attempt burning a full aux +summarization. + +Wire truth (``_chat_messages_to_responses_input``): the codex_responses +input builder never reads the text thinking keys; reasoning continuity rides +the encrypted ``codex_reasoning_items`` sidecar, which both estimators charge +unconditionally. So the TRIGGER overcounted reality and the fix makes the +trigger route-aware (charge stale thinking only when the route echoes it), +while echo-back chat-completions routes (DeepSeek/Kimi/MiMo thinking mode) +now charge it in the walk too — one policy per session shape, chosen by +``message_sanitization.stale_thinking_reaches_wire``. +""" + +from unittest.mock import patch + +from agent.context_compressor import ( + ContextCompressor, + _estimate_msg_budget_tokens, +) +from agent.message_sanitization import stale_thinking_reaches_wire +from agent.model_metadata import ( + estimate_messages_tokens_rough, + estimate_request_tokens_rough, +) + + +STALE_THINKING = "considering the next move carefully... " * 200 # ~2K tok + + +def _reasoning_heavy_session(n_turns: int = 40) -> list: + """Transcript whose bulk is stale reasoning replay (the #84371 shape).""" + msgs = [{"role": "system", "content": "You are Hermes."}] + msgs.append({"role": "user", "content": "do the big task"}) + for i in range(n_turns): + msgs.append( + { + "role": "assistant", + "content": f"step {i}", + "reasoning_content": STALE_THINKING, + "tool_calls": [ + { + "id": f"c{i}", + "type": "function", + "function": {"name": "t", "arguments": "{}"}, + } + ], + } + ) + msgs.append({"role": "tool", "tool_call_id": f"c{i}", "content": f"r{i}"}) + return msgs + + +class TestWireTruthPredicate: + def test_codex_responses_never_ships_stale_thinking_text(self): + assert stale_thinking_reaches_wire( + "codex_responses", "deepseek", "deepseek-v4-flash", "" + ) is False + + def test_chat_completions_echo_family_ships_it(self): + # DeepSeek thinking mode over chat_completions echoes stored + # reasoning_content back on every assistant turn. + assert stale_thinking_reaches_wire( + "", "deepseek", "deepseek-reasoner", "https://api.deepseek.com" + ) is True + + def test_strict_chat_completions_strips_it(self): + assert stale_thinking_reaches_wire( + "", "mistral", "mistral-large", "https://api.mistral.ai" + ) is False + + +class TestEstimatorParity: + """Trigger-fires must imply the walk finds a compactable middle.""" + + def test_trigger_fires_implies_walk_finds_middle(self): + msgs = _reasoning_heavy_session() + wire = stale_thinking_reaches_wire( + "codex_responses", "deepseek", "deepseek-v4-flash", "" + ) + trigger = estimate_messages_tokens_rough( + msgs, charge_stale_thinking=wire + ) + + cc = ContextCompressor( + model="deepseek-v4-flash", + provider="deepseek", + api_mode="codex_responses", + quiet_mode=True, + config_context_length=200_000, + ) + # THE RELATION, not literals: whenever the route-aware trigger says + # the session is over threshold, the tail walk must leave a real + # middle region so the fired compaction can actually make progress. + if trigger >= cc.threshold_tokens: + start = cc._protect_head_size(msgs) + end = cc._find_tail_cut_by_tokens(msgs, start) + assert end > start, ( + "trigger fired but the tail walk protected everything — " + "the #84371 dead-loop shape" + ) + middle_tokens = estimate_messages_tokens_rough(msgs[start:end]) + assert middle_tokens > 0 + + def test_route_aware_trigger_matches_walk_size_class(self): + """On codex_responses the trigger no longer counts stale thinking + the walk excludes: both figures land in the same size class.""" + msgs = _reasoning_heavy_session() + legacy = estimate_messages_tokens_rough(msgs) + route_aware = estimate_messages_tokens_rough( + msgs, charge_stale_thinking=False + ) + from agent.context_compressor import _last_assistant_index + + newest = _last_assistant_index(msgs) + walk = sum( + _estimate_msg_budget_tokens(m, charge_stale_thinking=(i == newest)) + for i, m in enumerate(msgs) + ) + # Stale thinking dominates this transcript, so the legacy figure is + # several times the walk's; the route-aware figure must not be. + assert legacy > 3 * walk + assert route_aware < 2 * walk + + def test_newest_turn_thinking_still_charged(self): + msgs = _reasoning_heavy_session(n_turns=2) + stripped = estimate_messages_tokens_rough( + msgs, charge_stale_thinking=False + ) + no_thinking = estimate_messages_tokens_rough( + [ + {k: v for k, v in m.items() + if k not in ("reasoning", "reasoning_content")} + for m in msgs + ] + ) + # The newest assistant turn's thinking survives the stale strip. + assert stripped > no_thinking + + def test_walk_charges_stale_thinking_on_echo_route(self): + """Echo-back chat_completions route: the walk now charges stale + thinking on every turn, matching the trigger's full charge; the + codex_responses route keeps newest-turn-only. Assert the per-route + charge policy directly — for each route, the walk's per-message sum + must land in the same size class as that route's trigger estimate.""" + msgs = _reasoning_heavy_session() + from agent.context_compressor import _last_assistant_index + + cc_echo = ContextCompressor( + model="deepseek-reasoner", + provider="deepseek", + api_mode="", + base_url="https://api.deepseek.com", + quiet_mode=True, + config_context_length=200_000, + ) + cc_codex = ContextCompressor( + model="deepseek-v4-flash", + provider="deepseek", + api_mode="codex_responses", + quiet_mode=True, + config_context_length=200_000, + ) + assert cc_echo._stale_thinking_on_wire() is True + assert cc_codex._stale_thinking_on_wire() is False + + newest = _last_assistant_index(msgs) + + def walk_sum(cc): + charge_all = cc._stale_thinking_on_wire() + return sum( + _estimate_msg_budget_tokens( + m, charge_stale_thinking=(charge_all or i == newest) + ) + for i, m in enumerate(msgs) + ) + + trigger_echo = estimate_messages_tokens_rough( + msgs, charge_stale_thinking=True + ) + trigger_codex = estimate_messages_tokens_rough( + msgs, charge_stale_thinking=False + ) + walk_echo = walk_sum(cc_echo) + walk_codex = walk_sum(cc_codex) + + # Per route, trigger and walk agree within a small rough-estimator + # factor — the pre-fix codex disagreement was >3x. + assert walk_echo <= trigger_echo * 2 and trigger_echo <= walk_echo * 2 + assert walk_codex <= trigger_codex * 2 and trigger_codex <= walk_codex * 2 + # And the echo route genuinely charges the stale thinking bulk. + assert walk_echo > 3 * walk_codex + + +class TestReasoningDoubleCount: + """``reasoning`` and ``reasoning_content`` carrying the same text must be + charged once — the wire ships at most one of them.""" + + def test_trigger_counts_identical_pair_once(self): + base = [{"role": "assistant", "content": "x"}] + rc_only = [{"role": "assistant", "content": "x", + "reasoning_content": "y" * 4000}] + both = [{"role": "assistant", "content": "x", + "reasoning": "y" * 4000, "reasoning_content": "y" * 4000}] + e0 = estimate_messages_tokens_rough(base) + e1 = estimate_messages_tokens_rough(rc_only) + e2 = estimate_messages_tokens_rough(both) + # Adding a duplicate `reasoning` key must not (materially) grow the + # estimate: the increment over rc_only stays far below a second copy. + assert (e2 - e0) < 1.5 * (e1 - e0) + + def test_walk_counts_identical_pair_once(self): + base = {"role": "assistant", "content": "x"} + rc_only = {"role": "assistant", "content": "x", + "reasoning_content": "y" * 4000} + both = {"role": "assistant", "content": "x", + "reasoning": "y" * 4000, "reasoning_content": "y" * 4000} + w0 = _estimate_msg_budget_tokens(base, charge_stale_thinking=True) + w1 = _estimate_msg_budget_tokens(rc_only, charge_stale_thinking=True) + w2 = _estimate_msg_budget_tokens(both, charge_stale_thinking=True) + assert (w2 - w0) < 1.5 * (w1 - w0) + + def test_reasoning_alone_still_charged(self): + """No reasoning_content to displace it → `reasoning` is the + promotion proxy and must stay counted.""" + base = [{"role": "assistant", "content": "x"}] + r_only = [{"role": "assistant", "content": "x", + "reasoning": "y" * 4000}] + assert ( + estimate_messages_tokens_rough(r_only) + > estimate_messages_tokens_rough(base) + 500 + ) + w_base = _estimate_msg_budget_tokens( + {"role": "assistant", "content": "x"}, charge_stale_thinking=True + ) + w_r = _estimate_msg_budget_tokens( + {"role": "assistant", "content": "x", "reasoning": "y" * 4000}, + charge_stale_thinking=True, + ) + assert w_r > w_base + 500 + + def test_pad_reasoning_content_does_not_displace(self): + """A one-space echo pad is not real content; `reasoning` still + carries the chargeable text.""" + msg = {"role": "assistant", "content": "x", + "reasoning": "y" * 4000, "reasoning_content": " "} + w = _estimate_msg_budget_tokens(msg, charge_stale_thinking=True) + w_base = _estimate_msg_budget_tokens( + {"role": "assistant", "content": "x", "reasoning_content": " "}, + charge_stale_thinking=True, + ) + assert w > w_base + 500 + + +class TestNoProgressDeadLoopBreaker: + """A fired compaction that returns the transcript unchanged must arm the + structural backoff so it cannot re-fire (and re-summarize) every turn.""" + + def test_no_progress_arms_structural_backoff(self): + import time + + cc = ContextCompressor( + model="deepseek-v4-flash", + provider="deepseek", + api_mode="codex_responses", + quiet_mode=True, + config_context_length=200_000, + ) + assert cc._structural_no_op_backoff_until <= time.monotonic() + cc._record_structural_no_op("compaction returned unchanged") + assert cc._structural_no_op_backoff_until > time.monotonic() + # Over-threshold but blocked: no second aux summarization this turn. + over = cc.threshold_tokens + 1 + assert cc.should_compress(over) is False + reason = cc._compression_block_reason() or "" + assert reason.startswith("structural_backoff") + + def test_commit_layer_no_progress_calls_recorder(self): + """The conversation_compression no_progress path must invoke the + compressor's structural no-op recorder (it used to record telemetry + only, so auto-compress re-fired next turn).""" + import tempfile + from pathlib import Path + from unittest.mock import MagicMock + import os + + from hermes_state import SessionDB + from run_agent import AIAgent + + with tempfile.TemporaryDirectory() as tmpdir: + db = SessionDB(db_path=Path(tmpdir) / "t.db") + with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}): + agent = AIAgent( + api_key="test-key", + base_url="https://openrouter.ai/api/v1", + model="test/model", + quiet_mode=True, + session_db=db, + session_id="s-84371", + skip_context_files=True, + skip_memory=True, + ) + agent.compression_in_place = False + compressor = MagicMock() + # No-op compression: returns input unchanged. + compressor.compress.side_effect = ( + lambda messages, **_kwargs: messages + ) + compressor._last_compress_aborted = False + agent.context_compressor = compressor + messages = [{"role": "user", "content": "request"}] + + returned, _ = agent._compress_context( + messages, "sys", approx_tokens=100 + ) + + assert returned is messages + assert compressor._record_structural_no_op.called, ( + "no_progress must arm the per-session backoff — otherwise " + "the dead loop re-fires a full aux summarization every turn" + ) From 64cc87e6681a3db4e158ed8b999ff77ba0b9d28a Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 30 Aug 2026 18:21:28 -0700 Subject: [PATCH 037/634] fix(compression): keep estimate seam positional-compatible for monkeypatched estimators Test seams and plugin engines monkeypatch estimate_messages_tokens_rough with (messages)-only signatures; route callers only pass the charge_stale_thinking kwarg on the False path. --- agent/conversation_loop.py | 10 ++++++---- agent/model_metadata.py | 12 +++++++++--- agent/turn_context.py | 8 +++++++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index f2b496a99a..c67550de78 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -2679,10 +2679,12 @@ def run_conversation( # disagreement that dead-looped compaction. from agent.turn_context import _agent_stale_thinking_on_wire - approx_tokens = estimate_messages_tokens_rough( - api_messages, - charge_stale_thinking=_agent_stale_thinking_on_wire(agent), - ) + if _agent_stale_thinking_on_wire(agent): + approx_tokens = estimate_messages_tokens_rough(api_messages) + else: + approx_tokens = estimate_messages_tokens_rough( + api_messages, charge_stale_thinking=False + ) # Route-aware pressure: when the upcoming request is eligible for # native Responses compaction the transport will checkpoint-prune # the payload before sending — the generic durable-history figure diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 2783c73458..f6fa3fdfbe 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -3836,9 +3836,15 @@ def estimate_request_tokens_rough( if system_prompt: total += estimate_tokens_rough(system_prompt) if messages: - total += estimate_messages_tokens_rough( - messages, charge_stale_thinking=charge_stale_thinking - ) + if charge_stale_thinking: + # Positional-compatible call: test seams and plugin engines + # monkeypatch estimate_messages_tokens_rough with (messages)-only + # signatures; only the route-aware False path needs the kwarg. + total += estimate_messages_tokens_rough(messages) + else: + total += estimate_messages_tokens_rough( + messages, charge_stale_thinking=False + ) if tools: total += _estimate_tools_tokens_rough(tools) return total diff --git a/agent/turn_context.py b/agent/turn_context.py index 698f630b69..3ffaa3f7cd 100644 --- a/agent/turn_context.py +++ b/agent/turn_context.py @@ -95,11 +95,17 @@ def _preflight_request_tokens( "using generic transcript estimate", exc_info=True, ) + if _agent_stale_thinking_on_wire(agent): + return estimate_request_tokens_rough( + messages, + system_prompt=system_prompt or "", + tools=tools, + ) return estimate_request_tokens_rough( messages, system_prompt=system_prompt or "", tools=tools, - charge_stale_thinking=_agent_stale_thinking_on_wire(agent), + charge_stale_thinking=False, ) From ec02d5179ada5bfafcd4f230d95f3f1a394354d1 Mon Sep 17 00:00:00 2001 From: David Metcalfe <80915+DavidMetcalfe@users.noreply.github.com> Date: Fri, 28 Aug 2026 21:37:40 -0700 Subject: [PATCH 038/634] fix(delegation): report provider-failed subagents as failed, not completed/max_iterations A provider-rejected child (e.g. HTTP 400 " is not a valid model ID") returns completed=False with failed=True + an error string as its terminal final_response. _run_single_child keyed status on summary presence alone and assumed completed=False meant iteration-budget exhaustion, so such a child was reported status=completed + exit_reason=max_iterations, rendering the false '"TRUNCATED: hit max_iterations"' banner. Consult the structured failure fields (failed / error) before falling back to the summary-presence heuristic, and derive exit_reason honestly: failure -> 'error', interrupted -> 'interrupted', completed -> 'completed', and only genuine budget exhaustion (completed=False, no failure) -> 'max_iterations'. The 'truncated' flag stays keyed on exit_reason == 'max_iterations', so it is now correct automatically. The batch renderer needed no change (the error field is already plumbed into the result entry for the parent). Closes #97655 --- tests/tools/test_delegate.py | 101 +++++++++++++++++++++++++++++++++++ tools/delegate_tool.py | 21 +++++--- 2 files changed, 115 insertions(+), 7 deletions(-) diff --git a/tests/tools/test_delegate.py b/tests/tools/test_delegate.py index 230e4ced40..a5aa206a65 100644 --- a/tests/tools/test_delegate.py +++ b/tests/tools/test_delegate.py @@ -705,6 +705,107 @@ class TestDelegateObservability(unittest.TestCase): self.assertIn("404", entry["error"]) +class TestDelegateFailedChildStatus(unittest.TestCase): + """Honest status / exit_reason for failed subagents (issue #97655). + + A child that fails on its first API call (e.g. an HTTP 400 "not a valid + model ID") returns completed=False with failed=True + an error string as + its terminal final_response. It must be reported as status=failed with an + honest exit_reason — never status=completed + exit_reason=max_iterations + (which mislabels provider rejections as iteration-budget exhaustion and + would render the false "TRUNCATED" banner). + """ + + def _delegate_single(self, child_result): + """Dispatch a single task whose mock child returns `child_result`, + returning the parsed child result entry dict.""" + parent = _make_mock_parent(depth=0) + with patch("run_agent.AIAgent") as MockAgent: + mock_child = MagicMock() + mock_child.model = "claude-sonnet-4-6" + mock_child.session_prompt_tokens = 0 + mock_child.session_completion_tokens = 0 + mock_child.run_conversation.return_value = child_result + MockAgent.return_value = mock_child + result = json.loads( + delegate_task(goal="Test child status", parent_agent=parent) + ) + return result["results"][0] + + def test_failed_flag_marks_status_failed(self): + """Regression (issue #97655): a provider-rejected child (HTTP 400 on its + first call) returns completed=False with failed=True + an error string. + It must be status=failed, exit_reason=error, and NOT truncated.""" + entry = self._delegate_single( + { + "final_response": "HTTP 400: upstage/solar-pro-4 is not a valid model ID", + "completed": False, + "interrupted": False, + "failed": True, + "error": "HTTP 400: upstage/solar-pro-4 is not a valid model ID", + "api_calls": 1, + "messages": [], + } + ) + self.assertEqual(entry["status"], "failed") + self.assertEqual(entry["exit_reason"], "error") + self.assertFalse(entry["truncated"]) + + def test_error_with_summary_still_failed(self): + """A child that returns BOTH an error field and a summary must still be + failed — the summary-presence heuristic must not override the + structured failure.""" + entry = self._delegate_single( + { + "final_response": "partial work before crashing", + "completed": False, + "interrupted": False, + "failed": True, + "error": "provider boom", + "api_calls": 3, + "messages": [], + } + ) + self.assertEqual(entry["status"], "failed") + self.assertEqual(entry["exit_reason"], "error") + self.assertFalse(entry["truncated"]) + + def test_genuine_truncation_stays_completed_max_iterations(self): + """REGRESSION GUARD: a child that genuinely exhausts its iteration + budget (completed=False, no failed flag, no error) but still returns a + summary must keep status=completed, exit_reason=max_iterations, and + truncated=True. This is the legitimate truncation path we must not + break while making failure labels honest.""" + entry = self._delegate_single( + { + "final_response": "made partial progress before the budget ran out", + "completed": False, + "interrupted": False, + "api_calls": 10, + "messages": [], + } + ) + self.assertEqual(entry["status"], "completed") + self.assertEqual(entry["exit_reason"], "max_iterations") + self.assertTrue(entry["truncated"]) + + def test_interrupted_unchanged(self): + """Interrupted children keep status=interrupted + exit_reason=interrupted + and are not marked truncated.""" + entry = self._delegate_single( + { + "final_response": "some partial output", + "completed": False, + "interrupted": True, + "api_calls": 2, + "messages": [], + } + ) + self.assertEqual(entry["status"], "interrupted") + self.assertEqual(entry["exit_reason"], "interrupted") + self.assertFalse(entry["truncated"]) + + class TestSubagentCostRollup(unittest.TestCase): """Port of Kilo-Org/kilocode#9448 — parent's session_estimated_cost_usd must include subagent spend, not just the parent's own API calls.""" diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py index 6cd04d5ccb..1d5f7f8ec2 100644 --- a/tools/delegate_tool.py +++ b/tools/delegate_tool.py @@ -3164,13 +3164,14 @@ def _run_single_child( if interrupted: status = "interrupted" - elif result.get("failed"): - # The child's conversation loop aborted (non-retryable HTTP - # error, retries exhausted, billing wall). final_response holds - # the error summary in this shape, NOT usable output — without - # this branch a provider 404/400 was classified "completed" with - # the error text as its summary, so no surface ever saw a - # failure (community report, Aug 2026). + elif result.get("failed") or result.get("error"): + # A structured failure (provider rejection / terminal exception) + # must WIN over the summary-presence heuristic below. The child's + # conversation loop returns the error text as final_response, so an + # error-shaped summary would otherwise be labeled "completed" here + # despite completed=False. The heuristic is only a fallback for + # legacy/mock results that omit the structured failure fields. + # (Community report Aug 2026; #97655.) status = "failed" elif summary and not _empty_sentinel: # A summary means the subagent produced usable output. @@ -3221,9 +3222,15 @@ def _run_single_child( # Determine exit reason if interrupted: exit_reason = "interrupted" + elif result.get("failed") or result.get("error"): + # Provider rejection / terminal failure. Do NOT report this as + # iteration-budget exhaustion — "max_iterations" is only truthful + # when the child actually hit its per-delegation iteration cap. + exit_reason = "error" elif completed: exit_reason = "completed" else: + # Genuine budget exhaustion: completed=False with no failure. exit_reason = "max_iterations" # Extract token counts (safe for mock objects) From c05d04fffb69301266f993e0bc1569bd3ac1d302 Mon Sep 17 00:00:00 2001 From: David Metcalfe <80915+DavidMetcalfe@users.noreply.github.com> Date: Fri, 28 Aug 2026 21:54:55 -0700 Subject: [PATCH 039/634] feat(delegation): surface config-level model_not_found notice in delegation batch reports When the configured Subagent Model is rejected by the provider (HTTP 400: " is not a valid model ID"), every subagent in a delegation batch dies before doing any work, but the batch report only buried the cause inside each per-task block. Detect the config-level case in the delegation batch renderer (both the multi-task fan-out and single-task variants) and emit one actionable notice at the top of the report naming the configured model + provider, and pointing at Settings -> Advanced -> Subagent Model (hermes config get delegation.model). The notice only fires when a result entry's error/summary both matches a model_not_found phrase AND names the currently configured model, so a stale task failing on a removed model isn't mis-attributed. Detection loads the delegation config lazily and fails open (no notice) on any error. When no fallback chain is configured, the notice calls out that no failover was attempted. Renderer-only change: no changes to delegate_tool status derivation or the result schema. Closes #97654. --- tests/tools/test_process_registry.py | 152 +++++++++++++++++++++++++++ tools/process_registry.py | 95 +++++++++++++++++ 2 files changed, 247 insertions(+) diff --git a/tests/tools/test_process_registry.py b/tests/tools/test_process_registry.py index 5fbde444dc..8725a7f507 100644 --- a/tests/tools/test_process_registry.py +++ b/tests/tools/test_process_registry.py @@ -2413,3 +2413,155 @@ class TestGetByPrefix: result = registry.poll("4dae56ca") assert result["session_id"] == "proc_4dae56ca81f6" assert result["status"] == "running" + + +# --------------------------------------------------------------------------- +# Config-level model_not_found notice in delegation batch reports (#97654) +# --------------------------------------------------------------------------- + + +def _make_delegation_batch_evt(results): + """A batch async-delegation event carrying a per-task ``results`` list.""" + return { + "type": "async_delegation", + "delegation_id": "deleg_97654", + "is_batch": True, + "results": results, + "goals": [r.get("goal") or "" for r in results], + "session_key": "agent:main:cli:dm:local", + "status": "completed", + "model": "upstage/solar-pro-4", + } + + +def _patch_delegation_config( + monkeypatch, model="upstage/solar-pro-4", provider="openrouter", **over +): + import tools.process_registry as _pr + + cfg = {"model": model, "provider": provider} + cfg.update(over) + monkeypatch.setattr(_pr, "_delegation_config", lambda: cfg) + return cfg + + +def _format_async(evt) -> str: + from tools.process_registry import format_process_notification + + text = format_process_notification(evt) + assert text is not None, "format_process_notification returned None" + return text + + +def test_model_not_found_notice_single_failure_once(monkeypatch): + evt = _make_delegation_batch_evt([ + { + "task_index": 0, + "status": "failed", + "exit_reason": "error", + "goal": "Create bridge module", + "error": "HTTP 400: upstage/solar-pro-4 is not a valid model ID", + "summary": "HTTP 400: upstage/solar-pro-4 is not a valid model ID", + } + ]) + _patch_delegation_config(monkeypatch) + text = _format_async(evt) + assert text is not None + assert text.count("SUBAGENT MODEL REJECTED") == 1 + assert "upstage/solar-pro-4" in text + assert "openrouter" in text + assert "No fallback chain is configured" in text + + +def test_model_not_found_notice_mixed_batch_named_model(monkeypatch): + evt = _make_delegation_batch_evt([ + { + "task_index": 0, + "status": "failed", + "exit_reason": "error", + "goal": "A", + "error": "HTTP 400: upstage/solar-pro-4 is not a valid model ID", + "summary": "HTTP 400: upstage/solar-pro-4 is not a valid model ID", + }, + { + "task_index": 1, + "status": "completed", + "goal": "B", + "summary": "ok", + "api_calls": 3, + }, + ]) + _patch_delegation_config(monkeypatch) + text = _format_async(evt) + assert text.count("SUBAGENT MODEL REJECTED") == 1 + assert "upstage/solar-pro-4" in text + + +def test_model_not_found_notice_absent_for_non_model_errors(monkeypatch): + evt = _make_delegation_batch_evt([ + { + "task_index": 0, + "status": "failed", + "goal": "A", + "error": "HTTP 429: rate limit exceeded", + }, + { + "task_index": 1, + "status": "failed", + "goal": "B", + "error": "Connection timed out", + }, + ]) + _patch_delegation_config(monkeypatch) + text = _format_async(evt) + assert "SUBAGENT MODEL REJECTED" not in text + + +def test_model_not_found_notice_absent_when_configured_model_not_named(monkeypatch): + evt = _make_delegation_batch_evt([ + { + "task_index": 0, + "status": "failed", + "goal": "A", + "error": "HTTP 400: gpt-99 is not a valid model ID", + } + ]) + # Configured model is upstage/solar-pro-4; the rejection names gpt-99. + _patch_delegation_config(monkeypatch) + text = _format_async(evt) + assert "SUBAGENT MODEL REJECTED" not in text + + +def test_model_not_found_notice_single_dispatch(monkeypatch): + evt = { + "type": "async_delegation", + "delegation_id": "deleg_single", + "session_key": "agent:main:cli:dm:local", + "goal": "task A", + "model": "upstage/solar-pro-4", + "status": "failed", + "error": "HTTP 400: upstage/solar-pro-4 is not a valid model ID", + "summary": "HTTP 400: upstage/solar-pro-4 is not a valid model ID", + } + _patch_delegation_config(monkeypatch) + text = _format_async(evt) + assert text.count("SUBAGENT MODEL REJECTED") == 1 + assert "upstage/solar-pro-4" in text + + +def test_model_not_found_notice_absent_when_fallback_chain_configured(monkeypatch): + evt = _make_delegation_batch_evt([ + { + "task_index": 0, + "status": "failed", + "goal": "A", + "error": "HTTP 400: upstage/solar-pro-4 is not a valid model ID", + } + ]) + _patch_delegation_config( + monkeypatch, + fallback_providers=[{"provider": "openrouter", "model": "upstage/solar-pro4"}], + ) + text = _format_async(evt) + assert text.count("SUBAGENT MODEL REJECTED") == 1 + assert "No fallback chain is configured" not in text diff --git a/tools/process_registry.py b/tools/process_registry.py index bca6f91a05..5d25bf7384 100644 --- a/tools/process_registry.py +++ b/tools/process_registry.py @@ -2960,6 +2960,93 @@ def _format_age(seconds: float) -> str: return f"{h}h" if m == 0 else f"{h}h{m}m" +# Model-not-found phrases lifted from agent/error_classifier.py so the +# delegation batch renderer can spot a config-level rejection without pulling +# the classifier's failover machinery. Kept in sync by hand. +_MODEL_NOT_FOUND_PATTERNS = ( + "is not a valid model", + "invalid model", + "model not found", + "model_not_found", + "does not exist", + "no such model", + "unknown model", + "unsupported model", +) + + +def _delegation_config() -> dict: + """Load the active delegation config (model/provider/fallbacks), fail-open. + + Mirrors ``tools.delegate_tool._load_config`` so the renderer sees the same + ``model`` / ``provider`` the dispatcher used, without importing the heavy + delegation module at import time. Returns ``{}`` on any error so callers + fail open to "no notice" rather than dropping the per-task blocks. + """ + try: + from tools.delegate_tool import _load_config as _cfg + + return _cfg() or {} + except Exception: + return {} + + +def _delegation_model_not_found(results, config) -> bool: + """True when a result entry reflects a config-level model_not_found rejection. + + Matches when at least one entry's error/summary text contains both a + model-not-found phrase AND the name of the currently-configured delegation + model — so a stale task failing on a *different* (removed) model is not + mis-attributed to the config-level root cause. + """ + model = (config or {}).get("model") + if not model: + return False + model = str(model).lower() + for r in results or []: + text = " ".join( + str(part) for part in (r.get("error"), r.get("summary")) if part + ).lower() + if not text or model not in text: + continue + if any(p in text for p in _MODEL_NOT_FOUND_PATTERNS): + return True + return False + + +def _delegation_model_not_found_notice(results) -> "list[str] | None": + """Build the config-level model_not_found notice lines, or None. + + Returns ``None`` unless at least one result entry shows the configured + delegation model being rejected by its provider, in which case a short + actionable block is returned. Every failure path fails open to ``None`` so + a config hiccup never hides the per-task blocks. Emit once per batch. + """ + config = _delegation_config() + if not _delegation_model_not_found(results, config): + return None + model = config.get("model") or "?" + provider = config.get("provider") or "configured provider" + lines = [ + "⚠ SUBAGENT MODEL REJECTED: the configured Subagent Model " + f'"{model}" was rejected by provider "{provider}" ' + "(HTTP 400: not a valid model ID).", + "Every task in this batch failed for this reason before doing any work.", + "Check Settings → Advanced → Subagent Model (or: " + "hermes config get delegation.model).", + ] + try: + from hermes_cli.fallback_config import get_fallback_chain + + if not get_fallback_chain(config): + lines.append( + "No fallback chain is configured, so no failover was attempted." + ) + except Exception: + pass + return lines + + def _format_async_delegation(evt: dict) -> str: """Format an async-delegation completion into a self-contained re-injection. @@ -3018,6 +3105,10 @@ def _format_async_delegation(evt: dict) -> str: lines.append("--- ERROR ---") lines.append(f"The batch did not complete successfully: {error}") return "\n".join(lines) + _notice = _delegation_model_not_found_notice(results) + if _notice: + lines.append("") + lines.extend(_notice) for r in sorted(results, key=lambda x: x.get("task_index", 0)): idx = r.get("task_index", 0) r_status = r.get("status", "?") @@ -3085,6 +3176,10 @@ def _format_async_delegation(evt: dict) -> str: if toolsets: lines.append(f"Toolsets: {', '.join(toolsets)}") lines.append(f"Role: {role} Model: {model}") + _notice = _delegation_model_not_found_notice([evt]) + if _notice: + lines.append("") + lines.extend(_notice) _trunc = " [TRUNCATED: hit max_iterations — work may be incomplete]" if truncated else "" lines.append(f"Status: {status} API calls: {api_calls} Duration: {duration}s{_trunc}") lines.append("--- RESULT ---") From b4d5174385e43d502fe078cceb9eda09e82c7d5e Mon Sep 17 00:00:00 2001 From: David Metcalfe <80915+DavidMetcalfe@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:49:28 -0700 Subject: [PATCH 040/634] fix(delegation): pin failure-status edge cases and document exit_reason enum MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-finding verdicts from the cross-vendor review of fix/status-fix (#97655/#97654): [1] Flash NIT (real, cheap) — FIXED. Added test_error_without_failed_flag_ marks_failed: an error string with the 'failed' key ABSENT (not False) must still be status=failed + exit_reason=error. The branch order (result.get('failed') or result.get('error')) already handles this; the test pins the error-alone path. [2] GPT-OSS SHOULD-FIX — PINNED. Added test_empty_error_with_summary_is_ completed: error='' is falsy so result.get('error') falls through to the summary-presence heuristic => status=completed. No code change; the existing branch is correct and the new test locks it in. [3] GPT-OSS SHOULD-FIX — VERIFIED, NO CHANGE. Grepped every delegation exit_reason consumer: * tools/delegation_live_log.py finalize() prints exit_reason generically and only special-cases == 'max_iterations' for a readable suffix. * tools/process_registry.py derives truncated as (truncated or exit_reason == 'max_iterations') — gated, not exhaustive. * tools/async_delegation.py passes exit_reason through generically. The gateway/status.py, cron/scheduler.py and run_agent.py 'exit_reason' hits are a DIFFERENT field (turn_exit_reason / gateway exit reason), not the delegation result's exit_reason. No exhaustive if/elif over the enum missing an 'error' case, so nothing to add. [4] GPT-OSS NIT — DONE. Enriched _run_single_child's docstring to enumerate status in {completed, interrupted, failed} and exit_reason in {completed, max_iterations, interrupted, error}, and added a compact enum comment at the result-entry construction. Verified the process_registry.py renderer comment (truncated <= exit_reason == 'max_iterations') still holds — the truncation flag is derived exactly that way, so no contradiction. [5] GPT-OSS NIT — REJECTED. The proposed 'fallback for legacy dicts that explicitly set failed=False' is not adopted. No consumer produces a result dict with an explicit failed=False and no summary while relying on completed semantics: run_agent.py sets failed=True only on genuine failure and omits the key on success (no failed=False producer). Also, the proposed elif would reintroduce ambiguity (explicit failed=False + no summary => 'completed'?) and diverge from the conservative else => 'failed'. result.get('failed') is falsy for both explicit-False and absent, so no distinction exists to preserve; the else is the correct default. Tests: 301 passed, 7 skipped (tests/tools -k 'delegate or process_registry'). TestDelegateFailedChildStatus: 6 passed. --- tests/tools/test_delegate.py | 41 ++++++++++++++++++++++++++++++++++++ tools/delegate_tool.py | 26 ++++++++++++++++++++++- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/tests/tools/test_delegate.py b/tests/tools/test_delegate.py index a5aa206a65..ec09b6e374 100644 --- a/tests/tools/test_delegate.py +++ b/tests/tools/test_delegate.py @@ -770,6 +770,47 @@ class TestDelegateFailedChildStatus(unittest.TestCase): self.assertEqual(entry["exit_reason"], "error") self.assertFalse(entry["truncated"]) + def test_error_without_failed_flag_marks_failed(self): + """A child result that carries a non-empty error string but OMITS the + ``failed`` key entirely (not ``failed=False`` — the key is absent, as in + legacy/partial result dicts) must still be status=failed + exit_reason=error. + The status branch checks ``result.get('failed') or result.get('error')``, + so the error field alone has to win — otherwise a dropped ``failed`` key + would silently mislabel a provider rejection as budget exhaustion.""" + entry = self._delegate_single( + { + "final_response": "connection reset while streaming", + "completed": False, + "interrupted": False, + "error": "connection reset", + "api_calls": 2, + "messages": [], + } + ) + self.assertEqual(entry["status"], "failed") + self.assertEqual(entry["exit_reason"], "error") + self.assertFalse(entry["truncated"]) + + def test_empty_error_with_summary_is_completed(self): + """REGRESSION PIN: an empty-string ``error`` field must NOT be treated as + a failure. ``result.get('error')`` returns ``''`` which is falsy, so the + failure branch correctly falls through to the summary-presence heuristic. + Empty error + a real summary => status=completed, exit_reason=completed + (or max_iterations if completed=False), never 'error'.""" + entry = self._delegate_single( + { + "final_response": "work produced", + "completed": True, + "interrupted": False, + "error": "", + "api_calls": 2, + "messages": [], + } + ) + self.assertEqual(entry["status"], "completed") + self.assertEqual(entry["exit_reason"], "completed") + self.assertFalse(entry["truncated"]) + def test_genuine_truncation_stays_completed_max_iterations(self): """REGRESSION GUARD: a child that genuinely exhausts its iteration budget (completed=False, no failed flag, no error) but still returns a diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py index 1d5f7f8ec2..89cab103c5 100644 --- a/tools/delegate_tool.py +++ b/tools/delegate_tool.py @@ -2579,7 +2579,27 @@ def _run_single_child( ) -> Dict[str, Any]: """ Run a pre-built child agent. Called from within a thread. - Returns a structured result dict. + Returns a structured result dict with a ``status`` and ``exit_reason`` + that are derived honestly from the child's structured completion fields. + + ``status`` ∈ {``"completed"``, ``"interrupted"``, ``"failed"``}: + * ``"completed"`` — the child reached a normal finish (may still have + hit its iteration budget; see ``exit_reason``). + * ``"interrupted"`` — the child was interrupted (``interrupted=True``). + * ``"failed"`` — a structured failure (``failed=True`` or a non-empty + ``error``) or a summary-less/invalid terminal state. + + ``exit_reason`` ∈ {``"completed"``, ``"max_iterations"``, ``"interrupted"``, + ``"error"``}: + * ``"completed"`` — normal finish. + * ``"max_iterations"`` — genuine per-child iteration-budget exhaustion + (``completed=False`` with no failure fields). + * ``"interrupted"`` — interrupted by the parent. + * ``"error"`` — provider rejection / terminal failure; NOT + budget exhaustion (this is the case #97655 fixed). + + ``truncated`` is derived as ``exit_reason == "max_iterations"`` only, so the + parent-visible truncation flag stays truthful for all of the above. """ child_start = time.monotonic() @@ -3238,6 +3258,10 @@ def _run_single_child( _output_tokens = getattr(child, "session_completion_tokens", 0) _model = getattr(child, "model", None) + # --- result entry contract (see _run_single_child docstring) --- + # status ∈ {completed, interrupted, failed} + # exit_reason ∈ {completed, max_iterations, interrupted, error} + # truncated is exactly (exit_reason == "max_iterations"). entry: Dict[str, Any] = { "task_index": task_index, "status": status, From cd2bd160579d5240e52d01e2f735da55ff4242ef Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 30 Aug 2026 20:42:45 -0700 Subject: [PATCH 041/634] test: drop duplicate failed-flag regression test superseded by TestDelegateFailedChildStatus --- tests/tools/test_delegate.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/tests/tools/test_delegate.py b/tests/tools/test_delegate.py index ec09b6e374..598819cf6e 100644 --- a/tests/tools/test_delegate.py +++ b/tests/tools/test_delegate.py @@ -675,35 +675,6 @@ class TestDelegateObservability(unittest.TestCase): result = json.loads(delegate_task(goal="Test empty sentinel", parent_agent=parent)) self.assertEqual(result["results"][0]["status"], "failed") - def test_failed_flag_marks_status_failed(self): - """Regression (Aug 2026 community report): a child whose conversation - loop aborts on a non-retryable HTTP error (404/400, billing wall) - returns failed=True with the ERROR SUMMARY in final_response. That - summary is not usable output — without checking `failed`, the entry - was classified 'completed' and no surface ever showed a failure.""" - parent = _make_mock_parent(depth=0) - - with patch("run_agent.AIAgent") as MockAgent: - mock_child = MagicMock() - mock_child.model = "totally/nonexistent-model" - mock_child.session_prompt_tokens = 0 - mock_child.session_completion_tokens = 0 - mock_child.run_conversation.return_value = { - "final_response": "HTTP 404: model not found", - "completed": False, - "failed": True, - "error": "HTTP 404: model not found", - "interrupted": False, - "api_calls": 1, - "messages": [], - } - MockAgent.return_value = mock_child - - result = json.loads(delegate_task(goal="Test failed flag", parent_agent=parent)) - entry = result["results"][0] - self.assertEqual(entry["status"], "failed") - self.assertIn("404", entry["error"]) - class TestDelegateFailedChildStatus(unittest.TestCase): """Honest status / exit_reason for failed subagents (issue #97655). From 93c7089f709f661b4ded75451b5953acbff8e02b Mon Sep 17 00:00:00 2001 From: David Dudok de Wit <5354424+dokterdok@users.noreply.github.com> Date: Sun, 30 Aug 2026 22:17:19 +0200 Subject: [PATCH 042/634] feat(bot-mode): run same-gateway Group Chats without Desktop --- gateway/hosted_room_discussion.py | 1320 +++++++++++++ gateway/hosted_room_driver.py | 1649 +++++++++++++++++ gateway/hosted_room_policy_checkpoint.py | 682 +++++++ gateway/hosted_rooms.py | 133 +- gateway/run.py | 66 + hermes_cli/web_server.py | 28 + tests/gateway/test_hosted_room_discussion.py | 708 +++++++ tests/gateway/test_hosted_room_driver.py | 1110 +++++++++++ .../test_hosted_room_gateway_lifecycle.py | 234 +++ .../test_hosted_room_local_boundary.py | 38 + tests/gateway/test_hosted_rooms.py | 145 +- tests/gateway/test_hosted_rooms_read_path.py | 31 + .../test_web_server_boot_handshake.py | 23 + tests/tui_gateway/test_auto_continue.py | 69 +- tests/tui_gateway/test_groups_methods.py | 197 +- .../test_hosted_room_driver_runtime.py | 1366 ++++++++++++++ .../test_hosted_room_prompt_fence.py | 151 ++ .../test_hosted_room_server_rpc.py | 177 ++ tests/tui_gateway/test_hosted_room_service.py | 748 ++++++++ tui_gateway/hosted_room_driver.py | 1276 +++++++++++++ tui_gateway/hosted_room_server_rpc.py | 213 +++ tui_gateway/hosted_room_service.py | 518 ++++++ tui_gateway/methods_bot_relay.py | 7 + tui_gateway/methods_groups.py | 264 ++- tui_gateway/methods_prompt.py | 80 +- tui_gateway/methods_session.py | 12 + tui_gateway/server.py | 61 +- 27 files changed, 11210 insertions(+), 96 deletions(-) create mode 100644 gateway/hosted_room_discussion.py create mode 100644 gateway/hosted_room_driver.py create mode 100644 gateway/hosted_room_policy_checkpoint.py create mode 100644 tests/gateway/test_hosted_room_discussion.py create mode 100644 tests/gateway/test_hosted_room_driver.py create mode 100644 tests/gateway/test_hosted_room_gateway_lifecycle.py create mode 100644 tests/gateway/test_hosted_room_local_boundary.py create mode 100644 tests/gateway/test_hosted_rooms_read_path.py create mode 100644 tests/tui_gateway/test_hosted_room_driver_runtime.py create mode 100644 tests/tui_gateway/test_hosted_room_prompt_fence.py create mode 100644 tests/tui_gateway/test_hosted_room_server_rpc.py create mode 100644 tests/tui_gateway/test_hosted_room_service.py create mode 100644 tui_gateway/hosted_room_driver.py create mode 100644 tui_gateway/hosted_room_server_rpc.py create mode 100644 tui_gateway/hosted_room_service.py diff --git a/gateway/hosted_room_discussion.py b/gateway/hosted_room_discussion.py new file mode 100644 index 0000000000..dc4c26fde1 --- /dev/null +++ b/gateway/hosted_room_discussion.py @@ -0,0 +1,1320 @@ +"""Deterministic policy for same-gateway hosted-room Discussions. + +This module translates a frozen local member roster and the complete typed room +log into one next driver task. It performs no I/O, starts no workers, and knows +nothing about transports or model runtimes. Callers persist the returned task +with :mod:`gateway.hosted_room_driver` and append publication plans with +:mod:`gateway.hosted_rooms`. + +The unpublished driver payload intentionally remains unchanged. Discussion +coordinates live in deterministic ``TaskIdentity`` values and typed terminal +events; a restart can therefore reconstruct a task without widening the driver +schema. Callers must reconcile terminal driver rows into publication plans +before asking for the next task. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from typing import Any, Literal + +from gateway import hosted_room_driver as driver +from gateway import hosted_rooms + + +MAX_DISCUSSION_MEMBERS = 6 +MIN_DISCUSSION_MEMBERS = 2 +MAX_DISCUSSION_ROUNDS = 3 +MAX_DISCUSSION_MESSAGES = 10 +MAX_DISCUSSION_DELTA_LINES = 24 +MAX_USER_TEXT_BYTES = 64 * 1024 +MAX_MEMBER_TEXT_BYTES = 64 * 1024 +_TRUNCATED_REPLY_NOTICE = ( + "\n\n[Reply truncated. Ask the Bot to share the full result as a file.]" +) + +DecisionStatus = Literal["idle", "task", "settled", "bounded"] +TerminalKind = Literal["settled", "failed", "cancelled", "deferred"] + +_IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]*$") +_MENTION_RE = re.compile(r"@([A-Za-z0-9][A-Za-z0-9._:-]*)", re.IGNORECASE) +_TURN_ID_RE = re.compile( + r"^d(?P[1-9][0-9]*)\.r(?P[0-2])\." + r"p(?P[0-5])\.s(?P[1-9][0-9]*)\." + r"m(?P[0-9a-f]{24})$" +) + +_MEMBER_FIELDS = frozenset({"member_id", "profile", "handle", "display_name"}) +_REMOTE_MEMBER_FIELDS = frozenset({ + "connectionId", + "connectionKind", + "connectionLabel", + "connection_id", + "connection_kind", + "connection_label", + "remoteSource", + "route", + "sourceMissing", + "sourceReachable", + "sourceScoped", + "targetProfile", + "target_profile", +}) +_USER_PAYLOAD_FIELDS = frozenset({"text", "thread_id"}) +_MEMBER_MESSAGE_FIELDS = frozenset({ + "discussion_event_id", + "member_id", + "member_index", + "round_index", + "task_id", + "text", + "thread_id", + "turn_id", +}) +_TERMINAL_COMMON_FIELDS = frozenset({ + "discussion_event_id", + "member_id", + "member_index", + "round_index", + "seen_through_seq", + "task_id", + "thread_id", + "turn_id", +}) +_TERMINAL_EXTRA_FIELDS = { + "turn.settled": frozenset({"message_event_id", "passed"}), + "turn.failed": frozenset({"error"}), + "turn.cancelled": frozenset({"reason"}), + "turn.deferred": frozenset({"execution_generation", "reason"}), +} +_TERMINAL_EVENT_KINDS = frozenset(_TERMINAL_EXTRA_FIELDS) +_ROOM_ACTIVITY_FIELDS = frozenset({ + "status", + "reason_code", + "thread_id", + "discussion_event_id", +}) +_ROOM_STOP_FIELDS = frozenset({"cancel_id"}) + + +class DiscussionPolicyError(ValueError): + """Base class for invalid policy input or unreconstructable state.""" + + +class DiscussionValidationError(DiscussionPolicyError): + """Raised when a room, roster, payload, or typed event is malformed.""" + + +class DiscussionReconstructionError(DiscussionPolicyError): + """Raised when a persisted task cannot be reproduced from durable state.""" + + +@dataclass(frozen=True) +class DiscussionMember: + """One immutable member local to the room's authority gateway.""" + + member_id: str + profile: str + handle: str + display_name: str = "" + + +@dataclass(frozen=True) +class DiscussionRoom: + """Validated policy projection of one active hosted room.""" + + room_id: str + name: str + members: tuple[DiscussionMember, ...] + gateway_id: str + authority_epoch: int + + +@dataclass(frozen=True) +class DiscussionTaskPlan: + """One deterministic member turn compatible with the driver schema.""" + + identity: driver.TaskIdentity + payload: Mapping[str, Any] + discussion_event_id: str + member: DiscussionMember + member_index: int + round_index: int + seen_through_seq: int + + +@dataclass(frozen=True) +class DiscussionDecision: + """Current result of replaying one room's Discussion policy.""" + + status: DecisionStatus + reason: str + discussion_event_id: str | None = None + source_event_seq: int | None = None + thread_id: str | None = None + task: DiscussionTaskPlan | None = None + + +@dataclass(frozen=True) +class EventPlan: + """One idempotent append for :func:`gateway.hosted_rooms.append_event`.""" + + event_id: str + kind: str + actor: Mapping[str, str] + payload: Mapping[str, Any] + authority_gateway_id: str + authority_epoch: int + + def append_kwargs(self, room_id: str) -> dict[str, Any]: + """Return keyword arguments accepted by ``append_event``.""" + + return { + "room_id": room_id, + "event_id": self.event_id, + "kind": self.kind, + "actor": dict(self.actor), + "payload": dict(self.payload), + "authority_gateway_id": self.authority_gateway_id, + "authority_epoch": self.authority_epoch, + } + + +@dataclass(frozen=True) +class PublicationPlan: + """Ordered visible and terminal effects for one driver task.""" + + task_id: str + terminal_kind: str + events: tuple[EventPlan, ...] + + +@dataclass(frozen=True) +class _ValidatedEvent: + raw: Mapping[str, Any] + seq: int + event_id: str + kind: str + actor: Mapping[str, Any] + payload: Mapping[str, Any] + + +def _identifier(value: Any, *, label: str) -> str: + if not isinstance(value, str): + raise DiscussionValidationError(f"{label} must be a string") + normalized = value.strip() + if ( + not normalized + or len(normalized) > driver.MAX_IDENTIFIER_CHARS + or not _IDENTIFIER_RE.fullmatch(normalized) + ): + raise DiscussionValidationError(f"invalid {label}") + return normalized + + +def _positive_int(value: Any, *, label: str, maximum: int | None = None) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise DiscussionValidationError(f"{label} must be a positive integer") + if maximum is not None and value > maximum: + raise DiscussionValidationError(f"{label} must be at most {maximum}") + return value + + +def _zero_based_int(value: Any, *, label: str, maximum: int) -> int: + if ( + isinstance(value, bool) + or not isinstance(value, int) + or not 0 <= value <= maximum + ): + raise DiscussionValidationError( + f"{label} must be an integer between 0 and {maximum}" + ) + return value + + +def _exact_fields( + value: Any, + *, + label: str, + required: frozenset[str], + optional: frozenset[str] = frozenset(), +) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise DiscussionValidationError(f"{label} must be an object") + keys = frozenset(value) + missing = required - keys + unknown = keys - required - optional + if missing: + raise DiscussionValidationError( + f"{label} is missing fields: {', '.join(sorted(missing))}" + ) + if unknown: + raise DiscussionValidationError( + f"{label} has unknown fields: {', '.join(sorted(unknown))}" + ) + return value + + +def validate_user_payload(value: Any) -> dict[str, Any]: + """Validate and normalize the exact ``message.user`` Discussion payload.""" + + payload = _exact_fields( + value, + label="user payload", + required=_USER_PAYLOAD_FIELDS, + ) + text = payload["text"] + if not isinstance(text, str): + raise DiscussionValidationError("user payload text must be a string") + text = text.strip() + if not text: + raise DiscussionValidationError("user payload text must not be empty") + if len(text.encode("utf-8")) > MAX_USER_TEXT_BYTES: + raise DiscussionValidationError("user payload text is too large") + thread_id = _identifier(payload["thread_id"], label="thread_id") + return {"text": text, "thread_id": thread_id} + + +def validate_roster( + value: Any, + *, + local_profiles: Iterable[str], +) -> tuple[DiscussionMember, ...]: + """Validate a frozen 2-6 member roster of profiles on this gateway.""" + + if not isinstance(value, list): + raise DiscussionValidationError("members must be a list") + if not MIN_DISCUSSION_MEMBERS <= len(value) <= MAX_DISCUSSION_MEMBERS: + raise DiscussionValidationError( + f"members must contain between {MIN_DISCUSSION_MEMBERS} and " + f"{MAX_DISCUSSION_MEMBERS} entries" + ) + + known_profiles = { + _identifier(profile, label="local profile") for profile in local_profiles + } + members: list[DiscussionMember] = [] + profiles: set[str] = set() + handles: set[str] = set() + member_ids: set[str] = set() + + for index, raw in enumerate(value): + if not isinstance(raw, Mapping): + raise DiscussionValidationError(f"member {index} must be an object") + remote_fields = frozenset(raw) & _REMOTE_MEMBER_FIELDS + if remote_fields: + raise DiscussionValidationError( + f"member {index} contains cross-gateway fields: " + f"{', '.join(sorted(remote_fields))}" + ) + member = _exact_fields( + raw, + label=f"member {index}", + required=frozenset({"member_id", "profile", "handle"}), + optional=frozenset({"display_name"}), + ) + member_id = _identifier(member["member_id"], label=f"member {index} id") + profile = _identifier(member["profile"], label=f"member {index} profile") + handle = _identifier(member["handle"], label=f"member {index} handle") + if profile not in known_profiles: + raise DiscussionValidationError( + f"member {index} profile '{profile}' is not local to this gateway" + ) + display_name = member.get("display_name", "") + if not isinstance(display_name, str): + raise DiscussionValidationError( + f"member {index} display_name must be a string" + ) + display_name = display_name.strip() + if len(display_name) > hosted_rooms.MAX_ACTOR_LABEL_CHARS: + raise DiscussionValidationError(f"member {index} display_name is too long") + + profile_key = profile.casefold() + handle_key = handle.casefold() + member_key = member_id.casefold() + if profile_key in profiles: + raise DiscussionValidationError("member profiles must be unique") + if handle_key in handles or handle_key in {"all", "everyone"}: + raise DiscussionValidationError( + "member handles must be unique and cannot reserve @all or @everyone" + ) + if member_key in member_ids: + raise DiscussionValidationError("member ids must be unique") + profiles.add(profile_key) + handles.add(handle_key) + member_ids.add(member_key) + members.append( + DiscussionMember( + member_id=member_id, + profile=profile, + handle=handle, + display_name=display_name, + ) + ) + return tuple(members) + + +def validate_room( + value: Any, + *, + local_profiles: Iterable[str], +) -> DiscussionRoom: + """Project a hosted-room row into the strict same-gateway policy shape.""" + + if not isinstance(value, Mapping): + raise DiscussionValidationError("room must be an object") + if value.get("disbanded_at") is not None: + raise DiscussionValidationError("room is disbanded") + room_id = _identifier(value.get("room_id"), label="room_id") + name = value.get("name") + if not isinstance(name, str) or not name.strip(): + raise DiscussionValidationError("room name must be a non-empty string") + name = name.strip() + if len(name) > hosted_rooms.MAX_ROOM_NAME_CHARS: + raise DiscussionValidationError("room name is too long") + gateway_id = _identifier( + value.get("authority_gateway_id"), + label="authority_gateway_id", + ) + authority_epoch = _positive_int( + value.get("authority_epoch"), + label="authority_epoch", + ) + members = validate_roster(value.get("members"), local_profiles=local_profiles) + return DiscussionRoom( + room_id=room_id, + name=name, + members=members, + gateway_id=gateway_id, + authority_epoch=authority_epoch, + ) + + +def is_pass_text(value: Any) -> bool: + """Return whether a settled member result is Discussion silence.""" + + text = str(value or "").strip() + return ( + not text + or re.fullmatch(r"\(?\s*pass\s*\)?\.?", text, re.IGNORECASE) is not None + ) + + +def resolve_mentions( + texts: Iterable[str], + members: Sequence[DiscussionMember], + *, + default_all: bool = True, +) -> tuple[DiscussionMember, ...]: + """Resolve member handles deterministically against the frozen roster.""" + + by_handle = {member.handle.casefold(): member for member in members} + mentioned: set[str] = set() + everyone = False + for text in texts: + for match in _MENTION_RE.finditer(str(text or "")): + handle = match.group(1).casefold() + if handle in {"all", "everyone"}: + everyone = True + elif handle in by_handle: + mentioned.add(handle) + if everyone or (default_all and not mentioned): + return tuple(members) + return tuple(member for member in members if member.handle.casefold() in mentioned) + + +def _unaddressed_member_mentions( + messages: Sequence[_ValidatedEvent], + room: DiscussionRoom, +) -> tuple[DiscussionMember, ...]: + """Return peers explicitly cited by a Bot and not heard from afterward.""" + + cited_at: dict[str, int] = {} + last_post_at: dict[str, int] = {} + for event in messages: + if event.kind != "message.member": + continue + speaker_id = str(event.payload["member_id"]) + last_post_at[speaker_id] = event.seq + cited = resolve_mentions( + (str(event.payload["text"]),), + room.members, + default_all=False, + ) + for member in cited: + if member.member_id != speaker_id: + cited_at[member.member_id] = event.seq + return tuple( + member + for member in room.members + if member.member_id in cited_at + and last_post_at.get(member.member_id, 0) <= cited_at[member.member_id] + ) + + +def _validate_event( + raw: Any, + *, + room: DiscussionRoom, + previous_seq: int, +) -> _ValidatedEvent: + if not isinstance(raw, Mapping): + raise DiscussionValidationError("room event must be an object") + if raw.get("room_id") != room.room_id: + raise DiscussionValidationError("room event belongs to a different room") + seq = _positive_int(raw.get("seq"), label="event seq") + if seq <= previous_seq: + raise DiscussionValidationError("room events must be in strict sequence order") + event_id = _identifier(raw.get("event_id"), label="event_id") + kind = raw.get("kind") + if not isinstance(kind, str): + raise DiscussionValidationError("event kind must be a string") + actor = raw.get("actor") + if not isinstance(actor, Mapping): + raise DiscussionValidationError("event actor must be an object") + payload = raw.get("payload") + if not isinstance(payload, Mapping): + raise DiscussionValidationError("event payload must be an object") + + if kind == "message.user": + payload = validate_user_payload(payload) + if actor.get("kind") != "user": + raise DiscussionValidationError("message.user requires a user actor") + elif kind == "message.member": + if raw.get("authority_epoch") != room.authority_epoch: + raise DiscussionValidationError( + "message.member authority epoch does not match the room" + ) + _validate_member_message(payload, actor=actor, room=room) + elif kind in _TERMINAL_EVENT_KINDS: + if raw.get("authority_epoch") != room.authority_epoch: + raise DiscussionValidationError( + f"{kind} authority epoch does not match the room" + ) + _validate_terminal_event(kind, payload, actor=actor, room=room) + elif kind == "room.activity": + if raw.get("authority_epoch") != room.authority_epoch: + raise DiscussionValidationError( + "room.activity authority epoch does not match the room" + ) + _exact_fields( + payload, + label="room.activity payload", + required=_ROOM_ACTIVITY_FIELDS, + ) + if payload.get("status") not in {"settled", "bounded"}: + raise DiscussionValidationError("invalid room.activity status") + _identifier(payload.get("reason_code"), label="reason_code") + _identifier(payload.get("thread_id"), label="thread_id") + _identifier(payload.get("discussion_event_id"), label="discussion_event_id") + if actor.get("kind") != "gateway" or actor.get("id") != room.gateway_id: + raise DiscussionValidationError("room.activity requires the room gateway") + elif kind == "room.stop_requested": + if raw.get("authority_epoch") != room.authority_epoch: + raise DiscussionValidationError( + "room.stop_requested authority epoch does not match the room" + ) + _exact_fields( + payload, + label="room.stop_requested payload", + required=_ROOM_STOP_FIELDS, + ) + _identifier(payload.get("cancel_id"), label="cancel_id") + if actor.get("kind") != "gateway" or actor.get("id") != room.gateway_id: + raise DiscussionValidationError( + "room.stop_requested requires the room gateway" + ) + + return _ValidatedEvent( + raw=raw, + seq=seq, + event_id=event_id, + kind=kind, + actor=actor, + payload=payload, + ) + + +def _member_by_id(room: DiscussionRoom, member_id: Any) -> DiscussionMember: + normalized = _identifier(member_id, label="member_id") + for member in room.members: + if member.member_id == normalized: + return member + raise DiscussionValidationError(f"unknown Discussion member '{normalized}'") + + +def _validate_turn_coordinates( + payload: Mapping[str, Any], room: DiscussionRoom +) -> None: + _member_by_id(room, payload.get("member_id")) + member_index = _zero_based_int( + payload.get("member_index"), + label="member_index", + maximum=MAX_DISCUSSION_MEMBERS - 1, + ) + round_index = _zero_based_int( + payload.get("round_index"), + label="round_index", + maximum=MAX_DISCUSSION_ROUNDS - 1, + ) + thread_id = _identifier(payload.get("thread_id"), label="thread_id") + task_id = _identifier(payload.get("task_id"), label="task_id") + turn_id = _identifier(payload.get("turn_id"), label="turn_id") + discussion_event_id = _identifier( + payload.get("discussion_event_id"), + label="discussion_event_id", + ) + del member_index, round_index, thread_id, task_id, turn_id, discussion_event_id + + +def _validate_member_message( + payload: Mapping[str, Any], + *, + actor: Mapping[str, Any], + room: DiscussionRoom, +) -> None: + _exact_fields( + payload, + label="message.member payload", + required=_MEMBER_MESSAGE_FIELDS, + ) + _validate_turn_coordinates(payload, room) + text = payload.get("text") + if not isinstance(text, str) or not text.strip() or is_pass_text(text): + raise DiscussionValidationError("message.member text must be a non-pass string") + member = _member_by_id(room, payload.get("member_id")) + if ( + actor.get("kind") != "member" + or actor.get("id") != member.member_id + or actor.get("profile") != member.profile + or actor.get("connection_id") is not None + ): + raise DiscussionValidationError("message.member actor does not match roster") + + +def _validate_terminal_event( + kind: str, + payload: Mapping[str, Any], + *, + actor: Mapping[str, Any], + room: DiscussionRoom, +) -> None: + required = _TERMINAL_COMMON_FIELDS | _TERMINAL_EXTRA_FIELDS[kind] + _exact_fields(payload, label=f"{kind} payload", required=required) + _validate_turn_coordinates(payload, room) + _positive_int(payload.get("seen_through_seq"), label="seen_through_seq") + if ( + actor.get("kind") != "gateway" + or actor.get("id") != room.gateway_id + or actor.get("connection_id") is not None + ): + raise DiscussionValidationError(f"{kind} requires a gateway actor") + if kind == "turn.settled": + if not isinstance(payload.get("passed"), bool): + raise DiscussionValidationError("turn.settled passed must be a boolean") + message_event_id = payload.get("message_event_id") + if payload["passed"]: + if message_event_id is not None: + raise DiscussionValidationError( + "a passed turn cannot reference a member message" + ) + else: + _identifier(message_event_id, label="message_event_id") + else: + field = "error" if kind == "turn.failed" else "reason" + if not isinstance(payload.get(field), str) or not payload[field].strip(): + raise DiscussionValidationError(f"{kind} {field} must be non-empty") + if kind == "turn.deferred": + _positive_int( + payload.get("execution_generation"), + label="execution_generation", + ) + + +def _validated_events( + events: Sequence[Mapping[str, Any]], + *, + room: DiscussionRoom, +) -> tuple[_ValidatedEvent, ...]: + validated: list[_ValidatedEvent] = [] + previous_seq = 0 + event_ids: set[str] = set() + for raw in events: + event = _validate_event(raw, room=room, previous_seq=previous_seq) + if event.event_id in event_ids: + raise DiscussionValidationError("room event ids must be unique") + validated.append(event) + previous_seq = event.seq + event_ids.add(event.event_id) + return tuple(validated) + + +def _discussion_user_events( + events: Sequence[_ValidatedEvent], +) -> tuple[_ValidatedEvent, ...]: + return tuple(event for event in events if event.kind == "message.user") + + +def _message_events( + events: Sequence[_ValidatedEvent], + *, + thread_id: str | None = None, + maximum_seq: int | None = None, +) -> tuple[_ValidatedEvent, ...]: + result = [] + for event in events: + if event.kind not in {"message.user", "message.member"}: + continue + if thread_id is not None and event.payload.get("thread_id") != thread_id: + continue + if maximum_seq is not None and event.seq > maximum_seq: + continue + result.append(event) + return tuple(result) + + +def derive_member_watermarks( + room_value: Any, + events: Sequence[Mapping[str, Any]], + *, + local_profiles: Iterable[str], +) -> dict[tuple[str, str], int]: + """Derive ``(thread_id, member_id)`` watermarks from terminal events.""" + + room = validate_room(room_value, local_profiles=local_profiles) + validated = _validated_events(events, room=room) + return _derive_member_watermarks(validated) + + +def _derive_member_watermarks( + events: Sequence[_ValidatedEvent], +) -> dict[tuple[str, str], int]: + messages_by_id = { + event.event_id: event for event in events if event.kind == "message.member" + } + terminal_by_task: dict[str, _ValidatedEvent] = {} + watermarks: dict[tuple[str, str], int] = {} + for event in events: + if event.kind not in _TERMINAL_EVENT_KINDS: + continue + task_id = str(event.payload["task_id"]) + previous = terminal_by_task.get(task_id) + if previous is not None: + if previous.kind != "turn.deferred": + raise DiscussionValidationError( + f"task '{task_id}' has more than one terminal room event" + ) + if event.kind == "turn.deferred" and int( + event.payload["execution_generation"] + ) <= int(previous.payload["execution_generation"]): + raise DiscussionValidationError( + f"task '{task_id}' deferral generation did not advance" + ) + terminal_by_task[task_id] = event + key = (str(event.payload["thread_id"]), str(event.payload["member_id"])) + watermark = int(event.payload["seen_through_seq"]) + if event.kind == "turn.settled" and not event.payload["passed"]: + message_id = str(event.payload["message_event_id"]) + message = messages_by_id.get(message_id) + if ( + message is None + or message.payload.get("task_id") != task_id + or message.payload.get("member_id") != event.payload.get("member_id") + or message.payload.get("thread_id") != event.payload.get("thread_id") + ): + raise DiscussionValidationError( + "turn.settled references no matching member message" + ) + watermark = max(watermark, message.seq) + watermarks[key] = max(watermarks.get(key, 0), watermark) + return watermarks + + +def _member_digest(member: DiscussionMember) -> str: + seed = f"{member.member_id}\0{member.profile}\0{member.handle}" + return hashlib.sha256(seed.encode("utf-8")).hexdigest()[:24] + + +def _rotate( + members: Sequence[DiscussionMember], round_index: int +) -> tuple[DiscussionMember, ...]: + if len(members) < 2: + return tuple(members) + shift = round_index % len(members) + return tuple((*members[shift:], *members[:shift])) + + +def _format_message(event: _ValidatedEvent, room: DiscussionRoom) -> str: + text = str(event.payload["text"]) + if event.kind == "message.user": + return f"User (user): {text}" + member = _member_by_id(room, event.payload["member_id"]) + return f"@{member.handle}: {text}" + + +def _truncate_utf8_text(value: Any, *, max_bytes: int, suffix: str = "") -> str: + text = str(value or "") + encoded = text.encode("utf-8") + if len(encoded) <= max_bytes: + return text + suffix_bytes = suffix.encode("utf-8") + prefix = encoded[: max(0, max_bytes - len(suffix_bytes))] + while prefix: + try: + return prefix.decode("utf-8") + suffix + except UnicodeDecodeError: + prefix = prefix[:-1] + return suffix.strip() + + +def _build_prompt( + *, + room: DiscussionRoom, + member: DiscussionMember, + messages: Sequence[_ValidatedEvent], + watermark: int, + seen_through_seq: int, +) -> str: + delta = [event for event in messages if watermark < event.seq <= seen_through_seq][ + -MAX_DISCUSSION_DELTA_LINES: + ] + peers = ", ".join( + f"@{candidate.handle}" + for candidate in room.members + if candidate.member_id != member.member_id + ) + opening = [ + f'[Discussion: "{room.name}"] You are @{member.handle}, one participant ' + f"with {peers or 'no other members'} and the user.", + "", + "New messages in this thread since your last turn (oldest first):", + ] + rules = [ + "", + "Rules for this Discussion:", + "- Reply with one conversational message only when you have something new worth adding.", + '- If you have nothing new to add, reply with exactly "(pass)".', + "- Mention a teammate by handle to pull them into the next round; do not repeat points already made.", + "- Never reveal content from private conversations. Your reply is published verbatim.", + ] + fixed_bytes = len("\n".join([*opening, *rules]).encode("utf-8")) + available = max(0, driver.MAX_PROMPT_BYTES - fixed_bytes - 1) + selected: list[str] = [] + omitted = False + for event in reversed(delta): + line = f" {_format_message(event, room)}" + line_bytes = len(line.encode("utf-8")) + 1 + if line_bytes <= available: + selected.append(line) + available -= line_bytes + continue + if not selected and available > 32: + selected.append(_truncate_utf8_text(line, max_bytes=available)) + omitted = True + break + selected.reverse() + if omitted: + selected.insert(0, " [Earlier content omitted to fit this turn.]") + prompt = "\n".join([*opening, *selected, *rules]) + if len(prompt.encode("utf-8")) > driver.MAX_PROMPT_BYTES: + raise DiscussionValidationError("Discussion prompt exceeds the driver limit") + return prompt + + +def _turn_id( + *, + source_event_seq: int, + round_index: int, + member_index: int, + seen_through_seq: int, + member: DiscussionMember, +) -> str: + return ( + f"d{source_event_seq}.r{round_index}.p{member_index}." + f"s{seen_through_seq}.m{_member_digest(member)}" + ) + + +def _task_id( + *, + room: DiscussionRoom, + discussion_event: _ValidatedEvent, + member: DiscussionMember, + member_index: int, + round_index: int, + seen_through_seq: int, + prompt: str, +) -> str: + seed = json.dumps( + { + "discussion_event_id": discussion_event.event_id, + "member_id": member.member_id, + "member_index": member_index, + "prompt_sha256": hashlib.sha256(prompt.encode("utf-8")).hexdigest(), + "room_id": room.room_id, + "round_index": round_index, + "seen_through_seq": seen_through_seq, + "source_event_seq": discussion_event.seq, + "thread_id": discussion_event.payload["thread_id"], + }, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + return f"dtask:{hashlib.sha256(seed.encode('utf-8')).hexdigest()[:48]}" + + +def _make_task_plan( + *, + room: DiscussionRoom, + discussion_event: _ValidatedEvent, + member: DiscussionMember, + member_index: int, + round_index: int, + seen_through_seq: int, + prompt: str, +) -> DiscussionTaskPlan: + turn_id = _turn_id( + source_event_seq=discussion_event.seq, + round_index=round_index, + member_index=member_index, + seen_through_seq=seen_through_seq, + member=member, + ) + task_id = _task_id( + room=room, + discussion_event=discussion_event, + member=member, + member_index=member_index, + round_index=round_index, + seen_through_seq=seen_through_seq, + prompt=prompt, + ) + identity = driver.TaskIdentity( + room_id=room.room_id, + task_id=task_id, + thread_id=str(discussion_event.payload["thread_id"]), + turn_id=turn_id, + ) + payload = { + "target_profile": member.profile, + "prompt": prompt, + "source_event_seq": discussion_event.seq, + } + return DiscussionTaskPlan( + identity=identity, + payload=payload, + discussion_event_id=discussion_event.event_id, + member=member, + member_index=member_index, + round_index=round_index, + seen_through_seq=seen_through_seq, + ) + + +def plan_next_task( + room_value: Any, + events: Sequence[Mapping[str, Any]], + *, + local_profiles: Iterable[str], + initial_watermarks: Mapping[tuple[str, str], int] | None = None, +) -> DiscussionDecision: + """Replay the complete room log and return at most one next member task.""" + + room = validate_room(room_value, local_profiles=local_profiles) + validated = _validated_events(events, room=room) + user_events = _discussion_user_events(validated) + stopped_through_seq = max( + (event.seq for event in validated if event.kind == "room.stop_requested"), + default=0, + ) + completed_discussion_ids = { + str(event.payload["discussion_event_id"]) + for event in validated + if event.kind == "room.activity" + and event.payload.get("status") in {"settled", "bounded"} + } + latest_by_thread: dict[str, _ValidatedEvent] = {} + for event in user_events: + latest_by_thread[str(event.payload["thread_id"])] = event + pending_user_events = tuple( + event + for event in sorted(latest_by_thread.values(), key=lambda item: item.seq) + if event.seq > stopped_through_seq + and event.event_id not in completed_discussion_ids + ) + if not pending_user_events: + return DiscussionDecision(status="idle", reason="no_pending_user_event") + + discussion = pending_user_events[0] + thread_id = str(discussion.payload["thread_id"]) + committed_member_message_ids = { + str(event.payload["message_event_id"]) + for event in validated + if event.kind == "turn.settled" + and event.payload.get("message_event_id") is not None + } + # Publication writes the visible member message before the terminal event. + # A crash in that gap leaves the message in the log, but it is not committed + # policy input yet: ignoring it reproduces the original task coordinates so + # the caller can inspect the terminal driver row and finish publication. + thread_messages = tuple( + event + for event in _message_events(validated, thread_id=thread_id) + if event.kind == "message.user" + or event.event_id in committed_member_message_ids + ) + discussion_messages = tuple( + event for event in thread_messages if event.seq >= discussion.seq + ) + member_messages = tuple( + event + for event in thread_messages + if event.kind == "message.member" + and event.payload.get("discussion_event_id") == discussion.event_id + ) + if len(member_messages) >= MAX_DISCUSSION_MESSAGES: + return DiscussionDecision( + status="bounded", + reason="max_messages", + discussion_event_id=discussion.event_id, + source_event_seq=discussion.seq, + thread_id=thread_id, + ) + + terminals = { + (int(event.payload["round_index"]), str(event.payload["member_id"])): event + for event in validated + if event.kind in _TERMINAL_EVENT_KINDS + and event.payload.get("discussion_event_id") == discussion.event_id + } + watermarks = { + (str(thread_id), str(member_id)): int(value) + for (thread_id, member_id), value in (initial_watermarks or {}).items() + if int(value) >= 0 + } + for key, value in _derive_member_watermarks(validated).items(): + watermarks[key] = max(watermarks.get(key, 0), value) + seen_through_seq = max(event.seq for event in thread_messages) + + for round_index in range(MAX_DISCUSSION_ROUNDS): + # The user's message selects the first round, with no mention meaning + # everyone. Later rounds are opt-in: only a peer explicitly cited by a + # Bot and not heard from afterward gets another turn. Every member's + # watermark remains intact, so a peer cited later still receives the + # complete bounded transcript delta without consuming turns meanwhile. + responders = ( + resolve_mentions((str(discussion.payload["text"]),), room.members) + if round_index == 0 + else _unaddressed_member_mentions(discussion_messages, room) + ) + ordered = _rotate(responders, round_index) + for member_index, member in enumerate(ordered): + if (round_index, member.member_id) in terminals: + continue + watermark = watermarks.get((thread_id, member.member_id), 0) + delta = [ + event + for event in thread_messages + if watermark < event.seq <= seen_through_seq + ] + if not delta: + continue + prompt = _build_prompt( + room=room, + member=member, + messages=thread_messages, + watermark=watermark, + seen_through_seq=seen_through_seq, + ) + task = _make_task_plan( + room=room, + discussion_event=discussion, + member=member, + member_index=member_index, + round_index=round_index, + seen_through_seq=seen_through_seq, + prompt=prompt, + ) + return DiscussionDecision( + status="task", + reason="member_turn", + discussion_event_id=discussion.event_id, + source_event_seq=discussion.seq, + thread_id=thread_id, + task=task, + ) + + spoke = any( + int(event.payload["round_index"]) == round_index + for event in member_messages + ) + if not spoke: + return DiscussionDecision( + status="settled", + reason="silent_round", + discussion_event_id=discussion.event_id, + source_event_seq=discussion.seq, + thread_id=thread_id, + ) + if round_index == MAX_DISCUSSION_ROUNDS - 1: + return DiscussionDecision( + status="bounded", + reason="max_rounds", + discussion_event_id=discussion.event_id, + source_event_seq=discussion.seq, + thread_id=thread_id, + ) + + raise AssertionError("bounded Discussion loop exhausted unexpectedly") + + +def reconstruct_task_plan( + room_value: Any, + events: Sequence[Mapping[str, Any]], + task: Mapping[str, Any], + *, + local_profiles: Iterable[str], +) -> DiscussionTaskPlan: + """Reconstruct and verify one persisted driver task after a restart.""" + + room = validate_room(room_value, local_profiles=local_profiles) + validated = _validated_events(events, room=room) + identity = task.get("identity") + payload = task.get("payload") + if not isinstance(identity, driver.TaskIdentity) or not isinstance( + payload, Mapping + ): + raise DiscussionReconstructionError( + "driver task has no valid identity or payload" + ) + if frozenset(payload) != frozenset({ + "target_profile", + "prompt", + "source_event_seq", + }): + raise DiscussionReconstructionError("driver task payload shape changed") + match = _TURN_ID_RE.fullmatch(identity.turn_id) + if match is None: + raise DiscussionReconstructionError("turn_id is not a Discussion coordinate") + source_event_seq = int(match.group("source")) + round_index = int(match.group("round")) + member_index = int(match.group("position")) + seen_through_seq = int(match.group("seen")) + if payload.get("source_event_seq") != source_event_seq: + raise DiscussionReconstructionError("task source event does not match turn_id") + discussion = next( + ( + event + for event in validated + if event.seq == source_event_seq and event.kind == "message.user" + ), + None, + ) + if discussion is None: + raise DiscussionReconstructionError("task source user event is missing") + if ( + identity.room_id != room.room_id + or identity.thread_id != discussion.payload["thread_id"] + ): + raise DiscussionReconstructionError( + "task identity does not match its room thread" + ) + profile = payload.get("target_profile") + member = next( + (candidate for candidate in room.members if candidate.profile == profile), None + ) + if member is None or _member_digest(member) != match.group("member"): + raise DiscussionReconstructionError("task target member does not match turn_id") + prompt = payload.get("prompt") + if not isinstance(prompt, str) or not prompt.strip(): + raise DiscussionReconstructionError("task prompt is missing") + if len(prompt.encode("utf-8")) > driver.MAX_PROMPT_BYTES: + raise DiscussionReconstructionError("task prompt exceeds the driver limit") + reconstructed = _make_task_plan( + room=room, + discussion_event=discussion, + member=member, + member_index=member_index, + round_index=round_index, + seen_through_seq=seen_through_seq, + prompt=prompt, + ) + if reconstructed.identity != identity or dict(reconstructed.payload) != dict( + payload + ): + raise DiscussionReconstructionError( + "driver task failed deterministic reconstruction" + ) + return reconstructed + + +def _terminal_text(result: Any, *, field: str, fallback: str) -> str: + if isinstance(result, Mapping): + value = result.get(field) + if value is None and field == "error": + value = result.get("text") + else: + value = result + text = str(value or "").strip() + return text or fallback + + +def plan_publication( + room_value: Any, + events: Sequence[Mapping[str, Any]], + task: DiscussionTaskPlan, + *, + status: TerminalKind, + result: Any = None, + execution_generation: int | None = None, + local_profiles: Iterable[str], +) -> PublicationPlan: + """Plan idempotent room effects for one terminal driver task. + + A newer user event in the same thread supersedes a late result. The task + remains terminal in driver state, but only a deterministic cancellation is + published, preventing stale prose and its watermark from hiding the newer + user message. + """ + + room = validate_room(room_value, local_profiles=local_profiles) + validated = _validated_events(events, room=room) + if task.identity.room_id != room.room_id: + raise DiscussionValidationError("task belongs to a different room") + if task.member not in room.members: + raise DiscussionValidationError("task member is not in the frozen roster") + if status not in {"settled", "failed", "cancelled", "deferred"}: + raise DiscussionValidationError("invalid terminal publication status") + if status == "deferred" and ( + isinstance(execution_generation, bool) + or not isinstance(execution_generation, int) + or execution_generation < 1 + ): + raise DiscussionValidationError( + "deferred publication requires an execution generation" + ) + + newer_same_thread = any( + event.kind == "message.user" + and event.seq > task.seen_through_seq + and event.payload.get("thread_id") == task.identity.thread_id + for event in validated + ) + effective_status: TerminalKind = ( + "cancelled" if newer_same_thread and status != "deferred" else status + ) + digest = task.identity.task_id.removeprefix("dtask:") + message_event_id = f"dmessage:{digest}" + terminal_event_id = ( + f"ddeferred:{digest}:g{execution_generation}" + if effective_status == "deferred" + else f"dterminal:{digest}" + ) + common = { + "discussion_event_id": task.discussion_event_id, + "member_id": task.member.member_id, + "member_index": task.member_index, + "round_index": task.round_index, + "seen_through_seq": task.seen_through_seq, + "task_id": task.identity.task_id, + "thread_id": task.identity.thread_id, + "turn_id": task.identity.turn_id, + } + effects: list[EventPlan] = [] + + if effective_status == "settled": + text = _truncate_utf8_text( + _terminal_text(result, field="text", fallback=""), + max_bytes=MAX_MEMBER_TEXT_BYTES, + suffix=_TRUNCATED_REPLY_NOTICE, + ) + passed = is_pass_text(text) + if not passed: + member_actor = { + "kind": "member", + "id": task.member.member_id, + "profile": task.member.profile, + } + if task.member.display_name: + member_actor["display_name"] = task.member.display_name + effects.append( + EventPlan( + event_id=message_event_id, + kind="message.member", + actor=member_actor, + payload={ + "discussion_event_id": task.discussion_event_id, + "member_id": task.member.member_id, + "member_index": task.member_index, + "round_index": task.round_index, + "task_id": task.identity.task_id, + "text": text, + "thread_id": task.identity.thread_id, + "turn_id": task.identity.turn_id, + }, + authority_gateway_id=room.gateway_id, + authority_epoch=room.authority_epoch, + ) + ) + terminal_payload = { + **common, + "message_event_id": None if passed else message_event_id, + "passed": passed, + } + terminal_kind = "turn.settled" + elif effective_status == "failed": + terminal_payload = { + **common, + "error": _terminal_text( + result, + field="error", + fallback="member turn failed", + ), + } + terminal_kind = "turn.failed" + elif effective_status == "cancelled": + terminal_payload = { + **common, + "reason": ( + "superseded_by_newer_user_event" + if newer_same_thread + else _terminal_text( + result, + field="reason", + fallback="member turn cancelled", + ) + ), + } + terminal_kind = "turn.cancelled" + else: + terminal_payload = { + **common, + "execution_generation": execution_generation, + "reason": _terminal_text( + result, + field="reason", + fallback="member_unavailable", + ), + } + terminal_kind = "turn.deferred" + + effects.append( + EventPlan( + event_id=terminal_event_id, + kind=terminal_kind, + actor={"kind": "gateway", "id": room.gateway_id}, + payload=terminal_payload, + authority_gateway_id=room.gateway_id, + authority_epoch=room.authority_epoch, + ) + ) + return PublicationPlan( + task_id=task.identity.task_id, + terminal_kind=terminal_kind, + events=tuple(effects), + ) diff --git a/gateway/hosted_room_driver.py b/gateway/hosted_room_driver.py new file mode 100644 index 0000000000..55cbed15fe --- /dev/null +++ b/gateway/hosted_room_driver.py @@ -0,0 +1,1649 @@ +"""Durable execution state for a same-gateway hosted room driver. + +This module owns only the driver lease and task state machine. It does not +invoke models, touch sessions, or depend on the hosted-room event log. Callers +provide both the database path and clock so recovery and fencing behavior can +be tested without process-global state. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +import sqlite3 +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Iterator, Literal + + +Clock = Callable[[], float] +TaskStatus = Literal[ + "queued", + "running", + "settled", + "failed", + "cancelled", + "indeterminate", + "deferred", + "stopping", +] +TerminalStatus = Literal["settled", "failed"] + +MAX_IDENTIFIER_CHARS = 128 +MAX_PROMPT_BYTES = 128 * 1024 +MAX_RESULT_JSON_BYTES = 256 * 1024 +TERMINAL_TASK_RETENTION_SECONDS = 30 * 24 * 60 * 60 +MAX_RETAINED_TERMINAL_TASKS = 2048 +MAX_TASK_PRUNE_BATCH = 1000 +TASK_STATUSES = frozenset({ + "queued", + "running", + "settled", + "failed", + "cancelled", + "indeterminate", + "deferred", + "stopping", +}) +TERMINAL_STATUSES = frozenset({"settled", "failed", "cancelled"}) + +_IDENTIFIER_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]*$") +_TASK_PAYLOAD_FIELDS = frozenset({"target_profile", "prompt", "source_event_seq"}) +_LEASE_COLUMNS = frozenset({ + "room_id", + "gateway_id", + "authority_epoch", + "process_generation", + "lease_generation", + "expires_at", + "acquired_at", + "updated_at", + "released_at", +}) +_TASK_COLUMNS = frozenset({ + "room_id", + "task_id", + "thread_id", + "turn_id", + "source_event_seq", + "payload_json", + "payload_digest", + "status", + "execution_generation", + "cancel_generation", + "run_gateway_id", + "run_process_generation", + "run_lease_generation", + "cancel_id", + "settlement_id", + "settlement_status", + "result_json", + "created_at", + "updated_at", + "started_at", + "terminal_at", + "indeterminate_at", +}) +_TASK_COLUMN_ORDER = ( + "room_id", + "task_id", + "thread_id", + "turn_id", + "source_event_seq", + "payload_json", + "payload_digest", + "status", + "execution_generation", + "cancel_generation", + "run_gateway_id", + "run_process_generation", + "run_lease_generation", + "cancel_id", + "settlement_id", + "settlement_status", + "result_json", + "created_at", + "updated_at", + "started_at", + "terminal_at", + "indeterminate_at", +) + + +class DriverStateError(ValueError): + """Base class for invalid or conflicting driver-state operations.""" + + +class DriverValidationError(DriverStateError): + """Raised when an identifier, clock, TTL, or payload is invalid.""" + + +class RoomUnavailableError(DriverStateError): + """Raised when the hosted room does not exist or was disbanded.""" + + +class LeaseHeldError(DriverStateError): + """Raised when another unexpired driver generation owns the room.""" + + +class StaleLeaseError(DriverStateError): + """Raised when a lease generation can no longer mutate room state.""" + + +class TaskConflictError(DriverStateError): + """Raised when an idempotency key is reused for different task state.""" + + +class StaleTaskError(DriverStateError): + """Raised when an obsolete task attempt or cancellation tries to commit.""" + + +class InvalidTaskTransitionError(DriverStateError): + """Raised when a requested task transition is not allowed.""" + + +def _identifier(value: Any, *, label: str) -> str: + if not isinstance(value, str): + raise DriverValidationError(f"{label} must be a string") + value = value.strip() + if ( + not value + or len(value) > MAX_IDENTIFIER_CHARS + or not _IDENTIFIER_RE.fullmatch(value) + ): + raise DriverValidationError(f"invalid {label}") + return value + + +def _timestamp(clock: Clock) -> float: + if not callable(clock): + raise DriverValidationError("clock must be callable") + try: + value = float(clock()) + except (TypeError, ValueError, OverflowError) as exc: + raise DriverValidationError("clock must return a finite number") from exc + if not math.isfinite(value): + raise DriverValidationError("clock must return a finite number") + return value + + +def _ttl(value: Any) -> float: + try: + ttl = float(value) + except (TypeError, ValueError, OverflowError) as exc: + raise DriverValidationError( + "ttl_seconds must be a finite positive number" + ) from exc + if not math.isfinite(ttl) or ttl <= 0: + raise DriverValidationError("ttl_seconds must be a finite positive number") + return ttl + + +def _expiry(now: float, ttl: float) -> float: + expires_at = now + ttl + if not math.isfinite(expires_at): + raise DriverValidationError("lease expiry must be finite") + return expires_at + + +def _canonical_json(value: Any) -> str: + try: + encoded = json.dumps( + value, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + except (TypeError, ValueError, RecursionError) as exc: + raise DriverValidationError("result must be JSON-serializable") from exc + if len(encoded.encode("utf-8")) > MAX_RESULT_JSON_BYTES: + raise DriverValidationError("result is too large") + return encoded + + +def _authority_epoch(value: Any) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise DriverValidationError("authority_epoch must be a positive integer") + return value + + +def _task_payload(value: Any) -> tuple[dict[str, Any], str, str]: + if not isinstance(value, dict): + raise DriverValidationError("payload must be an object") + unknown = set(value) - _TASK_PAYLOAD_FIELDS + missing = _TASK_PAYLOAD_FIELDS - set(value) + if unknown: + raise DriverValidationError( + f"unknown payload fields: {', '.join(sorted(unknown))}" + ) + if missing: + raise DriverValidationError( + f"missing payload fields: {', '.join(sorted(missing))}" + ) + + target_profile = _identifier(value["target_profile"], label="target_profile") + prompt = value["prompt"] + if not isinstance(prompt, str): + raise DriverValidationError("prompt must be a string") + if not prompt.strip(): + raise DriverValidationError("prompt must not be empty") + if len(prompt.encode("utf-8")) > MAX_PROMPT_BYTES: + raise DriverValidationError("prompt is too large") + source_event_seq = value["source_event_seq"] + if ( + isinstance(source_event_seq, bool) + or not isinstance(source_event_seq, int) + or source_event_seq < 1 + ): + raise DriverValidationError("source_event_seq must be a positive integer") + + normalized = { + "target_profile": target_profile, + "prompt": prompt, + "source_event_seq": source_event_seq, + } + encoded = json.dumps( + normalized, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + digest = hashlib.sha256(encoded.encode("utf-8")).hexdigest() + return normalized, encoded, digest + + +@dataclass(frozen=True) +class TaskIdentity: + """Stable identity for one admitted room turn.""" + + room_id: str + task_id: str + thread_id: str + turn_id: str + + def __post_init__(self) -> None: + for field in ("room_id", "task_id", "thread_id", "turn_id"): + object.__setattr__( + self, + field, + _identifier(getattr(self, field), label=field), + ) + + +@dataclass(frozen=True) +class DriverLease: + """A fenced lease held by one gateway process incarnation.""" + + room_id: str + gateway_id: str + authority_epoch: int + process_generation: str + lease_generation: int + expires_at: float + reclaimed: bool = False + + +@dataclass(frozen=True) +class TaskAttempt: + """The exact running generation authorized to settle one task.""" + + identity: TaskIdentity + lease: DriverLease + execution_generation: int + cancel_generation: int + + +def _create_task_table( + conn: sqlite3.Connection, table: str = "hosted_room_driver_tasks" +) -> None: + if table not in {"hosted_room_driver_tasks", "hosted_room_driver_tasks_next"}: + raise DriverStateError("invalid hosted-room task table name") + conn.execute( + f"""CREATE TABLE IF NOT EXISTS {table} ( + room_id TEXT NOT NULL, + task_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + source_event_seq INTEGER NOT NULL CHECK (source_event_seq >= 1), + payload_json TEXT NOT NULL, + payload_digest TEXT NOT NULL, + status TEXT NOT NULL CHECK ( + status IN ( + 'queued', 'running', 'settled', 'failed', + 'cancelled', 'indeterminate', 'deferred', 'stopping' + ) + ), + execution_generation INTEGER NOT NULL DEFAULT 0 + CHECK (execution_generation >= 0), + cancel_generation INTEGER NOT NULL DEFAULT 0 + CHECK (cancel_generation >= 0), + run_gateway_id TEXT, + run_process_generation TEXT, + run_lease_generation INTEGER, + cancel_id TEXT, + settlement_id TEXT, + settlement_status TEXT, + result_json TEXT, + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + started_at REAL, + terminal_at REAL, + indeterminate_at REAL, + PRIMARY KEY (room_id, task_id), + UNIQUE (room_id, thread_id, turn_id), + FOREIGN KEY (room_id) REFERENCES hosted_rooms(room_id) + )""" + ) + + +def _initialize_schema(conn: sqlite3.Connection) -> None: + conn.execute( + """CREATE TABLE IF NOT EXISTS hosted_room_driver_leases ( + room_id TEXT PRIMARY KEY, + gateway_id TEXT NOT NULL, + authority_epoch INTEGER NOT NULL CHECK (authority_epoch >= 1), + process_generation TEXT NOT NULL, + lease_generation INTEGER NOT NULL CHECK (lease_generation >= 1), + expires_at REAL NOT NULL, + acquired_at REAL NOT NULL, + updated_at REAL NOT NULL, + released_at REAL, + FOREIGN KEY (room_id) REFERENCES hosted_rooms(room_id) + )""" + ) + _create_task_table(conn) + _validate_schema(conn) + conn.execute( + """CREATE INDEX IF NOT EXISTS idx_hosted_room_driver_tasks_status + ON hosted_room_driver_tasks( + room_id, status, source_event_seq, created_at, task_id + )""" + ) + + +def _validate_schema(conn: sqlite3.Connection) -> None: + lease_columns = frozenset( + row[1] for row in conn.execute("PRAGMA table_info(hosted_room_driver_leases)") + ) + task_columns = frozenset( + row[1] for row in conn.execute("PRAGMA table_info(hosted_room_driver_tasks)") + ) + if lease_columns != _LEASE_COLUMNS or task_columns != _TASK_COLUMNS: + raise DriverStateError( + "unsupported unpublished hosted-room driver schema; " + "recreate the driver tables before starting the driver" + ) + + for table in ("hosted_room_driver_leases", "hosted_room_driver_tasks"): + foreign_keys = conn.execute(f"PRAGMA foreign_key_list({table})").fetchall() + if not any( + row[2] == "hosted_rooms" and row[3] == "room_id" and row[4] == "room_id" + for row in foreign_keys + ): + raise DriverStateError(f"{table} is missing its hosted_rooms foreign key") + + +def _schema_objects_exist(conn: sqlite3.Connection) -> bool: + rows = conn.execute( + """SELECT name FROM sqlite_master + WHERE type='table' AND name IN ( + 'hosted_room_driver_leases', 'hosted_room_driver_tasks' + )""" + ).fetchall() + tables = {row[0] for row in rows} + if tables != {"hosted_room_driver_leases", "hosted_room_driver_tasks"}: + return False + index = conn.execute( + """SELECT 1 FROM sqlite_master + WHERE type='index' AND name='idx_hosted_room_driver_tasks_status'""" + ).fetchone() + return index is not None + + +def _task_schema_supports_current_statuses(conn: sqlite3.Connection) -> bool: + row = conn.execute( + """SELECT sql FROM sqlite_master + WHERE type='table' AND name='hosted_room_driver_tasks'""" + ).fetchone() + sql = str(row[0] or "").lower() if row else "" + return "'stopping'" in sql and "'deferred'" in sql + + +def _migrate_task_status_constraint(conn: sqlite3.Connection) -> None: + """Expand the unpublished task-state CHECK without losing durable work.""" + conn.execute("DROP INDEX IF EXISTS idx_hosted_room_driver_tasks_status") + _create_task_table(conn, "hosted_room_driver_tasks_next") + columns = ", ".join(_TASK_COLUMN_ORDER) + conn.execute( + f"""INSERT INTO hosted_room_driver_tasks_next ({columns}) + SELECT {columns} FROM hosted_room_driver_tasks""" + ) + conn.execute("DROP TABLE hosted_room_driver_tasks") + conn.execute( + "ALTER TABLE hosted_room_driver_tasks_next RENAME TO hosted_room_driver_tasks" + ) + conn.execute( + """CREATE INDEX idx_hosted_room_driver_tasks_status + ON hosted_room_driver_tasks( + room_id, status, source_event_seq, created_at, task_id + )""" + ) + + +def _connect(db_path: Path | str) -> sqlite3.Connection: + from hermes_state import apply_wal_with_fallback + + path = Path(db_path) + path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(path, timeout=10) + conn.row_factory = sqlite3.Row + try: + apply_wal_with_fallback(conn, db_label="state.db (hosted_room_driver)") + conn.execute("PRAGMA foreign_keys=ON") + if _schema_objects_exist(conn): + if not _task_schema_supports_current_statuses(conn): + conn.execute("BEGIN IMMEDIATE") + _migrate_task_status_constraint(conn) + conn.commit() + _validate_schema(conn) + return conn + # Schema creation is one database-wide transaction. The driver schema + # has never shipped, so an incompatible draft schema fails closed + # instead of attempting a partial in-place migration. + conn.execute("BEGIN IMMEDIATE") + _initialize_schema(conn) + conn.commit() + except Exception: + conn.rollback() + conn.close() + raise + return conn + + +@contextmanager +def _transaction(db_path: Path | str) -> Iterator[sqlite3.Connection]: + conn = _connect(db_path) + try: + conn.execute("BEGIN IMMEDIATE") + yield conn + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + +def _lease_from_row( + row: sqlite3.Row | dict[str, Any], *, reclaimed: bool = False +) -> DriverLease: + return DriverLease( + room_id=row["room_id"], + gateway_id=row["gateway_id"], + authority_epoch=int(row["authority_epoch"]), + process_generation=row["process_generation"], + lease_generation=int(row["lease_generation"]), + expires_at=float(row["expires_at"]), + reclaimed=reclaimed, + ) + + +def _task_identity_from_row(row: sqlite3.Row) -> TaskIdentity: + return TaskIdentity( + room_id=row["room_id"], + task_id=row["task_id"], + thread_id=row["thread_id"], + turn_id=row["turn_id"], + ) + + +def _task_from_row(row: sqlite3.Row, *, idempotent: bool = False) -> dict[str, Any]: + try: + raw_payload = json.loads(row["payload_json"]) + payload, encoded_payload, payload_digest = _task_payload(raw_payload) + except (TypeError, json.JSONDecodeError, DriverValidationError) as exc: + raise TaskConflictError("stored task payload is invalid") from exc + if ( + encoded_payload != row["payload_json"] + or payload_digest != row["payload_digest"] + or payload["source_event_seq"] != int(row["source_event_seq"]) + ): + raise TaskConflictError("stored task payload failed its integrity check") + result = json.loads(row["result_json"]) if row["result_json"] is not None else None + return { + "identity": _task_identity_from_row(row), + "payload": payload, + "payload_digest": row["payload_digest"], + "status": row["status"], + "execution_generation": int(row["execution_generation"]), + "cancel_generation": int(row["cancel_generation"]), + "run_gateway_id": row["run_gateway_id"], + "run_process_generation": row["run_process_generation"], + "run_lease_generation": ( + int(row["run_lease_generation"]) + if row["run_lease_generation"] is not None + else None + ), + "cancel_id": row["cancel_id"], + "settlement_id": row["settlement_id"], + "settlement_status": row["settlement_status"], + "result": result, + "created_at": float(row["created_at"]), + "updated_at": float(row["updated_at"]), + "started_at": ( + float(row["started_at"]) if row["started_at"] is not None else None + ), + "terminal_at": ( + float(row["terminal_at"]) if row["terminal_at"] is not None else None + ), + "indeterminate_at": ( + float(row["indeterminate_at"]) + if row["indeterminate_at"] is not None + else None + ), + "idempotent": idempotent, + } + + +def _load_task(conn: sqlite3.Connection, identity: TaskIdentity) -> sqlite3.Row: + row = conn.execute( + """SELECT * FROM hosted_room_driver_tasks + WHERE room_id=? AND task_id=?""", + (identity.room_id, identity.task_id), + ).fetchone() + if row is None: + raise TaskConflictError("task does not exist") + if _task_identity_from_row(row) != identity: + raise TaskConflictError("task_id is already bound to a different turn") + return row + + +def _load_active_room(conn: sqlite3.Connection, room_id: str) -> sqlite3.Row: + try: + row = conn.execute( + """SELECT room_id, authority_gateway_id, authority_epoch, disbanded_at + FROM hosted_rooms WHERE room_id=?""", + (room_id,), + ).fetchone() + except sqlite3.OperationalError as exc: + if "no such table" in str(exc).lower(): + raise RoomUnavailableError("hosted room does not exist") from exc + raise + if row is None: + raise RoomUnavailableError("hosted room does not exist") + if row["disbanded_at"] is not None: + raise RoomUnavailableError("hosted room is disbanded") + return row + + +def _require_room_authority( + conn: sqlite3.Connection, + *, + room_id: str, + gateway_id: str, + authority_epoch: int, +) -> sqlite3.Row: + room = _load_active_room(conn, room_id) + if ( + room["authority_gateway_id"] != gateway_id + or int(room["authority_epoch"]) != authority_epoch + ): + raise StaleLeaseError("hosted room authority changed") + return room + + +def _require_active_lease( + conn: sqlite3.Connection, + lease: DriverLease, + *, + now: float, +) -> sqlite3.Row: + _require_room_authority( + conn, + room_id=lease.room_id, + gateway_id=lease.gateway_id, + authority_epoch=lease.authority_epoch, + ) + row = conn.execute( + "SELECT * FROM hosted_room_driver_leases WHERE room_id=?", + (lease.room_id,), + ).fetchone() + if ( + row is None + or row["gateway_id"] != lease.gateway_id + or int(row["authority_epoch"]) != lease.authority_epoch + or row["process_generation"] != lease.process_generation + or int(row["lease_generation"]) != lease.lease_generation + or row["released_at"] is not None + or float(row["expires_at"]) <= now + ): + raise StaleLeaseError("driver lease is stale or expired") + return row + + +def acquire_lease( + db_path: Path | str, + *, + room_id: Any, + gateway_id: Any, + authority_epoch: Any, + process_generation: Any, + ttl_seconds: Any, + clock: Clock, +) -> DriverLease: + """Acquire an empty or expired room lease with a monotonic generation.""" + room_id = _identifier(room_id, label="room_id") + gateway_id = _identifier(gateway_id, label="gateway_id") + authority_epoch = _authority_epoch(authority_epoch) + process_generation = _identifier( + process_generation, + label="process_generation", + ) + ttl_seconds = _ttl(ttl_seconds) + now = _timestamp(clock) + expires_at = _expiry(now, ttl_seconds) + + with _transaction(db_path) as conn: + _require_room_authority( + conn, + room_id=room_id, + gateway_id=gateway_id, + authority_epoch=authority_epoch, + ) + row = conn.execute( + "SELECT * FROM hosted_room_driver_leases WHERE room_id=?", + (room_id,), + ).fetchone() + if row is None: + conn.execute( + """INSERT INTO hosted_room_driver_leases ( + room_id, gateway_id, authority_epoch, process_generation, + lease_generation, expires_at, acquired_at, + updated_at, released_at + ) VALUES (?, ?, ?, ?, 1, ?, ?, ?, NULL)""", + ( + room_id, + gateway_id, + authority_epoch, + process_generation, + expires_at, + now, + now, + ), + ) + row = conn.execute( + "SELECT * FROM hosted_room_driver_leases WHERE room_id=?", + (room_id,), + ).fetchone() + return _lease_from_row(row) + + if ( + row["gateway_id"] == gateway_id + and int(row["authority_epoch"]) == authority_epoch + and row["process_generation"] == process_generation + and row["released_at"] is None + and float(row["expires_at"]) > now + ): + renewed_expiry = max(float(row["expires_at"]), expires_at) + conn.execute( + """UPDATE hosted_room_driver_leases + SET expires_at=?, updated_at=? + WHERE room_id=? AND lease_generation=?""", + (renewed_expiry, now, room_id, int(row["lease_generation"])), + ) + current = dict(row) + current["expires_at"] = renewed_expiry + return _lease_from_row(current) + + same_authority = ( + row["gateway_id"] == gateway_id + and int(row["authority_epoch"]) == authority_epoch + ) + if ( + same_authority + and row["released_at"] is None + and float(row["expires_at"]) > now + ): + raise LeaseHeldError("room driver lease is held by another generation") + + previous_generation = int(row["lease_generation"]) + updated = conn.execute( + """UPDATE hosted_room_driver_leases + SET gateway_id=?, authority_epoch=?, process_generation=?, + lease_generation=lease_generation + 1, + expires_at=?, acquired_at=?, updated_at=?, released_at=NULL + WHERE room_id=? AND lease_generation=? + AND ( + gateway_id != ? OR authority_epoch != ? + OR released_at IS NOT NULL OR expires_at <= ? + )""", + ( + gateway_id, + authority_epoch, + process_generation, + expires_at, + now, + now, + room_id, + previous_generation, + gateway_id, + authority_epoch, + now, + ), + ) + if updated.rowcount != 1: + raise LeaseHeldError("room driver lease changed during acquisition") + current = conn.execute( + "SELECT * FROM hosted_room_driver_leases WHERE room_id=?", + (room_id,), + ).fetchone() + return _lease_from_row(current, reclaimed=True) + + +def renew_lease( + db_path: Path | str, + lease: DriverLease, + *, + ttl_seconds: Any, + clock: Clock, +) -> DriverLease: + """Renew the exact active lease generation or fail closed.""" + ttl_seconds = _ttl(ttl_seconds) + now = _timestamp(clock) + requested_expiry = _expiry(now, ttl_seconds) + with _transaction(db_path) as conn: + current = _require_active_lease(conn, lease, now=now) + expires_at = max(float(current["expires_at"]), requested_expiry) + updated = conn.execute( + """UPDATE hosted_room_driver_leases + SET expires_at=?, updated_at=? + WHERE room_id=? AND gateway_id=? AND process_generation=? + AND lease_generation=? AND released_at IS NULL AND expires_at > ?""", + ( + expires_at, + now, + lease.room_id, + lease.gateway_id, + lease.process_generation, + lease.lease_generation, + now, + ), + ) + if updated.rowcount != 1: + raise StaleLeaseError("driver lease changed during renewal") + return DriverLease( + room_id=lease.room_id, + gateway_id=lease.gateway_id, + authority_epoch=lease.authority_epoch, + process_generation=lease.process_generation, + lease_generation=lease.lease_generation, + expires_at=expires_at, + ) + + +def release_lease( + db_path: Path | str, + lease: DriverLease, + *, + clock: Clock, +) -> dict[str, Any]: + """Release the exact active lease generation idempotently.""" + now = _timestamp(clock) + with _transaction(db_path) as conn: + _require_room_authority( + conn, + room_id=lease.room_id, + gateway_id=lease.gateway_id, + authority_epoch=lease.authority_epoch, + ) + row = conn.execute( + "SELECT * FROM hosted_room_driver_leases WHERE room_id=?", + (lease.room_id,), + ).fetchone() + if ( + row is None + or row["gateway_id"] != lease.gateway_id + or int(row["authority_epoch"]) != lease.authority_epoch + or row["process_generation"] != lease.process_generation + or int(row["lease_generation"]) != lease.lease_generation + ): + raise StaleLeaseError("driver lease is stale") + if row["released_at"] is not None: + return {"lease": _lease_from_row(row), "idempotent": True} + if float(row["expires_at"]) <= now: + raise StaleLeaseError("driver lease expired before release") + running = conn.execute( + """SELECT 1 FROM hosted_room_driver_tasks + WHERE room_id=? AND status='running' LIMIT 1""", + (lease.room_id,), + ).fetchone() + if running is not None: + raise InvalidTaskTransitionError( + "cannot release a room lease while tasks are running" + ) + conn.execute( + """UPDATE hosted_room_driver_leases + SET expires_at=?, updated_at=?, released_at=? + WHERE room_id=? AND lease_generation=?""", + (now, now, now, lease.room_id, lease.lease_generation), + ) + current = dict(row) + current["expires_at"] = now + current["updated_at"] = now + current["released_at"] = now + return {"lease": _lease_from_row(current), "idempotent": False} + + +def admit_task( + db_path: Path | str, + identity: TaskIdentity, + *, + payload: Any, + clock: Clock, +) -> dict[str, Any]: + """Persist a queued task, or return the identical admission.""" + normalized_payload, payload_json, payload_digest = _task_payload(payload) + now = _timestamp(clock) + with _transaction(db_path) as conn: + _load_active_room(conn, identity.room_id) + existing = conn.execute( + """SELECT * FROM hosted_room_driver_tasks + WHERE room_id=? AND task_id=?""", + (identity.room_id, identity.task_id), + ).fetchone() + if existing is not None: + if _task_identity_from_row(existing) != identity: + raise TaskConflictError("task_id is already bound to a different turn") + if ( + existing["payload_digest"] != payload_digest + or existing["payload_json"] != payload_json + ): + raise TaskConflictError( + "task_id is already bound to a different payload" + ) + return _task_from_row(existing, idempotent=True) + + turn = conn.execute( + """SELECT * FROM hosted_room_driver_tasks + WHERE room_id=? AND thread_id=? AND turn_id=?""", + (identity.room_id, identity.thread_id, identity.turn_id), + ).fetchone() + if turn is not None: + raise TaskConflictError("thread_id and turn_id are already bound to a task") + + conn.execute( + """INSERT INTO hosted_room_driver_tasks ( + room_id, task_id, thread_id, turn_id, + source_event_seq, payload_json, payload_digest, status, + execution_generation, cancel_generation, + created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, 'queued', 0, 0, ?, ?)""", + ( + identity.room_id, + identity.task_id, + identity.thread_id, + identity.turn_id, + normalized_payload["source_event_seq"], + payload_json, + payload_digest, + now, + now, + ), + ) + row = _load_task(conn, identity) + return _task_from_row(row) + + +def start_task( + db_path: Path | str, + identity: TaskIdentity, + lease: DriverLease, + *, + expected_cancel_generation: int, + clock: Clock, +) -> TaskAttempt: + """Move one queued task to running under the current driver lease.""" + if lease.room_id != identity.room_id: + raise DriverValidationError("lease and task belong to different rooms") + if ( + not isinstance(expected_cancel_generation, int) + or expected_cancel_generation < 0 + ): + raise DriverValidationError("expected_cancel_generation must be non-negative") + now = _timestamp(clock) + with _transaction(db_path) as conn: + _require_active_lease(conn, lease, now=now) + row = _load_task(conn, identity) + if int(row["cancel_generation"]) != expected_cancel_generation: + raise StaleTaskError("task cancellation generation changed") + if row["status"] != "queued": + raise InvalidTaskTransitionError( + f"cannot start task in state '{row['status']}'" + ) + unresolved = conn.execute( + """SELECT task_id, status FROM hosted_room_driver_tasks + WHERE room_id=? AND status IN ('running', 'indeterminate', 'stopping') + ORDER BY source_event_seq, created_at, task_id LIMIT 1""", + (identity.room_id,), + ).fetchone() + if unresolved is not None: + raise InvalidTaskTransitionError( + "room recovery must resolve the prior task before starting new work" + ) + next_queued = conn.execute( + """SELECT task_id FROM hosted_room_driver_tasks + WHERE room_id=? AND status='queued' + ORDER BY source_event_seq, created_at, task_id LIMIT 1""", + (identity.room_id,), + ).fetchone() + if next_queued is None or next_queued["task_id"] != identity.task_id: + raise InvalidTaskTransitionError( + "task is not next in the hosted room event order" + ) + execution_generation = int(row["execution_generation"]) + 1 + updated = conn.execute( + """UPDATE hosted_room_driver_tasks + SET status='running', execution_generation=?, + run_gateway_id=?, run_process_generation=?, + run_lease_generation=?, started_at=?, updated_at=? + WHERE room_id=? AND task_id=? AND status='queued' + AND cancel_generation=?""", + ( + execution_generation, + lease.gateway_id, + lease.process_generation, + lease.lease_generation, + now, + now, + identity.room_id, + identity.task_id, + expected_cancel_generation, + ), + ) + if updated.rowcount != 1: + raise StaleTaskError("task changed during start") + return TaskAttempt( + identity=identity, + lease=lease, + execution_generation=execution_generation, + cancel_generation=expected_cancel_generation, + ) + + +def settle_task( + db_path: Path | str, + attempt: TaskAttempt, + *, + settlement_id: Any, + status: TerminalStatus, + result: Any, + clock: Clock, +) -> dict[str, Any]: + """Commit one terminal result if every lease and task fence still matches.""" + settlement_id = _identifier(settlement_id, label="settlement_id") + if status not in {"settled", "failed"}: + raise DriverValidationError("status must be 'settled' or 'failed'") + result_json = _canonical_json(result) + now = _timestamp(clock) + + with _transaction(db_path) as conn: + row = _load_task(conn, attempt.identity) + if row["settlement_id"] is not None: + if ( + row["settlement_id"] == settlement_id + and row["settlement_status"] == status + and row["result_json"] == result_json + ): + return _task_from_row(row, idempotent=True) + raise TaskConflictError("task already has a different terminal settlement") + + _require_active_lease(conn, attempt.lease, now=now) + expected = ( + row["status"] == "running" + and int(row["execution_generation"]) == attempt.execution_generation + and int(row["cancel_generation"]) == attempt.cancel_generation + and row["run_gateway_id"] == attempt.lease.gateway_id + and row["run_process_generation"] == attempt.lease.process_generation + and int(row["run_lease_generation"]) == attempt.lease.lease_generation + ) + if not expected: + raise StaleTaskError("task attempt is stale or cancelled") + + updated = conn.execute( + """UPDATE hosted_room_driver_tasks + SET status=?, settlement_id=?, settlement_status=?, + result_json=?, terminal_at=?, updated_at=? + WHERE room_id=? AND task_id=? AND status='running' + AND execution_generation=? AND cancel_generation=? + AND run_gateway_id=? AND run_process_generation=? + AND run_lease_generation=?""", + ( + status, + settlement_id, + status, + result_json, + now, + now, + attempt.identity.room_id, + attempt.identity.task_id, + attempt.execution_generation, + attempt.cancel_generation, + attempt.lease.gateway_id, + attempt.lease.process_generation, + attempt.lease.lease_generation, + ), + ) + if updated.rowcount != 1: + raise StaleTaskError("task changed during settlement") + return _task_from_row(_load_task(conn, attempt.identity)) + + +def settle_stopping_task( + db_path: Path | str, + identity: TaskIdentity, + lease: DriverLease, + *, + expected_execution_generation: int, + expected_cancel_generation: int, + settlement_id: Any, + status: TerminalStatus, + result: Any, + clock: Clock, +) -> dict[str, Any]: + """Commit a completion that won the race with an unacknowledged Stop.""" + settlement_id = _identifier(settlement_id, label="settlement_id") + if status not in {"settled", "failed"}: + raise DriverValidationError("status must be 'settled' or 'failed'") + if expected_execution_generation < 1 or expected_cancel_generation < 1: + raise DriverValidationError("stopping settlement generations are invalid") + result_json = _canonical_json(result) + now = _timestamp(clock) + with _transaction(db_path) as conn: + row = _load_task(conn, identity) + if row["settlement_id"] is not None: + if ( + row["settlement_id"] == settlement_id + and row["settlement_status"] == status + and row["result_json"] == result_json + ): + return _task_from_row(row, idempotent=True) + raise TaskConflictError("task already has a different terminal settlement") + _require_active_lease(conn, lease, now=now) + updated = conn.execute( + """UPDATE hosted_room_driver_tasks + SET status=?, settlement_id=?, settlement_status=?, + result_json=?, terminal_at=?, updated_at=? + WHERE room_id=? AND task_id=? AND status='stopping' + AND execution_generation=? AND cancel_generation=?""", + ( + status, + settlement_id, + status, + result_json, + now, + now, + identity.room_id, + identity.task_id, + expected_execution_generation, + expected_cancel_generation, + ), + ) + if updated.rowcount != 1: + raise StaleTaskError("task completion lost the stop race") + return _task_from_row(_load_task(conn, identity)) + + +def resolve_indeterminate_task( + db_path: Path | str, + identity: TaskIdentity, + lease: DriverLease, + *, + expected_execution_generation: int, + expected_cancel_generation: int, + settlement_id: Any, + status: TerminalStatus, + result: Any, + clock: Clock, +) -> dict[str, Any]: + """Commit a verified historical receipt under the current room lease.""" + if lease.room_id != identity.room_id: + raise DriverValidationError("lease and task belong to different rooms") + if ( + not isinstance(expected_execution_generation, int) + or expected_execution_generation < 1 + ): + raise DriverValidationError( + "expected_execution_generation must be a positive integer" + ) + if ( + not isinstance(expected_cancel_generation, int) + or expected_cancel_generation < 0 + ): + raise DriverValidationError("expected_cancel_generation must be non-negative") + settlement_id = _identifier(settlement_id, label="settlement_id") + if status not in {"settled", "failed"}: + raise DriverValidationError("status must be 'settled' or 'failed'") + result_json = _canonical_json(result) + now = _timestamp(clock) + + with _transaction(db_path) as conn: + _require_active_lease(conn, lease, now=now) + row = _load_task(conn, identity) + if row["settlement_id"] is not None: + if ( + row["settlement_id"] == settlement_id + and row["settlement_status"] == status + and row["result_json"] == result_json + ): + return _task_from_row(row, idempotent=True) + raise TaskConflictError("task already has a different terminal settlement") + if ( + row["status"] != "indeterminate" + or int(row["execution_generation"]) != expected_execution_generation + or int(row["cancel_generation"]) != expected_cancel_generation + ): + raise StaleTaskError("indeterminate task generation changed") + updated = conn.execute( + """UPDATE hosted_room_driver_tasks + SET status=?, settlement_id=?, settlement_status=?, + result_json=?, terminal_at=?, updated_at=? + WHERE room_id=? AND task_id=? AND status='indeterminate' + AND execution_generation=? AND cancel_generation=?""", + ( + status, + settlement_id, + status, + result_json, + now, + now, + identity.room_id, + identity.task_id, + expected_execution_generation, + expected_cancel_generation, + ), + ) + if updated.rowcount != 1: + raise StaleTaskError("indeterminate task changed during reconciliation") + return _task_from_row(_load_task(conn, identity)) + + +def requeue_indeterminate_task( + db_path: Path | str, + identity: TaskIdentity, + lease: DriverLease, + *, + expected_execution_generation: int, + expected_cancel_generation: int, + clock: Clock, +) -> dict[str, Any]: + """Explicitly retry uncertain work after an operator accepts at-least-once risk.""" + if lease.room_id != identity.room_id: + raise DriverValidationError("lease and task belong to different rooms") + if ( + not isinstance(expected_execution_generation, int) + or expected_execution_generation < 1 + ): + raise DriverValidationError( + "expected_execution_generation must be a positive integer" + ) + if ( + not isinstance(expected_cancel_generation, int) + or expected_cancel_generation < 0 + ): + raise DriverValidationError("expected_cancel_generation must be non-negative") + now = _timestamp(clock) + with _transaction(db_path) as conn: + _require_active_lease(conn, lease, now=now) + row = _load_task(conn, identity) + if ( + row["status"] != "indeterminate" + or int(row["execution_generation"]) != expected_execution_generation + or int(row["cancel_generation"]) != expected_cancel_generation + ): + raise StaleTaskError("indeterminate task generation changed") + updated = conn.execute( + """UPDATE hosted_room_driver_tasks + SET status='queued', run_gateway_id=NULL, + run_process_generation=NULL, run_lease_generation=NULL, + started_at=NULL, indeterminate_at=NULL, updated_at=? + WHERE room_id=? AND task_id=? AND status='indeterminate' + AND execution_generation=? AND cancel_generation=?""", + ( + now, + identity.room_id, + identity.task_id, + expected_execution_generation, + expected_cancel_generation, + ), + ) + if updated.rowcount != 1: + raise StaleTaskError("indeterminate task changed during requeue") + return _task_from_row(_load_task(conn, identity)) + + +def defer_indeterminate_task( + db_path: Path | str, + identity: TaskIdentity, + lease: DriverLease, + *, + expected_execution_generation: int, + expected_cancel_generation: int, + reason: Any, + clock: Clock, +) -> dict[str, Any]: + """Fence one uncertain attempt and release later room work.""" + + if lease.room_id != identity.room_id: + raise DriverValidationError("lease and task belong to different rooms") + if ( + not isinstance(expected_execution_generation, int) + or expected_execution_generation < 1 + ): + raise DriverValidationError( + "expected_execution_generation must be a positive integer" + ) + if ( + not isinstance(expected_cancel_generation, int) + or expected_cancel_generation < 0 + ): + raise DriverValidationError("expected_cancel_generation must be non-negative") + reason = _identifier(reason, label="defer_reason") + result_json = _canonical_json({"reason": reason, "retryable": True}) + now = _timestamp(clock) + with _transaction(db_path) as conn: + _require_active_lease(conn, lease, now=now) + row = _load_task(conn, identity) + if ( + row["status"] == "deferred" + and int(row["execution_generation"]) == expected_execution_generation + and int(row["cancel_generation"]) == expected_cancel_generation + and row["result_json"] == result_json + ): + return _task_from_row(row, idempotent=True) + if ( + row["status"] != "indeterminate" + or int(row["execution_generation"]) != expected_execution_generation + or int(row["cancel_generation"]) != expected_cancel_generation + ): + raise StaleTaskError("indeterminate task generation changed") + updated = conn.execute( + """UPDATE hosted_room_driver_tasks + SET status='deferred', result_json=?, terminal_at=?, updated_at=? + WHERE room_id=? AND task_id=? AND status='indeterminate' + AND execution_generation=? AND cancel_generation=?""", + ( + result_json, + now, + now, + identity.room_id, + identity.task_id, + expected_execution_generation, + expected_cancel_generation, + ), + ) + if updated.rowcount != 1: + raise StaleTaskError("indeterminate task changed during deferral") + return _task_from_row(_load_task(conn, identity)) + + +def requeue_deferred_task( + db_path: Path | str, + identity: TaskIdentity, + lease: DriverLease, + *, + expected_execution_generation: int, + expected_cancel_generation: int, + clock: Clock, +) -> dict[str, Any]: + """Explicitly retry a fenced deferred turn under a new generation.""" + + if lease.room_id != identity.room_id: + raise DriverValidationError("lease and task belong to different rooms") + if ( + not isinstance(expected_execution_generation, int) + or expected_execution_generation < 1 + ): + raise DriverValidationError( + "expected_execution_generation must be a positive integer" + ) + if ( + not isinstance(expected_cancel_generation, int) + or expected_cancel_generation < 0 + ): + raise DriverValidationError("expected_cancel_generation must be non-negative") + now = _timestamp(clock) + with _transaction(db_path) as conn: + _require_active_lease(conn, lease, now=now) + row = _load_task(conn, identity) + if ( + row["status"] != "deferred" + or int(row["execution_generation"]) != expected_execution_generation + or int(row["cancel_generation"]) != expected_cancel_generation + ): + raise StaleTaskError("deferred task generation changed") + updated = conn.execute( + """UPDATE hosted_room_driver_tasks + SET status='queued', run_gateway_id=NULL, + run_process_generation=NULL, run_lease_generation=NULL, + result_json=NULL, started_at=NULL, terminal_at=NULL, + indeterminate_at=NULL, updated_at=? + WHERE room_id=? AND task_id=? AND status='deferred' + AND execution_generation=? AND cancel_generation=?""", + ( + now, + identity.room_id, + identity.task_id, + expected_execution_generation, + expected_cancel_generation, + ), + ) + if updated.rowcount != 1: + raise StaleTaskError("deferred task changed during requeue") + return _task_from_row(_load_task(conn, identity)) + + +def cancel_task( + db_path: Path | str, + identity: TaskIdentity, + *, + cancel_id: Any, + expected_cancel_generation: int, + clock: Clock, +) -> dict[str, Any]: + """Cancel a queued task before any external work was admitted.""" + cancel_id = _identifier(cancel_id, label="cancel_id") + if ( + not isinstance(expected_cancel_generation, int) + or expected_cancel_generation < 0 + ): + raise DriverValidationError("expected_cancel_generation must be non-negative") + now = _timestamp(clock) + with _transaction(db_path) as conn: + row = _load_task(conn, identity) + if row["status"] == "cancelled" and row["cancel_id"] == cancel_id: + return _task_from_row(row, idempotent=True) + if row["status"] in TERMINAL_STATUSES: + raise InvalidTaskTransitionError( + f"cannot cancel task in state '{row['status']}'" + ) + if row["status"] not in {"queued", "deferred"}: + raise InvalidTaskTransitionError( + "running work requires acknowledged two-phase cancellation" + ) + if int(row["cancel_generation"]) != expected_cancel_generation: + raise StaleTaskError("task cancellation generation changed") + + next_generation = expected_cancel_generation + 1 + updated = conn.execute( + """UPDATE hosted_room_driver_tasks + SET status='cancelled', cancel_generation=?, cancel_id=?, + terminal_at=?, updated_at=? + WHERE room_id=? AND task_id=? + AND status IN ('queued', 'deferred') + AND cancel_generation=?""", + ( + next_generation, + cancel_id, + now, + now, + identity.room_id, + identity.task_id, + expected_cancel_generation, + ), + ) + if updated.rowcount != 1: + raise StaleTaskError("task changed during cancellation") + return _task_from_row(_load_task(conn, identity)) + + +def begin_task_cancel( + db_path: Path | str, + identity: TaskIdentity, + *, + cancel_id: Any, + expected_cancel_generation: int, + clock: Clock, +) -> dict[str, Any]: + """Persist a stop intent without claiming the remote run has stopped.""" + cancel_id = _identifier(cancel_id, label="cancel_id") + if ( + not isinstance(expected_cancel_generation, int) + or expected_cancel_generation < 0 + ): + raise DriverValidationError("expected_cancel_generation must be non-negative") + now = _timestamp(clock) + with _transaction(db_path) as conn: + row = _load_task(conn, identity) + if row["status"] == "stopping" and row["cancel_id"] == cancel_id: + return _task_from_row(row, idempotent=True) + if row["status"] in TERMINAL_STATUSES or row["status"] == "queued": + raise InvalidTaskTransitionError( + f"cannot request remote stop in state '{row['status']}'" + ) + if int(row["cancel_generation"]) != expected_cancel_generation: + raise StaleTaskError("task cancellation generation changed") + updated = conn.execute( + """UPDATE hosted_room_driver_tasks + SET status='stopping', cancel_generation=?, cancel_id=?, + updated_at=? + WHERE room_id=? AND task_id=? + AND status IN ('running', 'indeterminate') + AND cancel_generation=?""", + ( + expected_cancel_generation + 1, + cancel_id, + now, + identity.room_id, + identity.task_id, + expected_cancel_generation, + ), + ) + if updated.rowcount != 1: + raise StaleTaskError("task changed during stop request") + return _task_from_row(_load_task(conn, identity)) + + +def complete_task_cancel( + db_path: Path | str, + identity: TaskIdentity, + *, + cancel_id: Any, + expected_cancel_generation: int, + clock: Clock, +) -> dict[str, Any]: + """Commit cancellation only after the transport acknowledges exact Stop.""" + cancel_id = _identifier(cancel_id, label="cancel_id") + now = _timestamp(clock) + with _transaction(db_path) as conn: + row = _load_task(conn, identity) + if row["status"] == "cancelled" and row["cancel_id"] == cancel_id: + return _task_from_row(row, idempotent=True) + if ( + row["status"] != "stopping" + or row["cancel_id"] != cancel_id + or int(row["cancel_generation"]) != expected_cancel_generation + ): + raise StaleTaskError("task stop acknowledgement is stale") + updated = conn.execute( + """UPDATE hosted_room_driver_tasks + SET status='cancelled', terminal_at=?, updated_at=? + WHERE room_id=? AND task_id=? AND status='stopping' + AND cancel_id=? AND cancel_generation=?""", + ( + now, + now, + identity.room_id, + identity.task_id, + cancel_id, + expected_cancel_generation, + ), + ) + if updated.rowcount != 1: + raise StaleTaskError("task changed during stop acknowledgement") + return _task_from_row(_load_task(conn, identity)) + + +def recover_room( + db_path: Path | str, + lease: DriverLease, + *, + clock: Clock, +) -> dict[str, list[TaskIdentity]]: + """Fence abandoned running attempts without requeueing uncertain work.""" + now = _timestamp(clock) + with _transaction(db_path) as conn: + _require_active_lease(conn, lease, now=now) + stale_rows = conn.execute( + """SELECT * FROM hosted_room_driver_tasks + WHERE room_id=? AND status='running' + AND NOT ( + run_gateway_id=? AND run_process_generation=? + AND run_lease_generation=? + ) + ORDER BY source_event_seq, created_at, task_id""", + ( + lease.room_id, + lease.gateway_id, + lease.process_generation, + lease.lease_generation, + ), + ).fetchall() + if stale_rows: + conn.execute( + """UPDATE hosted_room_driver_tasks + SET status='indeterminate', indeterminate_at=?, updated_at=? + WHERE room_id=? AND status='running' + AND NOT ( + run_gateway_id=? AND run_process_generation=? + AND run_lease_generation=? + )""", + ( + now, + now, + lease.room_id, + lease.gateway_id, + lease.process_generation, + lease.lease_generation, + ), + ) + queued_rows = conn.execute( + """SELECT * FROM hosted_room_driver_tasks + WHERE room_id=? AND status='queued' + ORDER BY source_event_seq, created_at, task_id""", + (lease.room_id,), + ).fetchall() + indeterminate_rows = conn.execute( + """SELECT * FROM hosted_room_driver_tasks + WHERE room_id=? AND status='indeterminate' + ORDER BY source_event_seq, created_at, task_id""", + (lease.room_id,), + ).fetchall() + return { + "queued": [_task_identity_from_row(row) for row in queued_rows], + "indeterminate": [ + _task_identity_from_row(row) for row in indeterminate_rows + ], + } + + +def get_task( + db_path: Path | str, + identity: TaskIdentity, +) -> dict[str, Any]: + """Read one task without mutating its state.""" + conn = _connect(db_path) + try: + return _task_from_row(_load_task(conn, identity)) + finally: + conn.close() + + +def list_tasks( + db_path: Path | str, + *, + room_id: Any, + status: TaskStatus | None = None, +) -> list[dict[str, Any]]: + """Return room tasks in deterministic admission order.""" + room_id = _identifier(room_id, label="room_id") + if status is not None and status not in TASK_STATUSES: + raise DriverValidationError("invalid task status") + conn = _connect(db_path) + try: + if status is None: + rows = conn.execute( + """SELECT * FROM hosted_room_driver_tasks + WHERE room_id=? + ORDER BY source_event_seq, created_at, task_id""", + (room_id,), + ).fetchall() + else: + rows = conn.execute( + """SELECT * FROM hosted_room_driver_tasks + WHERE room_id=? AND status=? + ORDER BY source_event_seq, created_at, task_id""", + (room_id, status), + ).fetchall() + return [_task_from_row(row) for row in rows] + finally: + conn.close() + + +def prune_published_terminal_tasks( + db_path: Path | str, + *, + room_id: Any, + clock: Clock, + retention_seconds: float = TERMINAL_TASK_RETENTION_SECONDS, + retain: int = MAX_RETAINED_TERMINAL_TASKS, +) -> int: + """Bound execution rows after outcomes are durable in the room log.""" + + room_id = _identifier(room_id, label="room_id") + now = _timestamp(clock) + if retention_seconds <= 0: + raise DriverValidationError("retention_seconds must be positive") + if isinstance(retain, bool) or not isinstance(retain, int) or retain < 0: + raise DriverValidationError("retain must be a non-negative integer") + + with _transaction(db_path) as conn: + publications = conn.execute( + """SELECT 1 FROM sqlite_master + WHERE type='table' AND name='hosted_room_policy_publications'""" + ).fetchone() + if publications is None: + return 0 + rows = conn.execute( + """SELECT t.task_id, t.terminal_at + FROM hosted_room_driver_tasks t + WHERE t.room_id=? + AND t.status IN ('settled', 'failed', 'cancelled') + AND EXISTS ( + SELECT 1 FROM hosted_room_policy_publications p + WHERE p.room_id=t.room_id AND p.task_id=t.task_id + AND p.kind IN ( + 'turn.settled', 'turn.failed', 'turn.cancelled' + ) + ) + ORDER BY t.terminal_at DESC, t.task_id ASC""", + (room_id,), + ).fetchall() + cutoff = now - float(retention_seconds) + candidates = [ + str(row["task_id"]) + for index, row in enumerate(rows) + if index >= retain + or (row["terminal_at"] is not None and float(row["terminal_at"]) <= cutoff) + ][:MAX_TASK_PRUNE_BATCH] + if not candidates: + return 0 + placeholders = ",".join("?" for _ in candidates) + deleted = conn.execute( + f"""DELETE FROM hosted_room_driver_tasks + WHERE room_id=? AND task_id IN ({placeholders})""", + (room_id, *candidates), + ) + return max(0, int(deleted.rowcount)) diff --git a/gateway/hosted_room_policy_checkpoint.py b/gateway/hosted_room_policy_checkpoint.py new file mode 100644 index 0000000000..e141b0e158 --- /dev/null +++ b/gateway/hosted_room_policy_checkpoint.py @@ -0,0 +1,682 @@ +"""Durable bounded policy projection for hosted Group Chat preparation. + +The append-only room log remains the user-visible source of truth. This module +materializes only the state needed to choose and reconstruct the next active +discussion, so a busy room does not replay its complete history every poll. +""" + +from __future__ import annotations + +import json +import sqlite3 +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +from gateway import hosted_rooms + + +MAX_ACTIVE_POLICY_EVENTS = 64 +MAX_THREAD_TRANSCRIPT_EVENTS = 24 +_TRANSCRIPT_SCHEMA_VERSION = 1 +_TERMINAL_KINDS = frozenset({ + "turn.settled", + "turn.failed", + "turn.cancelled", + "turn.deferred", +}) + + +@dataclass(frozen=True) +class PolicySnapshot: + """Bounded active policy input at one durable room-log cursor.""" + + through_seq: int + stopped_through_seq: int + events: tuple[dict[str, Any], ...] + watermarks: Mapping[tuple[str, str], int] + + +class HostedRoomPolicyCheckpoint: + """Incrementally index room policy without compacting visible history.""" + + def __init__(self, db_path: Path | str) -> None: + self.db_path = Path(db_path) + self._initialize() + + def _connect(self) -> sqlite3.Connection: + from hermes_state import apply_wal_with_fallback + + self.db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(self.db_path, timeout=10) + conn.row_factory = sqlite3.Row + apply_wal_with_fallback(conn, db_label="state.db (room policy checkpoint)") + return conn + + def _initialize(self) -> None: + with self._connect() as conn: + conn.execute( + """CREATE TABLE IF NOT EXISTS hosted_room_policy_cursors ( + room_id TEXT PRIMARY KEY, + through_seq INTEGER NOT NULL DEFAULT 0, + stopped_through_seq INTEGER NOT NULL DEFAULT 0, + updated_at REAL NOT NULL DEFAULT 0 + )""" + ) + conn.execute( + """CREATE TABLE IF NOT EXISTS hosted_room_policy_threads ( + room_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + discussion_event_id TEXT NOT NULL, + latest_user_seq INTEGER NOT NULL, + completed INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY(room_id, thread_id) + )""" + ) + conn.execute( + """CREATE INDEX IF NOT EXISTS idx_hosted_room_policy_pending + ON hosted_room_policy_threads( + room_id, completed, latest_user_seq, thread_id + )""" + ) + conn.execute( + """CREATE TABLE IF NOT EXISTS hosted_room_policy_events ( + room_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + discussion_event_id TEXT NOT NULL, + seq INTEGER NOT NULL, + event_json TEXT NOT NULL, + PRIMARY KEY(room_id, seq) + )""" + ) + conn.execute( + """CREATE INDEX IF NOT EXISTS idx_hosted_room_policy_events_active + ON hosted_room_policy_events( + room_id, discussion_event_id, seq + )""" + ) + conn.execute( + """CREATE TABLE IF NOT EXISTS hosted_room_policy_watermarks ( + room_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + member_id TEXT NOT NULL, + seen_through_seq INTEGER NOT NULL, + PRIMARY KEY(room_id, thread_id, member_id) + )""" + ) + conn.execute( + """CREATE TABLE IF NOT EXISTS hosted_room_policy_publications ( + room_id TEXT NOT NULL, + task_id TEXT NOT NULL, + kind TEXT NOT NULL, + execution_generation INTEGER NOT NULL DEFAULT 0, + seq INTEGER NOT NULL, + PRIMARY KEY(room_id, task_id, kind, execution_generation) + )""" + ) + conn.execute( + # Store only references into the already bounded room log. This + # avoids duplicating prompt payloads outside room byte limits. + """CREATE TABLE IF NOT EXISTS hosted_room_policy_transcript ( + room_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + seq INTEGER NOT NULL, + kind TEXT NOT NULL, + settled_seq INTEGER, + PRIMARY KEY(room_id, thread_id, seq) + )""" + ) + conn.execute( + """CREATE TABLE IF NOT EXISTS hosted_room_policy_transcript_state ( + room_id TEXT PRIMARY KEY, + schema_version INTEGER NOT NULL + )""" + ) + + @staticmethod + def _event_json(event: Mapping[str, Any]) -> str: + return json.dumps( + dict(event), + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + + def _store_active_event( + self, + conn: sqlite3.Connection, + *, + event: Mapping[str, Any], + thread_id: str, + discussion_event_id: str, + ) -> None: + conn.execute( + """INSERT OR IGNORE INTO hosted_room_policy_events( + room_id, thread_id, discussion_event_id, seq, event_json + ) VALUES (?, ?, ?, ?, ?)""", + ( + event["room_id"], + thread_id, + discussion_event_id, + int(event["seq"]), + self._event_json(event), + ), + ) + + def _store_transcript_event( + self, + conn: sqlite3.Connection, + *, + event: Mapping[str, Any], + thread_id: str, + settled_seq: int | None = None, + ) -> None: + conn.execute( + """INSERT INTO hosted_room_policy_transcript( + room_id, thread_id, seq, kind, settled_seq + ) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(room_id, thread_id, seq) DO UPDATE SET + settled_seq=COALESCE( + excluded.settled_seq, + hosted_room_policy_transcript.settled_seq + )""", + ( + event["room_id"], + thread_id, + int(event["seq"]), + str(event["kind"]), + settled_seq, + ), + ) + if event["kind"] in {"message.user", "message.member"}: + cutoff = conn.execute( + """SELECT seq FROM hosted_room_policy_transcript + WHERE room_id=? AND thread_id=? + AND kind IN ('message.user', 'message.member') + ORDER BY seq DESC LIMIT 1 OFFSET ?""", + ( + event["room_id"], + thread_id, + MAX_THREAD_TRANSCRIPT_EVENTS - 1, + ), + ).fetchone() + if cutoff is not None: + conn.execute( + """DELETE FROM hosted_room_policy_transcript + WHERE room_id=? AND thread_id=? AND seq dict[str, Any]: + return { + "room_id": str(row["room_id"]), + "seq": int(row["seq"]), + "event_id": str(row["event_id"]), + "kind": str(row["kind"]), + "actor": json.loads(row["actor_json"]), + "authority_epoch": row["authority_epoch"], + "payload": json.loads(row["payload_json"]), + "created_at": float(row["created_at"]), + "idempotent": False, + } + + def _backfill_transcript( + self, + conn: sqlite3.Connection, + *, + room_id: str, + through_seq: int, + ) -> None: + """Migrate bounded committed thread history from the durable room log.""" + + if through_seq <= 0: + return + settled_seq_by_message: dict[str, int] = {} + for row in conn.execute( + """SELECT seq, payload_json FROM hosted_room_events + WHERE room_id=? AND seq<=? AND kind='turn.settled' + ORDER BY seq""", + (room_id, through_seq), + ): + message_event_id = str( + json.loads(row["payload_json"]).get("message_event_id") or "" + ) + if message_event_id: + settled_seq_by_message[message_event_id] = int(row["seq"]) + rows = conn.execute( + """SELECT room_id, seq, event_id, kind, actor_json, + authority_epoch, payload_json, created_at + FROM hosted_room_events + WHERE room_id=? AND seq<=? + AND kind IN ('message.user', 'message.member') + ORDER BY seq""", + (room_id, through_seq), + ) + for row in rows: + if ( + row["kind"] == "message.member" + and row["event_id"] not in settled_seq_by_message + ): + continue + event = self._event_from_room_row(row) + thread_id = str(event["payload"].get("thread_id") or "") + if thread_id: + self._store_transcript_event( + conn, + event=event, + thread_id=thread_id, + settled_seq=settled_seq_by_message.get(str(row["event_id"])), + ) + + def _transcript_events( + self, + conn: sqlite3.Connection, + *, + room_id: str, + thread_id: str, + ) -> list[dict[str, Any]]: + rows = conn.execute( + """WITH transcript_events(seq) AS ( + SELECT seq FROM hosted_room_policy_transcript + WHERE room_id=? AND thread_id=? + UNION ALL + SELECT settled_seq FROM hosted_room_policy_transcript + WHERE room_id=? AND thread_id=? AND settled_seq IS NOT NULL + ) + SELECT events.room_id, events.seq, events.event_id, + events.kind, events.actor_json, + events.authority_epoch, events.payload_json, + events.created_at + FROM transcript_events + JOIN hosted_room_events AS events + ON events.room_id=? AND events.seq=transcript_events.seq + ORDER BY events.seq""", + (room_id, thread_id, room_id, thread_id, room_id), + ).fetchall() + return [self._event_from_room_row(row) for row in rows] + + def _apply_event(self, conn: sqlite3.Connection, event: Mapping[str, Any]) -> None: + room_id = str(event["room_id"]) + seq = int(event["seq"]) + kind = str(event.get("kind") or "") + payload = event.get("payload") + payload = payload if isinstance(payload, Mapping) else {} + + if kind == "message.user": + thread_id = str(payload.get("thread_id") or "") + event_id = str(event.get("event_id") or "") + if not thread_id or not event_id: + return + conn.execute( + """INSERT INTO hosted_room_policy_threads( + room_id, thread_id, discussion_event_id, + latest_user_seq, completed + ) VALUES (?, ?, ?, ?, 0) + ON CONFLICT(room_id, thread_id) DO UPDATE SET + discussion_event_id=excluded.discussion_event_id, + latest_user_seq=excluded.latest_user_seq, + completed=0""", + (room_id, thread_id, event_id, seq), + ) + self._store_active_event( + conn, + event=event, + thread_id=thread_id, + discussion_event_id=event_id, + ) + self._store_transcript_event( + conn, + event=event, + thread_id=thread_id, + ) + return + + if kind in {"message.member", *_TERMINAL_KINDS}: + thread_id = str(payload.get("thread_id") or "") + discussion_event_id = str(payload.get("discussion_event_id") or "") + source = conn.execute( + """SELECT 1 FROM hosted_room_policy_events + WHERE room_id=? AND discussion_event_id=? LIMIT 1""", + (room_id, discussion_event_id), + ).fetchone() + if source is None: + return + self._store_active_event( + conn, + event=event, + thread_id=thread_id, + discussion_event_id=discussion_event_id, + ) + if kind in _TERMINAL_KINDS: + task_id = str(payload.get("task_id") or "") + execution_generation = ( + int(payload.get("execution_generation") or 0) + if kind == "turn.deferred" + else 0 + ) + if task_id: + conn.execute( + """INSERT OR IGNORE INTO hosted_room_policy_publications( + room_id, task_id, kind, execution_generation, seq + ) VALUES (?, ?, ?, ?, ?)""", + ( + room_id, + task_id, + kind, + execution_generation, + seq, + ), + ) + member_id = str(payload.get("member_id") or "") + seen_through_seq = int(payload.get("seen_through_seq") or 0) + if kind == "turn.settled" and payload.get("message_event_id"): + messages = conn.execute( + """SELECT seq, event_json FROM hosted_room_policy_events + WHERE room_id=? AND discussion_event_id=?""", + (room_id, discussion_event_id), + ).fetchall() + decoded_messages = [ + json.loads(message["event_json"]) + for message in messages + ] + committed = next( + ( + message + for message in decoded_messages + if message.get("event_id") + == payload["message_event_id"] + ), + None, + ) + if committed is not None: + seen_through_seq = max( + seen_through_seq, + int(committed["seq"]), + ) + self._store_transcript_event( + conn, + event=committed, + thread_id=thread_id, + settled_seq=int(event["seq"]), + ) + if member_id and seen_through_seq > 0: + conn.execute( + """INSERT INTO hosted_room_policy_watermarks( + room_id, thread_id, member_id, seen_through_seq + ) VALUES (?, ?, ?, ?) + ON CONFLICT(room_id, thread_id, member_id) DO UPDATE SET + seen_through_seq=MAX( + hosted_room_policy_watermarks.seen_through_seq, + excluded.seen_through_seq + )""", + (room_id, thread_id, member_id, seen_through_seq), + ) + return + + if kind == "room.activity": + thread_id = str(payload.get("thread_id") or "") + discussion_event_id = str(payload.get("discussion_event_id") or "") + conn.execute( + """DELETE FROM hosted_room_policy_events + WHERE room_id=? AND discussion_event_id=?""", + (room_id, discussion_event_id), + ) + conn.execute( + """DELETE FROM hosted_room_policy_threads + WHERE room_id=? AND thread_id=?""", + (room_id, thread_id), + ) + return + + if kind == "room.stop_requested": + conn.execute( + """UPDATE hosted_room_policy_cursors + SET stopped_through_seq=MAX(stopped_through_seq, ?) + WHERE room_id=?""", + (seq, room_id), + ) + + def sync(self, *, room_id: str, latest_seq: int) -> int: + """Materialize each unseen event exactly once by durable cursor.""" + + with self._connect() as conn: + conn.execute("BEGIN IMMEDIATE") + if conn.execute( + "SELECT 1 FROM hosted_rooms WHERE room_id=?", + (room_id,), + ).fetchone() is None: + raise hosted_rooms.RoomNotFoundError("hosted room not found") + conn.execute( + """INSERT OR IGNORE INTO hosted_room_policy_cursors( + room_id, through_seq, stopped_through_seq, updated_at + ) VALUES (?, 0, 0, 0)""", + (room_id,), + ) + row = conn.execute( + "SELECT through_seq FROM hosted_room_policy_cursors WHERE room_id=?", + (room_id,), + ).fetchone() + cursor = int(row["through_seq"]) + transcript_state = conn.execute( + """SELECT schema_version + FROM hosted_room_policy_transcript_state WHERE room_id=?""", + (room_id,), + ).fetchone() + if ( + transcript_state is None + or int(transcript_state["schema_version"]) + < _TRANSCRIPT_SCHEMA_VERSION + ): + self._backfill_transcript( + conn, + room_id=room_id, + through_seq=cursor, + ) + conn.execute( + """INSERT INTO hosted_room_policy_transcript_state( + room_id, schema_version + ) VALUES (?, ?) + ON CONFLICT(room_id) DO UPDATE SET + schema_version=excluded.schema_version""", + (room_id, _TRANSCRIPT_SCHEMA_VERSION), + ) + if cursor > latest_seq: + raise RuntimeError("room policy cursor is ahead of the durable log") + + while cursor < latest_seq: + page = hosted_rooms.read_events( + self.db_path, + room_id=room_id, + since_seq=cursor, + limit=hosted_rooms.MAX_LOG_LIMIT, + ) + rows = [ + event for event in page.get("events", []) if isinstance(event, Mapping) + ] + next_cursor = int(page.get("cursor") or cursor) + if not rows or next_cursor <= cursor: + raise RuntimeError("hosted room policy cursor did not advance") + with self._connect() as conn: + conn.execute("BEGIN IMMEDIATE") + if conn.execute( + "SELECT 1 FROM hosted_rooms WHERE room_id=?", + (room_id,), + ).fetchone() is None: + raise hosted_rooms.RoomNotFoundError("hosted room not found") + for event in rows: + self._apply_event(conn, event) + updated = conn.execute( + """UPDATE hosted_room_policy_cursors + SET through_seq=?, updated_at=? WHERE room_id=?""", + ( + next_cursor, + float(rows[-1].get("created_at") or 0), + room_id, + ), + ) + if updated.rowcount != 1: + raise RuntimeError("room policy cursor disappeared during replay") + cursor = next_cursor + return cursor + + def snapshot(self, *, room_id: str, latest_seq: int) -> PolicySnapshot: + """Return only the oldest active discussion and its watermark set.""" + + through_seq = self.sync(room_id=room_id, latest_seq=latest_seq) + with self._connect() as conn: + cursor = conn.execute( + """SELECT stopped_through_seq FROM hosted_room_policy_cursors + WHERE room_id=?""", + (room_id,), + ).fetchone() + stopped_through_seq = int(cursor["stopped_through_seq"]) + thread = conn.execute( + """SELECT thread_id, discussion_event_id + FROM hosted_room_policy_threads + WHERE room_id=? AND completed=0 AND latest_user_seq>? + ORDER BY latest_user_seq, thread_id LIMIT 1""", + (room_id, stopped_through_seq), + ).fetchone() + if thread is None: + return PolicySnapshot( + through_seq=through_seq, + stopped_through_seq=stopped_through_seq, + events=(), + watermarks={}, + ) + active_rows = conn.execute( + """SELECT event_json FROM hosted_room_policy_events + WHERE room_id=? AND discussion_event_id=? + ORDER BY seq LIMIT ?""", + ( + room_id, + str(thread["discussion_event_id"]), + MAX_ACTIVE_POLICY_EVENTS + 1, + ), + ).fetchall() + if len(active_rows) > MAX_ACTIVE_POLICY_EVENTS: + raise RuntimeError("active room policy projection exceeded its bound") + transcript_events = self._transcript_events( + conn, + room_id=room_id, + thread_id=str(thread["thread_id"]), + ) + watermark_rows = conn.execute( + """SELECT member_id, seen_through_seq + FROM hosted_room_policy_watermarks + WHERE room_id=? AND thread_id=?""", + (room_id, str(thread["thread_id"])), + ).fetchall() + events_by_seq = { + int(event["seq"]): event + for event in ( + *transcript_events, + *(json.loads(row["event_json"]) for row in active_rows), + ) + } + return PolicySnapshot( + through_seq=through_seq, + stopped_through_seq=stopped_through_seq, + events=tuple(events_by_seq[seq] for seq in sorted(events_by_seq)), + watermarks={ + (str(thread["thread_id"]), str(row["member_id"])): int( + row["seen_through_seq"] + ) + for row in watermark_rows + }, + ) + + def publication_exists( + self, + *, + room_id: str, + task_id: str, + status: str, + execution_generation: int, + ) -> bool: + """Return whether one exact driver outcome is already in the room log.""" + + kind = f"turn.{status}" + generation = execution_generation if status == "deferred" else 0 + with self._connect() as conn: + if status == "deferred": + row = conn.execute( + """SELECT 1 FROM hosted_room_policy_publications + WHERE room_id=? AND task_id=? AND kind=? + AND execution_generation=?""", + (room_id, task_id, kind, generation), + ).fetchone() + else: + row = conn.execute( + """SELECT 1 FROM hosted_room_policy_publications + WHERE room_id=? AND task_id=? AND kind IN ( + 'turn.settled', 'turn.failed', 'turn.cancelled' + )""", + (room_id, task_id), + ).fetchone() + return row is not None + + def events_for_task( + self, + *, + room_id: str, + source_event_seq: int, + ) -> list[dict[str, Any]]: + """Load one bounded discussion projection for terminal reconstruction.""" + + with self._connect() as conn: + source = conn.execute( + """SELECT discussion_event_id, thread_id + FROM hosted_room_policy_events + WHERE room_id=? AND seq=?""", + (room_id, source_event_seq), + ).fetchone() + if source is None: + return [] + active_rows = conn.execute( + """SELECT event_json FROM hosted_room_policy_events + WHERE room_id=? AND discussion_event_id=? + ORDER BY seq LIMIT ?""", + ( + room_id, + str(source["discussion_event_id"]), + MAX_ACTIVE_POLICY_EVENTS + 1, + ), + ).fetchall() + transcript_events = self._transcript_events( + conn, + room_id=room_id, + thread_id=str(source["thread_id"]), + ) + if len(active_rows) > MAX_ACTIVE_POLICY_EVENTS: + raise RuntimeError("task policy projection exceeded its bound") + events_by_seq = { + int(event["seq"]): event + for event in ( + *transcript_events, + *(json.loads(row["event_json"]) for row in active_rows), + ) + } + return [events_by_seq[seq] for seq in sorted(events_by_seq)] + + def compact_completed(self, *, room_id: str) -> None: + """Drop any completed projections left by an interrupted sync.""" + + with self._connect() as conn: + completed = conn.execute( + """SELECT discussion_event_id FROM hosted_room_policy_threads + WHERE room_id=? AND completed=1""", + (room_id,), + ).fetchall() + for row in completed: + conn.execute( + """DELETE FROM hosted_room_policy_events + WHERE room_id=? AND discussion_event_id=?""", + (room_id, str(row["discussion_event_id"])), + ) + conn.execute( + """DELETE FROM hosted_room_policy_threads + WHERE room_id=? AND completed=1""", + (room_id,), + ) diff --git a/gateway/hosted_rooms.py b/gateway/hosted_rooms.py index ec8da694f1..625bfea9df 100644 --- a/gateway/hosted_rooms.py +++ b/gateway/hosted_rooms.py @@ -81,6 +81,7 @@ _EVENT_KINDS_BY_ACTOR = { "gateway": frozenset({ "member.unavailable", "room.activity", + "room.stop_requested", "turn.deferred", "turn.reassigned", "turn.cancelled", @@ -118,6 +119,10 @@ class RoomConflictError(HostedRoomError): """Raised when an idempotency key is reused for different room state.""" +class RoomProbeUnavailableError(HostedRoomError): + """Raised when a non-blocking ownership probe cannot read the room store.""" + + class EventConflictError(HostedRoomError): """Raised when an event id is reused with different immutable content.""" @@ -444,6 +449,27 @@ def _connect(db_path: Path | str) -> sqlite3.Connection: return conn +def _read_connection(db_path: Path | str) -> sqlite3.Connection: + """Open the room store without steady-state journal or migration writes.""" + + path = Path(db_path) + if not path.is_file(): + initialized = _connect(path) + initialized.close() + conn = sqlite3.connect(path, timeout=10) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys=ON") + if _schema_is_current(conn): + return conn + conn.close() + migrated = _connect(path) + migrated.close() + conn = sqlite3.connect(path, timeout=10) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys=ON") + return conn + + @contextmanager def _transaction( db_path: Path | str, *, immediate: bool = False @@ -481,6 +507,16 @@ def _raise_room_not_found(conn: sqlite3.Connection, room_id: str) -> NoReturn: raise RoomNotFoundError("hosted room not found") +def _table_exists(conn: sqlite3.Connection, table: str) -> bool: + return ( + conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", + (table,), + ).fetchone() + is not None + ) + + def _room_from_row(row: sqlite3.Row, *, idempotent: bool = False) -> dict[str, Any]: room = { "room_id": row["room_id"], @@ -609,10 +645,24 @@ def _prune_disbanded_rooms_locked( WHERE room_id IN ({placeholders}) AND disbanded_at IS NOT NULL""", room_ids, ) - conn.execute( - f"DELETE FROM hosted_room_events WHERE room_id IN ({placeholders})", - room_ids, + dependent_tables = ( + "hosted_room_policy_transcript_state", + "hosted_room_policy_transcript", + "hosted_room_policy_publications", + "hosted_room_policy_watermarks", + "hosted_room_policy_events", + "hosted_room_policy_threads", + "hosted_room_policy_cursors", + "hosted_room_driver_tasks", + "hosted_room_driver_leases", + "hosted_room_events", ) + for table in dependent_tables: + if _table_exists(conn, table): + conn.execute( + f"DELETE FROM {table} WHERE room_id IN ({placeholders})", + room_ids, + ) conn.execute( f"DELETE FROM hosted_rooms WHERE room_id IN ({placeholders})", room_ids, @@ -823,7 +873,7 @@ def list_rooms( limit: int = MAX_ROOM_LIST_LIMIT, offset: int = 0, ) -> list[dict[str, Any]]: - """Return one bounded page of rooms ordered by most recent change.""" + """Return one bounded read-only page ordered by most recent change.""" if ( isinstance(limit, bool) or not isinstance(limit, int) @@ -832,8 +882,8 @@ def list_rooms( raise HostedRoomError(f"limit must be between 1 and {MAX_ROOM_LIST_LIMIT}") if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0: raise HostedRoomError("offset must be a non-negative integer") - with _transaction(db_path, immediate=True) as conn: - _prune_disbanded_rooms_locked(conn, now=None) + conn = _read_connection(db_path) + try: rows = conn.execute( """SELECT room_id, name, members_json, authority_gateway_id, authority_epoch, next_seq, revision, created_at, updated_at, @@ -844,6 +894,8 @@ def list_rooms( LIMIT ? OFFSET ?""", (int(include_disbanded), limit, offset), ).fetchall() + finally: + conn.close() return [_room_from_row(row) for row in rows] @@ -1002,6 +1054,47 @@ def append_event( return result +def probe_hosted_room(db_path: Path | str, *, room_id: Any) -> bool: + """Check room ownership without creating or migrating the shared store. + + This runs on the synchronous prompt-admission path for older Desktop + clients, so it fails quickly under contention instead of blocking the + WebSocket reader for SQLite's normal ten-second timeout. + """ + + checked_room_id = _validate_identifier( + room_id, + label="room_id", + max_chars=MAX_ROOM_ID_CHARS, + ) + path = Path(db_path) + if not path.is_file(): + return False + try: + conn = sqlite3.connect(path, timeout=0.05) + try: + table = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' " + "AND name='hosted_rooms' LIMIT 1" + ).fetchone() + if table is None: + return False + return ( + conn.execute( + "SELECT 1 FROM hosted_rooms WHERE room_id=? " + "AND disbanded_at IS NULL LIMIT 1", + (checked_room_id,), + ).fetchone() + is not None + ) + finally: + conn.close() + except sqlite3.Error as exc: + raise RoomProbeUnavailableError( + "hosted room ownership is temporarily unavailable" + ) from exc + + def room_state( db_path: Path | str, *, @@ -1041,6 +1134,34 @@ def room_state( return state +def request_room_stop( + db_path: Path | str, + *, + room_id: Any, + cancel_id: Any, + expected_gateway_id: Any, + expected_epoch: Any, +) -> dict[str, Any]: + """Append an idempotent fence that supersedes earlier user turns.""" + + cancel_id = _validate_identifier( + cancel_id, + label="cancel_id", + max_chars=MAX_EVENT_ID_CHARS, + ) + digest = hashlib.sha256(cancel_id.encode()).hexdigest()[:32] + return append_event( + db_path, + room_id=room_id, + event_id=f"room-stop:{digest}", + kind="room.stop_requested", + actor={"kind": "gateway", "id": expected_gateway_id}, + payload={"cancel_id": cancel_id}, + authority_gateway_id=expected_gateway_id, + authority_epoch=expected_epoch, + ) + + def claim_authority( db_path: Path | str, *, diff --git a/gateway/run.py b/gateway/run.py index 7aa6cf7930..23dadfe430 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -13082,6 +13082,43 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew logger.warning("Legacy session recovery on startup failed: %s", exc) return exact, fallback + @staticmethod + def _start_hosted_room_worker_sync(): + """Start the local Group Chat worker without importing the dashboard.""" + + import tui_gateway.server # noqa: F401 + from tui_gateway import methods_groups + + service = methods_groups.get_hosted_room_service() + if service is None: + service = methods_groups.start_hosted_room_service() + if service is None: + raise RuntimeError("Group Chat worker has no bound session backend") + status = service.runtime.status() + if not status.get("running") or status.get("stopping"): + raise RuntimeError("Group Chat worker did not start") + return service + + async def _ensure_hosted_room_worker(self): + return await asyncio.to_thread(self._start_hosted_room_worker_sync) + + async def _hosted_room_worker_watcher(self, interval: float = 1.0) -> None: + """Keep the room worker alive for the messaging gateway lifetime.""" + + while self._running: + await self._ensure_hosted_room_worker() + await asyncio.sleep(interval) + + async def _stop_hosted_room_worker(self, timeout: float = 5.0) -> bool: + """Pause room execution durably without interrupting accepted turns.""" + + from tui_gateway import methods_groups + + return await asyncio.to_thread( + methods_groups.stop_hosted_room_service, + timeout=timeout, + ) + def _start_loop_heartbeat_task(self) -> None: """Start the loop-liveness heartbeat task (#66892), idempotent. @@ -13888,6 +13925,19 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._install_plugin_message_injector() self._update_runtime_status("running") + try: + await self._ensure_hosted_room_worker() + except Exception: + logger.error( + "Group Chat worker failed to start; mutating Group Chat commands " + "will fail closed until supervision recovers it", + exc_info=True, + ) + self._spawn_supervised( + self._hosted_room_worker_watcher, + "hosted_room_worker", + ) + self._start_loop_heartbeat_task() # Emit gateway:startup hook @@ -15744,6 +15794,22 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew self._clear_plugin_message_injector() self._draining = True + stop_room_worker = getattr(self, "_stop_hosted_room_worker", None) + if callable(stop_room_worker): + try: + stopped = await stop_room_worker(timeout=5.0) + if not stopped: + logger.warning( + "Group Chat worker is still settling durable work; " + "the next gateway start will recover it" + ) + except Exception: + logger.warning( + "Group Chat worker could not stop cleanly; the next gateway " + "start will recover durable work", + exc_info=True, + ) + stop_watchdog = getattr(self, "_stop_systemd_watchdog", None) if callable(stop_watchdog): await stop_watchdog() diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index fab84d150a..9dfc5ef643 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -431,6 +431,31 @@ async def _lifespan(app: "FastAPI"): record_boot_fingerprint() + # Hosted Bot rooms belong to the backend process, not to any connected + # Desktop socket. Recovery may need a contended state.db migration, so keep + # it off the lifespan's pre-yield path: Group Chat startup must degrade on + # its own instead of preventing every dashboard/Desktop feature from booting. + from tui_gateway import methods_groups as _hosted_groups + import tui_gateway.server # noqa: F401 + + hosted_room_start_cancel = threading.Event() + + def _start_hosted_rooms() -> None: + try: + _hosted_groups.start_hosted_room_service() + except Exception: + _log.exception("Hosted Group Chat recovery failed during backend startup") + finally: + if hosted_room_start_cancel.is_set(): + _hosted_groups.stop_hosted_room_service(timeout=1.0) + + hosted_room_start_thread = threading.Thread( + target=_start_hosted_rooms, + daemon=True, + name="hosted-room-startup", + ) + hosted_room_start_thread.start() + # Desktop-spawned backends (HERMES_DESKTOP=1) fire cron jobs themselves, # since the app has no gateway running the scheduler. Server `hermes # dashboard` is unaffected — it relies on its own gateway. @@ -473,6 +498,9 @@ async def _lifespan(app: "FastAPI"): try: yield finally: + hosted_room_start_cancel.set() + _hosted_groups.stop_hosted_room_service(timeout=5.0) + hosted_room_start_thread.join(timeout=1.0) if cron_stop is not None: cron_stop.set() pty_reaper_task.cancel() diff --git a/tests/gateway/test_hosted_room_discussion.py b/tests/gateway/test_hosted_room_discussion.py new file mode 100644 index 0000000000..a653be1df0 --- /dev/null +++ b/tests/gateway/test_hosted_room_discussion.py @@ -0,0 +1,708 @@ +"""Behavior tests for deterministic same-gateway Discussion policy.""" + +from __future__ import annotations + +import time +from pathlib import Path + +import pytest + +from gateway import hosted_room_discussion as discussion +from gateway import hosted_room_driver as driver +from gateway import hosted_rooms + + +ROOM_ID = "room-1" +GATEWAY_ID = "gateway-a" +LOCAL_PROFILES = ("research", "build", "review", "ops", "qa", "docs") +MEMBERS = [ + { + "member_id": f"member-{profile}", + "profile": profile, + "handle": profile, + "display_name": profile.title(), + } + for profile in LOCAL_PROFILES[:3] +] + + +@pytest.fixture +def room_db(tmp_path: Path) -> tuple[Path, dict]: + db = tmp_path / "state.db" + room = hosted_rooms.create_room( + db, + room_id=ROOM_ID, + name="Release", + members=MEMBERS, + authority_gateway_id=GATEWAY_ID, + now=1, + ) + return db, room + + +def _events(db: Path) -> list[dict]: + return hosted_rooms.read_events( + db, + room_id=ROOM_ID, + since_seq=0, + limit=hosted_rooms.MAX_LOG_LIMIT, + )["events"] + + +def _append_user( + db: Path, + *, + event_id: str, + text: str, + thread_id: str = "thread-1", +) -> dict: + return hosted_rooms.append_event( + db, + room_id=ROOM_ID, + event_id=event_id, + kind="message.user", + actor={"kind": "user", "id": "local-user"}, + authority_gateway_id=GATEWAY_ID, + authority_epoch=1, + payload={"text": text, "thread_id": thread_id}, + now=time.time(), + ) + + +def _append_publication( + db: Path, + plan: discussion.PublicationPlan, +) -> list[dict]: + return [ + hosted_rooms.append_event( + db, + **event.append_kwargs(ROOM_ID), + now=time.time(), + ) + for event in plan.events + ] + + +def _append_activity( + db: Path, + *, + event_id: str, + discussion_event_id: str, + thread_id: str, +) -> dict: + return hosted_rooms.append_event( + db, + room_id=ROOM_ID, + event_id=event_id, + kind="room.activity", + actor={"kind": "gateway", "id": GATEWAY_ID}, + payload={ + "status": "settled", + "reason_code": "silent_round", + "thread_id": thread_id, + "discussion_event_id": discussion_event_id, + }, + authority_gateway_id=GATEWAY_ID, + authority_epoch=1, + ) + + +def _next_task(room: dict, db: Path) -> discussion.DiscussionTaskPlan: + decision = discussion.plan_next_task( + room, + _events(db), + local_profiles=LOCAL_PROFILES, + ) + assert decision.status == "task", decision + assert decision.task is not None + return decision.task + + +def _settle_next( + room: dict, + db: Path, + *, + text: str, +) -> discussion.DiscussionTaskPlan: + task = _next_task(room, db) + publication = discussion.plan_publication( + room, + _events(db), + task, + status="settled", + result={"text": text}, + local_profiles=LOCAL_PROFILES, + ) + _append_publication(db, publication) + return task + + +def test_deferred_member_allows_next_mentioned_member_and_later_terminal_result( + room_db, +): + db, room = room_db + _append_user(db, event_id="user-1", text="Report.") + first = _next_task(room, db) + deferred = discussion.plan_publication( + room, + _events(db), + first, + status="deferred", + result={"reason": "member_unavailable"}, + execution_generation=1, + local_profiles=LOCAL_PROFILES, + ) + _append_publication(db, deferred) + + second = _next_task(room, db) + assert second.member.member_id != first.member.member_id + + settled = discussion.plan_publication( + room, + _events(db), + first, + status="settled", + result={"text": "Recovered on explicit retry."}, + local_profiles=LOCAL_PROFILES, + ) + _append_publication(db, settled) + decision = discussion.plan_next_task( + room, + _events(db), + local_profiles=LOCAL_PROFILES, + ) + assert decision.status == "task" + assert decision.task is not None + assert decision.task.member.member_id == second.member.member_id + + +def test_distinct_threads_are_planned_fifo_without_skipping(room_db): + db, room = room_db + _append_user(db, event_id="user-1", text="First", thread_id="thread-1") + _append_user(db, event_id="user-2", text="Second", thread_id="thread-2") + + first = _next_task(room, db) + assert first.discussion_event_id == "user-1" + _append_activity( + db, + event_id="activity-1", + discussion_event_id="user-1", + thread_id="thread-1", + ) + second = _next_task(room, db) + assert second.discussion_event_id == "user-2" + + +def test_room_stop_fences_old_work_but_allows_a_later_message(room_db): + db, room = room_db + _append_user(db, event_id="user-1", text="First", thread_id="thread-1") + stop = hosted_rooms.request_room_stop( + db, + room_id=ROOM_ID, + cancel_id="user-stop-1", + expected_gateway_id=str(room["authority_gateway_id"]), + expected_epoch=int(room["authority_epoch"]), + ) + decision = discussion.plan_next_task( + room, + _events(db), + local_profiles=LOCAL_PROFILES, + ) + assert decision.status == "idle" + assert stop["kind"] == "room.stop_requested" + + _append_user(db, event_id="user-2", text="Continue", thread_id="thread-2") + resumed = _next_task(room, db) + assert resumed.discussion_event_id == "user-2" + + +def test_deterministic_task_fits_existing_driver_and_reconstructs_after_restart( + room_db: tuple[Path, dict], +): + db, room = room_db + user = _append_user(db, event_id="user-1", text="Check the release.") + + first = _next_task(room, db) + repeated = _next_task(room, db) + assert first == repeated + assert first.identity.thread_id == "thread-1" + assert first.payload == { + "target_profile": "research", + "prompt": first.payload["prompt"], + "source_event_seq": user["seq"], + } + assert set(first.payload) == {"target_profile", "prompt", "source_event_seq"} + + admitted = driver.admit_task( + db, + first.identity, + payload=first.payload, + clock=time.time, + ) + stored = driver.get_task(db, first.identity) + reconstructed = discussion.reconstruct_task_plan( + room, + _events(db), + stored, + local_profiles=LOCAL_PROFILES, + ) + assert admitted["status"] == "queued" + assert reconstructed == first + + reopened_events = _events(db) + assert ( + discussion.reconstruct_task_plan( + room, + reopened_events, + driver.get_task(db, first.identity), + local_profiles=LOCAL_PROFILES, + ) + == first + ) + + +@pytest.mark.parametrize( + ("text", "expected_profile"), + [ + ("@build please inspect this", "build"), + ("@all inspect this", "research"), + ("@everyone inspect this", "research"), + ("inspect this", "research"), + ("@unknown inspect this", "research"), + ], +) +def test_mentions_select_handles_or_everyone( + room_db: tuple[Path, dict], + text: str, + expected_profile: str, +): + db, room = room_db + _append_user(db, event_id="user-1", text=text) + + assert _next_task(room, db).member.profile == expected_profile + + +def test_member_mention_joins_the_next_round_not_the_current_round( + room_db: tuple[Path, dict], +): + db, room = room_db + _append_user(db, event_id="user-1", text="@research lead this") + + first = _settle_next(room, db, text="@build can add the implementation detail.") + second = _next_task(room, db) + + assert first.member.profile == "research" + assert first.round_index == 0 + assert second.member.profile == "build" + assert second.round_index == 1 + assert "@research lead this" in second.payload["prompt"] + assert "@build can add the implementation detail." in second.payload["prompt"] + + +def test_plain_member_reply_does_not_wake_another_bot_round( + room_db: tuple[Path, dict], +): + db, room = room_db + _append_user(db, event_id="user-1", text="@research answer the user") + _settle_next(room, db, text="The answer is ready for the user.") + + decision = discussion.plan_next_task( + room, + _events(db), + local_profiles=LOCAL_PROFILES, + ) + + assert decision.status == "settled" + assert decision.reason == "silent_round" + + +@pytest.mark.parametrize("value", ["", "pass", "pass.", "(pass)", " ( PASS ). "]) +def test_pass_detection(value: str): + assert discussion.is_pass_text(value) + + +def test_real_text_is_not_a_pass(): + assert not discussion.is_pass_text("I found the issue.") + + +def test_full_pass_round_settles_without_member_messages( + room_db: tuple[Path, dict], +): + db, room = room_db + _append_user(db, event_id="user-1", text="Any concerns?") + + for _member in MEMBERS: + _settle_next(room, db, text="(pass)") + + decision = discussion.plan_next_task( + room, + _events(db), + local_profiles=LOCAL_PROFILES, + ) + assert decision.status == "settled" + assert decision.reason == "silent_round" + assert [event["kind"] for event in _events(db)].count("message.member") == 0 + + +def test_failed_members_advance_the_round_as_silence( + room_db: tuple[Path, dict], +): + db, room = room_db + _append_user(db, event_id="user-1", text="Any concerns?") + + for expected in ("research", "build", "review"): + task = _next_task(room, db) + assert task.member.profile == expected + publication = discussion.plan_publication( + room, + _events(db), + task, + status="failed", + result={"error": f"{expected} unavailable"}, + local_profiles=LOCAL_PROFILES, + ) + assert publication.terminal_kind == "turn.failed" + assert len(publication.events) == 1 + _append_publication(db, publication) + + decision = discussion.plan_next_task( + room, + _events(db), + local_profiles=LOCAL_PROFILES, + ) + assert decision.status == "settled" + assert decision.reason == "silent_round" + + +def test_publication_is_idempotent_and_changed_result_conflicts( + room_db: tuple[Path, dict], +): + db, room = room_db + _append_user(db, event_id="user-1", text="Report.") + task = _next_task(room, db) + publication = discussion.plan_publication( + room, + _events(db), + task, + status="settled", + result={"text": "Ready."}, + local_profiles=LOCAL_PROFILES, + ) + + first = _append_publication(db, publication) + repeated = _append_publication(db, publication) + assert [event["seq"] for event in first] == [event["seq"] for event in repeated] + assert all(event["idempotent"] for event in repeated) + + changed = discussion.plan_publication( + room, + _events(db), + task, + status="settled", + result={"text": "Different."}, + local_profiles=LOCAL_PROFILES, + ) + with pytest.raises(hosted_rooms.EventConflictError): + _append_publication(db, changed) + + +def test_partial_publication_replays_same_effects_before_policy_advances( + room_db: tuple[Path, dict], +): + db, room = room_db + _append_user(db, event_id="user-1", text="Report.") + task = _next_task(room, db) + publication = discussion.plan_publication( + room, + _events(db), + task, + status="settled", + result={"text": "Ready."}, + local_profiles=LOCAL_PROFILES, + ) + + message_effect = publication.events[0] + hosted_rooms.append_event( + db, + **message_effect.append_kwargs(ROOM_ID), + now=time.time(), + ) + assert _next_task(room, db).identity == task.identity + + replayed = discussion.plan_publication( + room, + _events(db), + task, + status="settled", + result={"text": "Ready."}, + local_profiles=LOCAL_PROFILES, + ) + _append_publication(db, replayed) + assert _next_task(room, db).member.profile == "build" + + +def test_watermark_excludes_a_members_old_input_and_own_reply( + room_db: tuple[Path, dict], +): + db, room = room_db + _append_user(db, event_id="user-1", text="Old request.") + first = _settle_next(room, db, text="Old answer.") + watermark = discussion.derive_member_watermarks( + room, + _events(db), + local_profiles=LOCAL_PROFILES, + )[("thread-1", first.member.member_id)] + assert watermark == max( + event["seq"] + for event in _events(db) + if event["kind"] == "message.member" + and event["payload"]["task_id"] == first.identity.task_id + ) + + latest = _append_user(db, event_id="user-2", text="New request.") + next_task = _next_task(room, db) + assert next_task.member.profile == "research" + assert next_task.payload["source_event_seq"] == latest["seq"] + assert "New request." in next_task.payload["prompt"] + assert "Old request." not in next_task.payload["prompt"] + assert "Old answer." not in next_task.payload["prompt"] + + +def test_newer_same_thread_user_event_cancels_a_late_result( + room_db: tuple[Path, dict], +): + db, room = room_db + _append_user(db, event_id="user-1", text="First request.") + stale = _next_task(room, db) + latest = _append_user(db, event_id="user-2", text="Second request.") + + publication = discussion.plan_publication( + room, + _events(db), + stale, + status="settled", + result={"text": "Late stale answer."}, + local_profiles=LOCAL_PROFILES, + ) + assert publication.terminal_kind == "turn.cancelled" + assert [event.kind for event in publication.events] == ["turn.cancelled"] + assert publication.events[0].payload["reason"] == "superseded_by_newer_user_event" + _append_publication(db, publication) + + current = _next_task(room, db) + assert current.payload["source_event_seq"] == latest["seq"] + assert "Second request." in current.payload["prompt"] + + +def test_cross_thread_newer_user_does_not_discard_completed_old_reply( + room_db: tuple[Path, dict], +): + db, room = room_db + _append_user(db, event_id="user-1", text="First request.", thread_id="thread-1") + old = _next_task(room, db) + _append_user(db, event_id="user-2", text="Other topic.", thread_id="thread-2") + + publication = discussion.plan_publication( + room, + _events(db), + old, + status="settled", + result={"text": "Completed first topic."}, + local_profiles=LOCAL_PROFILES, + ) + assert [event.kind for event in publication.events] == [ + "message.member", + "turn.settled", + ] + + +def test_oversized_member_reply_is_truncated_and_next_turn_stays_serviceable( + room_db: tuple[Path, dict], +): + db, room = room_db + _append_user( + db, + event_id="user-large", + text="u" * discussion.MAX_USER_TEXT_BYTES, + ) + first = _next_task(room, db) + publication = discussion.plan_publication( + room, + _events(db), + first, + status="settled", + result={"text": "é" * (discussion.MAX_MEMBER_TEXT_BYTES + 100)}, + local_profiles=LOCAL_PROFILES, + ) + + member_event = next(event for event in publication.events if event.kind == "message.member") + member_text = member_event.payload["text"] + assert len(member_text.encode("utf-8")) <= discussion.MAX_MEMBER_TEXT_BYTES + assert member_text.endswith("share the full result as a file.]") + _append_publication(db, publication) + + followup = _next_task(room, db) + assert len(followup.payload["prompt"].encode("utf-8")) <= driver.MAX_PROMPT_BYTES + assert "Earlier content omitted" in followup.payload["prompt"] + + +def test_three_round_bound(room_db: tuple[Path, dict]): + db, room = room_db + room["members"] = MEMBERS[:2] + _append_user(db, event_id="user-1", text="Discuss.") + + for index in range(6): + task = _next_task(room, db) + peer = "build" if task.member.profile == "research" else "research" + publication = discussion.plan_publication( + room, + _events(db), + task, + status="settled", + result={"text": f"Reply {index}. @{peer}"}, + local_profiles=LOCAL_PROFILES, + ) + _append_publication(db, publication) + + decision = discussion.plan_next_task( + room, + _events(db), + local_profiles=LOCAL_PROFILES, + ) + assert decision.status == "bounded" + assert decision.reason == "max_rounds" + + +def test_ten_message_bound(tmp_path: Path): + db = tmp_path / "state.db" + members = [ + { + "member_id": f"member-{profile}", + "profile": profile, + "handle": profile, + } + for profile in LOCAL_PROFILES + ] + room = hosted_rooms.create_room( + db, + room_id=ROOM_ID, + name="Large", + members=members, + authority_gateway_id=GATEWAY_ID, + now=1, + ) + _append_user(db, event_id="user-1", text="Discuss.") + + for index in range(discussion.MAX_DISCUSSION_MESSAGES): + _settle_next(room, db, text=f"Reply {index}. @everyone") + + decision = discussion.plan_next_task( + room, + _events(db), + local_profiles=LOCAL_PROFILES, + ) + assert decision.status == "bounded" + assert decision.reason == "max_messages" + + +def test_prompt_delta_is_bounded_to_24_message_lines( + room_db: tuple[Path, dict], +): + db, room = room_db + for index in range(30): + _append_user( + db, + event_id=f"user-{index}", + text=f"Message {index}.", + ) + + task = _next_task(room, db) + assert task.payload["prompt"].count("User (user):") == 24 + assert "Message 5." not in task.payload["prompt"] + assert "Message 6." in task.payload["prompt"] + assert "Message 29." in task.payload["prompt"] + + +def test_attachment_payload_is_rejected_by_local_text_only_boundary(): + with pytest.raises(discussion.DiscussionValidationError, match="unknown fields"): + discussion.validate_user_payload({ + "text": "Review.", + "thread_id": "thread-1", + "attachments": [{"name": "notes.txt"}], + }) + + +@pytest.mark.parametrize( + ("members", "match"), + [ + (MEMBERS[:1], "between 2 and 6"), + (MEMBERS + MEMBERS + MEMBERS[:1], "between 2 and 6"), + ( + [MEMBERS[0], {**MEMBERS[1], "profile": "research"}], + "profiles must be unique", + ), + ([MEMBERS[0], {**MEMBERS[1], "handle": "RESEARCH"}], "handles must be unique"), + ( + [MEMBERS[0], {**MEMBERS[1], "member_id": "MEMBER-RESEARCH"}], + "ids must be unique", + ), + ([MEMBERS[0], {**MEMBERS[1], "route": {"mode": "ssh"}}], "cross-gateway"), + ([MEMBERS[0], {**MEMBERS[1], "connectionId": "remote"}], "cross-gateway"), + ([MEMBERS[0], {**MEMBERS[1], "profile": "missing"}], "not local"), + ], +) +def test_malformed_or_remote_roster_is_rejected(members: list[dict], match: str): + with pytest.raises(discussion.DiscussionValidationError, match=match): + discussion.validate_roster(members, local_profiles=LOCAL_PROFILES) + + +@pytest.mark.parametrize( + "payload", + [ + {"text": "hello"}, + {"text": "hello", "thread_id": "thread-1", "images": []}, + {"text": "", "thread_id": "thread-1"}, + {"text": "hello", "thread_id": "../escape"}, + {"text": ["hello"], "thread_id": "thread-1"}, + ], +) +def test_user_payload_is_exact_and_text_only(payload: dict): + with pytest.raises(discussion.DiscussionValidationError): + discussion.validate_user_payload(payload) + + +def test_malformed_log_and_task_reconstruction_fail_closed( + room_db: tuple[Path, dict], +): + db, room = room_db + _append_user(db, event_id="user-1", text="Report.") + _append_user(db, event_id="user-2", text="Report again.") + task = _next_task(room, db) + events = _events(db) + + with pytest.raises(discussion.DiscussionValidationError, match="sequence order"): + discussion.plan_next_task( + room, + list(reversed(events)), + local_profiles=LOCAL_PROFILES, + ) + + malformed = { + "identity": driver.TaskIdentity( + room_id=task.identity.room_id, + task_id="dtask:wrong", + thread_id=task.identity.thread_id, + turn_id=task.identity.turn_id, + ), + "payload": dict(task.payload), + } + with pytest.raises( + discussion.DiscussionReconstructionError, + match="deterministic reconstruction", + ): + discussion.reconstruct_task_plan( + room, + events, + malformed, + local_profiles=LOCAL_PROFILES, + ) diff --git a/tests/gateway/test_hosted_room_driver.py b/tests/gateway/test_hosted_room_driver.py new file mode 100644 index 0000000000..78f6fe9c9d --- /dev/null +++ b/tests/gateway/test_hosted_room_driver.py @@ -0,0 +1,1110 @@ +"""Behavior tests for the hosted-room driver state machine.""" + +from __future__ import annotations + +import sqlite3 +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor + +import pytest + +from gateway import hosted_rooms as rooms +from gateway import hosted_room_driver as driver + + +class FakeClock: + def __init__(self, value: float = 100.0): + self.value = value + + def __call__(self) -> float: + return self.value + + def advance(self, seconds: float) -> None: + self.value += seconds + + +def _identity(task_id: str = "task-1", *, turn_id: str = "turn-1"): + return driver.TaskIdentity( + room_id="room-1", + task_id=task_id, + thread_id="thread-1", + turn_id=turn_id, + ) + + +def _payload( + *, + target_profile: str = "ops", + prompt: str = "Inspect the release candidate.", + source_event_seq: int = 1, +): + return { + "target_profile": target_profile, + "prompt": prompt, + "source_event_seq": source_event_seq, + } + + +@pytest.fixture +def db(tmp_path): + path = tmp_path / "state.db" + rooms.create_room( + path, + room_id="room-1", + name="Release room", + members=[{"profile": "ops", "handle": "ops"}], + authority_gateway_id="gateway-a", + now=90, + ) + return path + + +def _lease( + db, + clock, + *, + gateway="gateway-a", + authority_epoch=1, + process="process-a", + ttl=30, +): + return driver.acquire_lease( + db, + room_id="room-1", + gateway_id=gateway, + authority_epoch=authority_epoch, + process_generation=process, + ttl_seconds=ttl, + clock=clock, + ) + + +def _admit(db, identity, clock, *, payload=None): + return driver.admit_task( + db, + identity, + payload=_payload() if payload is None else payload, + clock=clock, + ) + + +def _open_driver_schema(path: str) -> int: + return len(driver.list_tasks(path, room_id="room-1")) + + +def test_two_contenders_have_one_winner(db): + clock = FakeClock() + + def contend(process): + try: + return _lease(db, clock, process=process) + except driver.LeaseHeldError: + return None + + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(contend, ["process-a", "process-b"])) + + winners = [result for result in results if result is not None] + assert len(winners) == 1 + assert winners[0].lease_generation == 1 + + +def test_expiry_allows_reclaim_and_fences_stale_renew_and_release(db): + clock = FakeClock() + first = _lease(db, clock, ttl=5) + + clock.advance(5) + second = _lease(db, clock, process="process-b") + + assert second.reclaimed is True + assert second.lease_generation == first.lease_generation + 1 + with pytest.raises(driver.StaleLeaseError): + driver.renew_lease(db, first, ttl_seconds=30, clock=clock) + with pytest.raises(driver.StaleLeaseError): + driver.release_lease(db, first, clock=clock) + + +def test_nonexistent_and_disbanded_rooms_cannot_lease_or_admit(db): + clock = FakeClock() + missing = driver.TaskIdentity("missing-room", "task", "thread", "turn") + + with pytest.raises(driver.RoomUnavailableError, match="does not exist"): + driver.acquire_lease( + db, + room_id="missing-room", + gateway_id="gateway-a", + authority_epoch=1, + process_generation="process-a", + ttl_seconds=30, + clock=clock, + ) + with pytest.raises(driver.RoomUnavailableError, match="does not exist"): + _admit(db, missing, clock) + + rooms.disband_room( + db, + room_id="room-1", + expected_gateway_id="gateway-a", + expected_epoch=1, + now=clock(), + ) + with pytest.raises(driver.RoomUnavailableError, match="disbanded"): + _lease(db, clock) + with pytest.raises(driver.RoomUnavailableError, match="disbanded"): + _admit(db, _identity(), clock) + + +@pytest.mark.parametrize( + ("gateway", "authority_epoch"), + [("gateway-b", 1), ("gateway-a", 2)], +) +def test_acquire_requires_current_room_authority(db, gateway, authority_epoch): + with pytest.raises(driver.StaleLeaseError, match="authority changed"): + _lease( + db, + FakeClock(), + gateway=gateway, + authority_epoch=authority_epoch, + ) + + +def test_same_process_acquire_and_release_are_idempotent(db): + clock = FakeClock() + first = _lease(db, clock) + repeated = _lease(db, clock) + + assert repeated.lease_generation == first.lease_generation + released = driver.release_lease(db, repeated, clock=clock) + released_again = driver.release_lease(db, repeated, clock=clock) + + assert released["idempotent"] is False + assert released_again["idempotent"] is True + + +def test_renew_extends_only_the_current_lease_generation(db): + clock = FakeClock() + lease = _lease(db, clock, ttl=5) + + clock.advance(2) + renewed = driver.renew_lease(db, lease, ttl_seconds=20, clock=clock) + + assert renewed.lease_generation == lease.lease_generation + assert renewed.expires_at == 122 + + +def test_authority_transfer_fences_lease_and_late_settlement(db): + clock = FakeClock() + identity = _identity() + queued = _identity("task-2", turn_id="turn-2") + old_lease = _lease(db, clock) + _admit(db, identity, clock) + _admit(db, queued, clock) + old_attempt = driver.start_task( + db, + identity, + old_lease, + expected_cancel_generation=0, + clock=clock, + ) + + rooms.claim_authority( + db, + room_id="room-1", + expected_gateway_id="gateway-a", + expected_epoch=1, + new_gateway_id="gateway-b", + event_id="claim-gateway-b", + now=clock(), + ) + + with pytest.raises(driver.StaleLeaseError, match="authority changed"): + driver.renew_lease(db, old_lease, ttl_seconds=30, clock=clock) + with pytest.raises(driver.StaleLeaseError, match="authority changed"): + driver.start_task( + db, + queued, + old_lease, + expected_cancel_generation=0, + clock=clock, + ) + with pytest.raises(driver.StaleLeaseError, match="authority changed"): + driver.recover_room(db, old_lease, clock=clock) + with pytest.raises(driver.StaleLeaseError, match="authority changed"): + driver.settle_task( + db, + old_attempt, + settlement_id="late-settlement", + status="settled", + result={"text": "late"}, + clock=clock, + ) + + new_lease = _lease( + db, + clock, + gateway="gateway-b", + authority_epoch=2, + process="process-b", + ) + recovery = driver.recover_room(db, new_lease, clock=clock) + assert new_lease.lease_generation == old_lease.lease_generation + 1 + assert recovery["indeterminate"] == [identity] + assert recovery["queued"] == [queued] + + +def test_room_disband_fences_active_lease_operations(db): + clock = FakeClock() + identity = _identity() + lease = _lease(db, clock) + _admit(db, identity, clock) + rooms.disband_room( + db, + room_id="room-1", + expected_gateway_id="gateway-a", + expected_epoch=1, + now=clock(), + ) + + with pytest.raises(driver.RoomUnavailableError, match="disbanded"): + driver.renew_lease(db, lease, ttl_seconds=30, clock=clock) + with pytest.raises(driver.RoomUnavailableError, match="disbanded"): + driver.start_task( + db, + identity, + lease, + expected_cancel_generation=0, + clock=clock, + ) + with pytest.raises(driver.RoomUnavailableError, match="disbanded"): + driver.recover_room(db, lease, clock=clock) + with pytest.raises(driver.RoomUnavailableError, match="disbanded"): + driver.release_lease(db, lease, clock=clock) + + +def test_task_admission_is_idempotent_and_identity_conflicts_fail(db): + clock = FakeClock() + identity = _identity() + + first = _admit(db, identity, clock) + repeated = _admit(db, identity, clock) + + assert first["status"] == "queued" + assert repeated["idempotent"] is True + + with pytest.raises(driver.TaskConflictError): + driver.admit_task( + db, + driver.TaskIdentity( + room_id="room-1", + task_id="task-1", + thread_id="thread-other", + turn_id="turn-other", + ), + payload=_payload(), + clock=clock, + ) + + with pytest.raises(driver.TaskConflictError, match="different payload"): + _admit( + db, + identity, + clock, + payload=_payload(prompt="A different immutable prompt."), + ) + with pytest.raises(driver.TaskConflictError): + driver.admit_task( + db, + driver.TaskIdentity( + room_id="room-1", + task_id="task-other", + thread_id="thread-1", + turn_id="turn-1", + ), + payload=_payload(), + clock=clock, + ) + + +def test_concurrent_task_start_has_one_winner(db): + clock = FakeClock() + identity = _identity() + lease = _lease(db, clock) + _admit(db, identity, clock) + + def start(_): + try: + return driver.start_task( + db, + identity, + lease, + expected_cancel_generation=0, + clock=clock, + ) + except driver.InvalidTaskTransitionError: + return None + + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(start, range(2))) + + winners = [result for result in results if result is not None] + assert len(winners) == 1 + assert winners[0].execution_generation == 1 + + +def test_stale_lease_cannot_start_or_commit_task(db): + clock = FakeClock() + identity = _identity() + first = _lease(db, clock, ttl=5) + _admit(db, identity, clock) + attempt = driver.start_task( + db, + identity, + first, + expected_cancel_generation=0, + clock=clock, + ) + + clock.advance(5) + second = _lease(db, clock, process="process-b") + driver.recover_room(db, second, clock=clock) + + with pytest.raises(driver.StaleLeaseError): + driver.settle_task( + db, + attempt, + settlement_id="settlement-old", + status="settled", + result={"text": "late"}, + clock=clock, + ) + assert driver.get_task(db, identity)["status"] == "indeterminate" + + +@pytest.mark.parametrize("status", ["settled", "failed"]) +def test_terminal_settlement_is_idempotent(db, status): + clock = FakeClock() + identity = _identity() + lease = _lease(db, clock) + _admit(db, identity, clock) + attempt = driver.start_task( + db, + identity, + lease, + expected_cancel_generation=0, + clock=clock, + ) + + first = driver.settle_task( + db, + attempt, + settlement_id="settlement-1", + status=status, + result={"text": "done"}, + clock=clock, + ) + repeated = driver.settle_task( + db, + attempt, + settlement_id="settlement-1", + status=status, + result={"text": "done"}, + clock=clock, + ) + + assert first["status"] == status + assert repeated["idempotent"] is True + with pytest.raises(driver.TaskConflictError): + driver.settle_task( + db, + attempt, + settlement_id="settlement-2", + status=status, + result={"text": "changed"}, + clock=clock, + ) + + +def test_cancellation_fences_late_success(db): + clock = FakeClock() + identity = _identity() + lease = _lease(db, clock) + _admit(db, identity, clock) + attempt = driver.start_task( + db, + identity, + lease, + expected_cancel_generation=0, + clock=clock, + ) + + stopping = driver.begin_task_cancel( + db, + identity, + cancel_id="cancel-1", + expected_cancel_generation=0, + clock=clock, + ) + cancelled = driver.complete_task_cancel( + db, + identity, + cancel_id="cancel-1", + expected_cancel_generation=1, + clock=clock, + ) + repeated = driver.complete_task_cancel( + db, + identity, + cancel_id="cancel-1", + expected_cancel_generation=1, + clock=clock, + ) + + assert stopping["status"] == "stopping" + assert cancelled["status"] == "cancelled" + assert cancelled["cancel_generation"] == 1 + assert repeated["idempotent"] is True + with pytest.raises(driver.StaleTaskError): + driver.settle_task( + db, + attempt, + settlement_id="late-success", + status="settled", + result={"text": "too late"}, + clock=clock, + ) + + +def test_release_fails_closed_while_its_task_is_running(db): + clock = FakeClock() + identity = _identity() + lease = _lease(db, clock) + _admit(db, identity, clock) + driver.start_task( + db, + identity, + lease, + expected_cancel_generation=0, + clock=clock, + ) + + with pytest.raises( + driver.InvalidTaskTransitionError, + match="tasks are running", + ): + driver.release_lease(db, lease, clock=clock) + assert driver.get_task(db, identity)["status"] == "running" + + driver.begin_task_cancel( + db, + identity, + cancel_id="cancel-before-release", + expected_cancel_generation=0, + clock=clock, + ) + driver.complete_task_cancel( + db, + identity, + cancel_id="cancel-before-release", + expected_cancel_generation=1, + clock=clock, + ) + assert driver.release_lease(db, lease, clock=clock)["idempotent"] is False + + +def test_restart_recovery_never_requeues_indeterminate_work(db): + clock = FakeClock() + running = _identity() + queued = _identity("task-2", turn_id="turn-2") + first = _lease(db, clock, ttl=5) + _admit(db, running, clock) + _admit(db, queued, clock) + driver.start_task( + db, + running, + first, + expected_cancel_generation=0, + clock=clock, + ) + + with pytest.raises(driver.LeaseHeldError): + _lease(db, clock, gateway="gateway-a", process="new-process") + + clock.advance(5) + recovered_lease = _lease( + db, + clock, + gateway="gateway-a", + process="new-process", + ) + recovery = driver.recover_room(db, recovered_lease, clock=clock) + repeated = driver.recover_room(db, recovered_lease, clock=clock) + + assert recovery == {"queued": [queued], "indeterminate": [running]} + assert repeated == recovery + with pytest.raises(driver.InvalidTaskTransitionError): + driver.start_task( + db, + running, + recovered_lease, + expected_cancel_generation=0, + clock=clock, + ) + assert [task["status"] for task in driver.list_tasks(db, room_id="room-1")] == [ + "indeterminate", + "queued", + ] + + +def test_recovery_is_required_before_starting_later_work(db): + clock = FakeClock() + running = _identity() + queued = _identity("task-2", turn_id="turn-2") + first = _lease(db, clock, ttl=5) + _admit(db, running, clock, payload=_payload(source_event_seq=1)) + _admit(db, queued, clock, payload=_payload(source_event_seq=2)) + driver.start_task( + db, + running, + first, + expected_cancel_generation=0, + clock=clock, + ) + clock.advance(5) + recovered = _lease(db, clock, process="new-process") + + with pytest.raises( + driver.InvalidTaskTransitionError, + match="recovery must resolve", + ): + driver.start_task( + db, + queued, + recovered, + expected_cancel_generation=0, + clock=clock, + ) + + +def test_current_lease_can_commit_verified_indeterminate_receipt(db): + clock = FakeClock() + running = _identity() + queued = _identity("task-2", turn_id="turn-2") + first = _lease(db, clock, ttl=5) + _admit(db, running, clock, payload=_payload(source_event_seq=1)) + _admit(db, queued, clock, payload=_payload(source_event_seq=2)) + attempt = driver.start_task( + db, + running, + first, + expected_cancel_generation=0, + clock=clock, + ) + clock.advance(5) + recovered = _lease(db, clock, process="new-process") + driver.recover_room(db, recovered, clock=clock) + + settled = driver.resolve_indeterminate_task( + db, + running, + recovered, + expected_execution_generation=attempt.execution_generation, + expected_cancel_generation=attempt.cancel_generation, + settlement_id="recovered-receipt", + status="settled", + result={"text": "recovered"}, + clock=clock, + ) + next_attempt = driver.start_task( + db, + queued, + recovered, + expected_cancel_generation=0, + clock=clock, + ) + + assert settled["status"] == "settled" + assert next_attempt.execution_generation == 1 + + +def test_indeterminate_retry_is_explicit_and_advances_execution_generation(db): + clock = FakeClock() + identity = _identity() + first = _lease(db, clock, ttl=5) + _admit(db, identity, clock) + original = driver.start_task( + db, + identity, + first, + expected_cancel_generation=0, + clock=clock, + ) + clock.advance(5) + recovered = _lease(db, clock, process="new-process") + driver.recover_room(db, recovered, clock=clock) + + requeued = driver.requeue_indeterminate_task( + db, + identity, + recovered, + expected_execution_generation=original.execution_generation, + expected_cancel_generation=original.cancel_generation, + clock=clock, + ) + retried = driver.start_task( + db, + identity, + recovered, + expected_cancel_generation=0, + clock=clock, + ) + + assert requeued["status"] == "queued" + assert retried.execution_generation == original.execution_generation + 1 + + +def test_indeterminate_task_can_be_deferred_retried_and_cancelled(db): + clock = FakeClock() + identity = _identity() + first = _lease(db, clock, ttl=5) + _admit(db, identity, clock) + original = driver.start_task( + db, + identity, + first, + expected_cancel_generation=0, + clock=clock, + ) + clock.advance(5) + recovered = _lease(db, clock, process="new-process", ttl=5) + driver.recover_room(db, recovered, clock=clock) + + deferred = driver.defer_indeterminate_task( + db, + identity, + recovered, + expected_execution_generation=original.execution_generation, + expected_cancel_generation=original.cancel_generation, + reason="member_unavailable", + clock=clock, + ) + repeated = driver.defer_indeterminate_task( + db, + identity, + recovered, + expected_execution_generation=original.execution_generation, + expected_cancel_generation=original.cancel_generation, + reason="member_unavailable", + clock=clock, + ) + requeued = driver.requeue_deferred_task( + db, + identity, + recovered, + expected_execution_generation=original.execution_generation, + expected_cancel_generation=original.cancel_generation, + clock=clock, + ) + retried = driver.start_task( + db, + identity, + recovered, + expected_cancel_generation=0, + clock=clock, + ) + + assert deferred["status"] == "deferred" + assert deferred["result"] == { + "reason": "member_unavailable", + "retryable": True, + } + assert repeated["idempotent"] is True + assert requeued["status"] == "queued" + assert retried.execution_generation == original.execution_generation + 1 + + clock.advance(5) + next_lease = _lease(db, clock, process="third-process") + driver.recover_room(db, next_lease, clock=clock) + driver.defer_indeterminate_task( + db, + identity, + next_lease, + expected_execution_generation=retried.execution_generation, + expected_cancel_generation=retried.cancel_generation, + reason="member_unavailable", + clock=clock, + ) + cancelled = driver.cancel_task( + db, + identity, + cancel_id="cancel-deferred", + expected_cancel_generation=0, + clock=clock, + ) + assert cancelled["status"] == "cancelled" + + +def test_state_survives_sqlite_reopen_and_concurrent_duplicate_admission(db): + clock = FakeClock() + identity = _identity() + + def admit(_): + return _admit(db, identity, clock) + + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(admit, range(8))) + + assert sum(not result["idempotent"] for result in results) == 1 + with sqlite3.connect(db) as conn: + count = conn.execute( + "SELECT COUNT(*) FROM hosted_room_driver_tasks" + ).fetchone()[0] + assert count == 1 + reopened = driver.get_task(db, identity) + listed = driver.list_tasks(db, room_id="room-1") + assert reopened["identity"] == identity + assert reopened["payload"] == _payload() + assert listed[0]["payload"] == _payload() + + +def test_prune_removes_only_old_published_terminal_tasks(db): + clock = FakeClock() + lease = _lease(db, clock) + published = _identity("task-published", turn_id="turn-published") + unpublished = _identity("task-unpublished", turn_id="turn-unpublished") + for identity in (published, unpublished): + _admit(db, identity, clock) + attempt = driver.start_task( + db, + identity, + lease, + expected_cancel_generation=0, + clock=clock, + ) + driver.settle_task( + db, + attempt, + settlement_id=f"result:{identity.task_id}", + status="settled", + result={"text": "done"}, + clock=clock, + ) + + with sqlite3.connect(db) as conn: + conn.execute( + """CREATE TABLE hosted_room_policy_publications ( + room_id TEXT NOT NULL, + task_id TEXT NOT NULL, + kind TEXT NOT NULL, + execution_generation INTEGER NOT NULL DEFAULT 0, + seq INTEGER NOT NULL, + PRIMARY KEY(room_id, task_id, kind, execution_generation) + )""" + ) + conn.execute( + """INSERT INTO hosted_room_policy_publications + VALUES ('room-1', 'task-published', 'turn.settled', 0, 3)""" + ) + + clock.advance(driver.TERMINAL_TASK_RETENTION_SECONDS + 1) + assert ( + driver.prune_published_terminal_tasks( + db, + room_id="room-1", + clock=clock, + ) + == 1 + ) + assert [ + task["identity"].task_id for task in driver.list_tasks(db, room_id="room-1") + ] == ["task-unpublished"] + + +def test_unpublished_legacy_driver_schema_fails_closed(db): + conn = sqlite3.connect(db) + try: + conn.execute("DROP TABLE IF EXISTS hosted_room_driver_tasks") + conn.execute("DROP TABLE IF EXISTS hosted_room_driver_leases") + conn.execute( + """CREATE TABLE hosted_room_driver_leases ( + room_id TEXT PRIMARY KEY, + gateway_id TEXT NOT NULL, + process_generation TEXT NOT NULL, + lease_generation INTEGER NOT NULL, + expires_at REAL NOT NULL, + acquired_at REAL NOT NULL, + updated_at REAL NOT NULL, + released_at REAL + )""" + ) + conn.execute( + """CREATE TABLE hosted_room_driver_tasks ( + room_id TEXT NOT NULL, + task_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + status TEXT NOT NULL, + execution_generation INTEGER NOT NULL, + cancel_generation INTEGER NOT NULL, + run_gateway_id TEXT, + run_process_generation TEXT, + run_lease_generation INTEGER, + cancel_id TEXT, + settlement_id TEXT, + settlement_status TEXT, + result_json TEXT, + created_at REAL NOT NULL, + updated_at REAL NOT NULL, + started_at REAL, + terminal_at REAL, + indeterminate_at REAL, + PRIMARY KEY (room_id, task_id), + UNIQUE (room_id, thread_id, turn_id) + )""" + ) + conn.execute( + """INSERT INTO hosted_room_driver_leases + VALUES ('room-1', 'gateway-a', 'old-process', 1, + 200, 100, 100, NULL)""" + ) + conn.execute( + """INSERT INTO hosted_room_driver_tasks + (room_id, task_id, thread_id, turn_id, status, + execution_generation, cancel_generation, + run_gateway_id, run_process_generation, run_lease_generation, + created_at, updated_at, started_at) + VALUES ('room-1', 'task-1', 'thread-1', 'turn-1', 'running', + 1, 0, 'gateway-a', 'old-process', 1, 100, 100, 100)""" + ) + conn.commit() + finally: + conn.close() + + with pytest.raises(driver.DriverStateError, match="unsupported unpublished"): + driver.get_task(db, _identity()) + + +def test_pre_stopping_schema_is_migrated_without_losing_tasks(db): + clock = FakeClock() + identity = _identity() + _admit(db, identity, clock) + + with sqlite3.connect(db) as conn: + current_sql = conn.execute( + "SELECT sql FROM sqlite_master WHERE name='hosted_room_driver_tasks'" + ).fetchone()[0] + old_sql = current_sql.replace(", 'stopping'", "") + conn.execute("DROP INDEX idx_hosted_room_driver_tasks_status") + conn.execute( + "ALTER TABLE hosted_room_driver_tasks RENAME TO hosted_room_driver_tasks_current" + ) + conn.execute(old_sql) + columns = ", ".join(driver._TASK_COLUMN_ORDER) + conn.execute( + f"""INSERT INTO hosted_room_driver_tasks ({columns}) + SELECT {columns} FROM hosted_room_driver_tasks_current""" + ) + conn.execute("DROP TABLE hosted_room_driver_tasks_current") + conn.execute( + """CREATE INDEX idx_hosted_room_driver_tasks_status + ON hosted_room_driver_tasks( + room_id, status, source_event_seq, created_at, task_id + )""" + ) + + assert driver.get_task(db, identity)["status"] == "queued" + lease = _lease(db, clock) + attempt = driver.start_task( + db, + identity, + lease, + expected_cancel_generation=0, + clock=clock, + ) + stopping = driver.begin_task_cancel( + db, + identity, + cancel_id="cancel-after-upgrade", + expected_cancel_generation=attempt.cancel_generation, + clock=clock, + ) + + assert stopping["status"] == "stopping" + with sqlite3.connect(db) as conn: + table_sql = conn.execute( + "SELECT sql FROM sqlite_master WHERE name='hosted_room_driver_tasks'" + ).fetchone()[0] + assert "'stopping'" in table_sql + + +def test_pre_deferred_schema_is_migrated_without_losing_tasks(db): + clock = FakeClock() + identity = _identity() + _admit(db, identity, clock) + + with sqlite3.connect(db) as conn: + current_sql = conn.execute( + "SELECT sql FROM sqlite_master WHERE name='hosted_room_driver_tasks'" + ).fetchone()[0] + old_sql = current_sql.replace(", 'deferred'", "") + conn.execute("DROP INDEX idx_hosted_room_driver_tasks_status") + conn.execute( + "ALTER TABLE hosted_room_driver_tasks RENAME TO hosted_room_driver_tasks_current" + ) + conn.execute(old_sql) + columns = ", ".join(driver._TASK_COLUMN_ORDER) + conn.execute( + f"""INSERT INTO hosted_room_driver_tasks ({columns}) + SELECT {columns} FROM hosted_room_driver_tasks_current""" + ) + conn.execute("DROP TABLE hosted_room_driver_tasks_current") + conn.execute( + """CREATE INDEX idx_hosted_room_driver_tasks_status + ON hosted_room_driver_tasks( + room_id, status, source_event_seq, created_at, task_id + )""" + ) + + assert driver.get_task(db, identity)["status"] == "queued" + with sqlite3.connect(db) as conn: + table_sql = conn.execute( + "SELECT sql FROM sqlite_master WHERE name='hosted_room_driver_tasks'" + ).fetchone()[0] + assert "'deferred'" in table_sql + + +def test_first_schema_creation_is_safe_across_processes(db): + with sqlite3.connect(db) as conn: + conn.execute("DROP TABLE IF EXISTS hosted_room_driver_tasks") + conn.execute("DROP TABLE IF EXISTS hosted_room_driver_leases") + conn.commit() + + with ProcessPoolExecutor(max_workers=4) as pool: + results = list(pool.map(_open_driver_schema, [str(db)] * 4)) + + assert results == [0, 0, 0, 0] + + +def test_tasks_follow_source_event_order_not_admission_time(db): + clock = FakeClock() + later = _identity("task-2", turn_id="turn-2") + earlier = _identity("task-1", turn_id="turn-1") + _admit(db, later, clock, payload=_payload(source_event_seq=2)) + _admit(db, earlier, clock, payload=_payload(source_event_seq=1)) + lease = _lease(db, clock) + + assert [task["identity"] for task in driver.list_tasks(db, room_id="room-1")] == [ + earlier, + later, + ] + with pytest.raises(driver.InvalidTaskTransitionError, match="event order"): + driver.start_task( + db, + later, + lease, + expected_cancel_generation=0, + clock=clock, + ) + assert ( + driver.start_task( + db, + earlier, + lease, + expected_cancel_generation=0, + clock=clock, + ).identity + == earlier + ) + + +def test_payload_digest_is_verified_on_read(db): + identity = _identity() + _admit(db, identity, FakeClock()) + with sqlite3.connect(db) as conn: + conn.execute( + """UPDATE hosted_room_driver_tasks + SET payload_json=REPLACE(payload_json, 'Inspect', 'Replace') + WHERE room_id=? AND task_id=?""", + (identity.room_id, identity.task_id), + ) + conn.commit() + + with pytest.raises(driver.TaskConflictError, match="integrity"): + driver.get_task(db, identity) + + +@pytest.mark.parametrize( + ("payload", "match"), + [ + ({"target_profile": "ops", "prompt": "hello"}, "missing payload fields"), + ( + {**_payload(), "unexpected": True}, + "unknown payload fields", + ), + (_payload(target_profile="bad profile"), "invalid target_profile"), + (_payload(prompt=" "), "prompt must not be empty"), + (_payload(prompt="x" * (driver.MAX_PROMPT_BYTES + 1)), "prompt is too large"), + (_payload(source_event_seq=0), "source_event_seq"), + (_payload(source_event_seq=True), "source_event_seq"), + ], +) +def test_invalid_task_payload_is_rejected(db, payload, match): + with pytest.raises(driver.DriverValidationError, match=match): + _admit(db, _identity(), FakeClock(), payload=payload) + + +@pytest.mark.parametrize( + ("factory", "match"), + [ + ( + lambda: driver.TaskIdentity("bad room", "task", "thread", "turn"), + "invalid room_id", + ), + ( + lambda: driver.TaskIdentity("room", "", "thread", "turn"), + "invalid task_id", + ), + ], +) +def test_invalid_task_identity_is_rejected(factory, match): + with pytest.raises(driver.DriverValidationError, match=match): + factory() + + +def test_invalid_lease_clock_ttl_and_settlement_schema_are_rejected(db): + clock = FakeClock() + + with pytest.raises(driver.DriverValidationError, match="ttl_seconds"): + _lease(db, clock, ttl=0) + with pytest.raises(driver.DriverValidationError, match="clock"): + _lease(db, lambda: float("nan")) + with pytest.raises(driver.DriverValidationError, match="expiry"): + _lease(db, FakeClock(1e308), ttl=1e308) + + identity = _identity() + lease = _lease(db, clock) + _admit(db, identity, clock) + attempt = driver.start_task( + db, + identity, + lease, + expected_cancel_generation=0, + clock=clock, + ) + with pytest.raises(driver.DriverValidationError, match="JSON-serializable"): + driver.settle_task( + db, + attempt, + settlement_id="settlement-1", + status="settled", + result={"bad": object()}, + clock=clock, + ) + + +def test_renewal_never_shortens_an_active_lease(db): + clock = FakeClock() + lease = _lease(db, clock, ttl=30) + clock.advance(1) + + renewed = driver.renew_lease(db, lease, ttl_seconds=2, clock=clock) + + assert renewed.expires_at == lease.expires_at diff --git a/tests/gateway/test_hosted_room_gateway_lifecycle.py b/tests/gateway/test_hosted_room_gateway_lifecycle.py new file mode 100644 index 0000000000..fe692d41cc --- /dev/null +++ b/tests/gateway/test_hosted_room_gateway_lifecycle.py @@ -0,0 +1,234 @@ +"""Messaging-gateway ownership tests for the hosted Group Chat worker.""" + +from __future__ import annotations + +import asyncio +import threading +import time +from types import SimpleNamespace + +import pytest + +from gateway import hosted_room_driver, hosted_rooms +from gateway.run import GatewayRunner +from tui_gateway.hosted_room_service import HostedRoomService + + +class _RPC: + def __init__(self) -> None: + self.sessions = {} + self.submits = [] + + def resolve_exact(self, *, profile, title, source): + del source + return self.sessions.get((profile, title)) + + def create(self, *, profile, title, source): + del source + session = {"session_id": f"{profile}-session", "title": title} + self.sessions[(profile, title)] = session + return session + + def resume(self, *, profile, session_id, source): + del profile, source + return {"session_id": session_id} + + def submit(self, **kwargs): + self.submits.append(kwargs["profile"]) + kwargs["on_terminal"]({ + "status": "settled", + "text": f"reply from {kwargs['profile']}", + }) + return {"accepted": True} + + def history(self, **kwargs): + del kwargs + return [] + + def info(self, **kwargs): + del kwargs + return {"active": False, "task_id": None} + + def interrupt(self, **kwargs): + del kwargs + raise AssertionError("gateway lifecycle must not interrupt room work") + + +def _server(): + return SimpleNamespace(_methods={}, _sessions={}, _sessions_lock=threading.Lock()) + + +def _service(db_path, *, profiles=("default",)): + service = HostedRoomService(_server(), db_path=db_path) + rpc = _RPC() + service.rpc = rpc + service.runtime.rpc = rpc + service.local_profiles = lambda: profiles + return service, rpc + + +def _wait_for(predicate, timeout=3.0): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.01) + raise AssertionError("condition did not settle before timeout") + + +@pytest.mark.asyncio +async def test_messaging_gateway_supervisor_starts_without_dashboard(monkeypatch): + from tui_gateway import methods_groups + + state = {"running": False, "starts": 0} + + class Runtime: + def status(self): + return {"running": state["running"], "stopping": False} + + service = SimpleNamespace(runtime=Runtime()) + + def get_service(): + return service if state["running"] else None + + def start_service(): + state["starts"] += 1 + state["running"] = True + return service + + monkeypatch.setattr(methods_groups, "get_hosted_room_service", get_service) + monkeypatch.setattr(methods_groups, "start_hosted_room_service", start_service) + + runner = GatewayRunner.__new__(GatewayRunner) + started = await runner._ensure_hosted_room_worker() + assert started is service + assert state == {"running": True, "starts": 1} + + # A dead child is restarted, while a healthy one is left alone. + await runner._ensure_hosted_room_worker() + assert state["starts"] == 1 + state["running"] = False + await runner._ensure_hosted_room_worker() + assert state["starts"] == 2 + + +@pytest.mark.asyncio +async def test_dead_room_worker_is_restarted_by_gateway_task_supervision(monkeypatch): + from tui_gateway import methods_groups + + starts = {"count": 0} + + def fail_start(): + starts["count"] += 1 + raise RuntimeError("worker unavailable") + + monkeypatch.setattr(methods_groups, "get_hosted_room_service", lambda: None) + monkeypatch.setattr(methods_groups, "start_hosted_room_service", fail_start) + monkeypatch.setattr(GatewayRunner, "_MAX_SUPERVISED_RESTARTS", 1) + monkeypatch.setattr( + GatewayRunner, + "_supervised_backoff", + staticmethod(lambda _attempt: 0), + ) + + runner = GatewayRunner.__new__(GatewayRunner) + runner._running = True + runner._background_tasks = set() + runner._spawn_supervised( + lambda: runner._hosted_room_worker_watcher(interval=0), + "hosted_room_worker", + ) + + for _ in range(200): + if starts["count"] == 2 and not runner._background_tasks: + break + await asyncio.sleep(0.01) + runner._running = False + + assert starts["count"] == 2 + assert runner._background_tasks == set() + + +def test_gateway_restart_resumes_queued_room_for_multiplexed_profile(tmp_path): + db = tmp_path / "state.db" + first, _ = _service(db, profiles=("default", "ops")) + first.create_room( + room_id="room-1", + name="Release room", + members=[ + { + "member_id": "default", + "profile": "default", + "handle": "hermes", + }, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ], + ) + first.send( + room_id="room-1", + event_id="user-1", + payload={"text": "@ops inspect", "thread_id": "thread-1"}, + ) + assert ( + len(hosted_room_driver.list_tasks(db, room_id="room-1", status="queued")) == 1 + ) + + resumed, rpc = _service(db, profiles=("default", "ops")) + resumed.start() + try: + _wait_for( + lambda: any( + event["kind"] == "message.member" + for event in hosted_rooms.read_events( + db, room_id="room-1", since_seq=0 + )["events"] + ) + ) + finally: + assert resumed.stop(timeout=1.0) + + assert rpc.submits == ["ops"] + assert hosted_room_driver.list_tasks(db, room_id="room-1", status="settled") + + +def test_dashboard_and_gateway_workers_share_one_fenced_execution_owner(tmp_path): + db = tmp_path / "state.db" + gateway, gateway_rpc = _service(db, profiles=("default", "ops")) + dashboard, dashboard_rpc = _service(db, profiles=("default", "ops")) + gateway.create_room( + room_id="room-1", + name="Release room", + members=[ + { + "member_id": "default", + "profile": "default", + "handle": "hermes", + }, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ], + ) + gateway.send( + room_id="room-1", + event_id="user-1", + payload={"text": "@ops inspect", "thread_id": "thread-1"}, + ) + + gateway.start() + dashboard.start() + try: + _wait_for( + lambda: any( + event["kind"] == "message.member" + for event in hosted_rooms.read_events( + db, room_id="room-1", since_seq=0 + )["events"] + ) + ) + time.sleep(0.05) + finally: + assert gateway.stop(timeout=1.0) + assert dashboard.stop(timeout=1.0) + + assert len(gateway_rpc.submits) + len(dashboard_rpc.submits) == 1 + events = hosted_rooms.read_events(db, room_id="room-1", since_seq=0)["events"] + assert sum(event["kind"] == "message.member" for event in events) == 1 diff --git a/tests/gateway/test_hosted_room_local_boundary.py b/tests/gateway/test_hosted_room_local_boundary.py new file mode 100644 index 0000000000..46c577e9f9 --- /dev/null +++ b/tests/gateway/test_hosted_room_local_boundary.py @@ -0,0 +1,38 @@ +"""Dependency boundary for the same-gateway hosted-room backend.""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +LOCAL_ROOM_MODULES = ( + "gateway/hosted_room_discussion.py", + "gateway/hosted_room_driver.py", + "gateway/hosted_room_policy_checkpoint.py", + "tui_gateway/hosted_room_driver.py", + "tui_gateway/hosted_room_service.py", + "tui_gateway/hosted_room_server_rpc.py", + "tui_gateway/methods_groups.py", +) +FORBIDDEN_SURFACES = ( + "apps/desktop", + "attachments", + "artifact", + "gateway.platforms", + "hosted_room_links", + "hosted_room_peer", + "messaging_refs", + "roomlink", + "transport_resolver", + "turn.handoff", +) + + +def test_local_room_modules_do_not_depend_on_excluded_surfaces(): + violations = {} + for relative_path in LOCAL_ROOM_MODULES: + source = (ROOT / relative_path).read_text(encoding="utf-8").lower() + found = [token for token in FORBIDDEN_SURFACES if token in source] + if found: + violations[relative_path] = found + + assert violations == {} diff --git a/tests/gateway/test_hosted_rooms.py b/tests/gateway/test_hosted_rooms.py index 80c9fdbf8d..3379b347a7 100644 --- a/tests/gateway/test_hosted_rooms.py +++ b/tests/gateway/test_hosted_rooms.py @@ -8,8 +8,10 @@ from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor import pytest +from gateway import hosted_room_driver as driver from gateway import hosted_rooms as rooms import hermes_state +from gateway.hosted_room_policy_checkpoint import HostedRoomPolicyCheckpoint from hermes_state import SessionDB USER = {"kind": "user", "id": "desktop-user", "display_name": "User"} @@ -396,7 +398,7 @@ def test_authority_scoped_events_require_gateway_and_epoch(tmp_path): _create(db) with pytest.raises(rooms.HostedRoomError, match="authority_gateway_id"): - _append( + rooms.append_event( db, room_id="room-1", event_id="turn-1", @@ -927,6 +929,147 @@ def test_byte_pressure_pruning_keeps_retired_room_id_reserved( _assert_retired_identity_stays_reserved(db, "room-full", fresh_id="room-new") +def test_tombstone_pruning_owns_only_room_log_driver_and_policy_tables(tmp_path): + db = tmp_path / "state.db" + _create(db) + identity = driver.TaskIdentity( + room_id="room-1", + task_id="task-1", + thread_id="thread-1", + turn_id="turn-1", + ) + driver.admit_task( + db, + identity, + payload={ + "target_profile": "ops", + "prompt": "Inspect.", + "source_event_seq": 1, + }, + clock=lambda: 20, + ) + driver.acquire_lease( + db, + room_id="room-1", + gateway_id="gateway-a", + authority_epoch=1, + process_generation="process-a", + ttl_seconds=30, + clock=lambda: 20, + ) + HostedRoomPolicyCheckpoint(db) + _disband(db, room_id="room-1", now=50) + with sqlite3.connect(db) as conn: + conn.execute( + """INSERT INTO hosted_room_policy_cursors( + room_id, through_seq, stopped_through_seq, updated_at + ) VALUES ('room-1', 1, 0, 50)""" + ) + conn.execute( + """INSERT INTO hosted_room_policy_threads + VALUES ('room-1', 'thread-1', 'user-1', 1, 0)""" + ) + conn.execute( + """INSERT INTO hosted_room_policy_events + VALUES ('room-1', 'thread-1', 'user-1', 1, '{}')""" + ) + conn.execute( + """INSERT INTO hosted_room_policy_watermarks + VALUES ('room-1', 'thread-1', 'ops', 1)""" + ) + conn.execute( + """INSERT INTO hosted_room_policy_publications + VALUES ('room-1', 'task-1', 'turn.settled', 0, 1)""" + ) + conn.execute( + """INSERT INTO hosted_room_policy_transcript + VALUES ( + 'room-1', 'thread-1', 1, 'message.user', NULL + )""" + ) + conn.execute( + """CREATE TABLE hosted_room_messaging_refs ( + room_id TEXT NOT NULL, + marker TEXT NOT NULL + )""" + ) + conn.execute( + """INSERT INTO hosted_room_policy_transcript_state + VALUES ('room-1', 1)""" + ) + conn.execute( + """INSERT INTO hosted_room_messaging_refs + VALUES ('room-1', 'outside-pr-b')""" + ) + + assert ( + rooms.prune_disbanded_rooms( + db, + now=50 + rooms.DISBANDED_ROOM_RETENTION_SECONDS + 1, + ) + == 1 + ) + with sqlite3.connect(db) as conn: + for table in ( + "hosted_rooms", + "hosted_room_events", + "hosted_room_driver_tasks", + "hosted_room_driver_leases", + "hosted_room_policy_cursors", + "hosted_room_policy_threads", + "hosted_room_policy_events", + "hosted_room_policy_watermarks", + "hosted_room_policy_publications", + "hosted_room_policy_transcript", + "hosted_room_policy_transcript_state", + ): + assert conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] == 0 + assert ( + conn.execute("SELECT marker FROM hosted_room_messaging_refs").fetchone()[0] + == "outside-pr-b" + ) + + +def test_policy_sync_cannot_recreate_projection_after_room_pruning( + tmp_path, + monkeypatch, +): + db = tmp_path / "state.db" + _create(db) + _append( + db, + room_id="room-1", + event_id="user-1", + kind="message.user", + actor=USER, + payload={"text": "hello", "thread_id": "thread-1"}, + now=11, + ) + checkpoint = HostedRoomPolicyCheckpoint(db) + original_read = rooms.read_events + + def read_then_prune(*args, **kwargs): + page = original_read(*args, **kwargs) + _disband(db, room_id="room-1", now=20) + rooms.prune_disbanded_rooms( + db, + now=20 + rooms.DISBANDED_ROOM_RETENTION_SECONDS + 1, + ) + return page + + monkeypatch.setattr(rooms, "read_events", read_then_prune) + with pytest.raises(rooms.RoomNotFoundError, match="not found"): + checkpoint.sync(room_id="room-1", latest_seq=1) + with sqlite3.connect(db) as conn: + for table in ( + "hosted_room_policy_cursors", + "hosted_room_policy_events", + "hosted_room_policy_transcript", + "hosted_room_policy_transcript_state", + ): + assert conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] == 0 + + def test_pre_actor_draft_database_migrates_with_explicit_legacy_identity(tmp_path): db = tmp_path / "state.db" _create_pre_actor_database(db) diff --git a/tests/gateway/test_hosted_rooms_read_path.py b/tests/gateway/test_hosted_rooms_read_path.py new file mode 100644 index 0000000000..983d6eaaf6 --- /dev/null +++ b/tests/gateway/test_hosted_rooms_read_path.py @@ -0,0 +1,31 @@ +"""Focused lock-behavior coverage for hosted Group Chat reads.""" + +from gateway import hosted_rooms + + +def test_list_rooms_does_not_enter_a_write_transaction_or_prune(tmp_path, monkeypatch): + db = tmp_path / "state.db" + hosted_rooms.create_room( + db, + room_id="room-1", + name="Release room", + members=[{"profile": "default", "handle": "hermes"}], + authority_gateway_id="gateway-a", + ) + + def reject_write_transaction(*_args, **_kwargs): + raise AssertionError("list_rooms must not enter a write transaction") + + def reject_prune(*_args, **_kwargs): + raise AssertionError("list_rooms must not prune retention state") + + monkeypatch.setattr(hosted_rooms, "_transaction", reject_write_transaction) + monkeypatch.setattr( + hosted_rooms, + "_prune_disbanded_rooms_locked", + reject_prune, + ) + + rows = hosted_rooms.list_rooms(db) + + assert [row["room_id"] for row in rows] == ["room-1"] diff --git a/tests/hermes_cli/test_web_server_boot_handshake.py b/tests/hermes_cli/test_web_server_boot_handshake.py index 3b37c242cc..df3d7c5367 100644 --- a/tests/hermes_cli/test_web_server_boot_handshake.py +++ b/tests/hermes_cli/test_web_server_boot_handshake.py @@ -84,6 +84,29 @@ def test_lifespan_warmup_is_synchronous(): ) +def test_hosted_room_recovery_cannot_block_or_abort_backend_startup(monkeypatch): + from fastapi.testclient import TestClient + from tui_gateway import methods_groups + + started = threading.Event() + release = threading.Event() + + def blocked_failure(): + started.set() + release.wait(timeout=2.0) + raise RuntimeError("state.db is locked") + + monkeypatch.setattr(web_server_mod, "_warm_gateway_module", lambda: None) + monkeypatch.setattr(methods_groups, "start_hosted_room_service", blocked_failure) + monkeypatch.setattr(methods_groups, "stop_hosted_room_service", lambda **_kwargs: True) + + before = time.perf_counter() + with TestClient(web_server_mod.app, raise_server_exceptions=False): + assert started.wait(timeout=1.0) + assert time.perf_counter() - before < 1.0 + release.set() + + # --------------------------------------------------------------------------- # Test 2 — get_status run_in_executor keeps event loop free for other requests # --------------------------------------------------------------------------- diff --git a/tests/tui_gateway/test_auto_continue.py b/tests/tui_gateway/test_auto_continue.py index 5910f421b9..950571418f 100644 --- a/tests/tui_gateway/test_auto_continue.py +++ b/tests/tui_gateway/test_auto_continue.py @@ -169,6 +169,60 @@ def test_handled_failure_still_clears_marker(emits, turn_env, marker_home): assert read_turn_marker(marker_home, "session-key") is None +def test_hosted_terminal_receipt_commits_before_marker_retire( + emits, turn_env, marker_home +): + observed = [] + + def _run(message, **kwargs): + return {"final_response": "done"} + + def _terminal(receipt): + observed.append((receipt, read_turn_marker(marker_home, "session-key"))) + + agent = types.SimpleNamespace( + session_id="session-key", run_conversation=_run, clear_interrupt=lambda: None + ) + session = _session(agent=agent, running=True, source="bot_room") + + server._run_prompt_submit( + "rid", + "sid", + session, + "do the thing", + terminal_callback=_terminal, + ) + + assert observed[0][0]["status"] == "settled" + assert observed[0][1] is not None + assert read_turn_marker(marker_home, "session-key") is None + + +def test_hosted_terminal_receipt_failure_keeps_crash_marker( + emits, turn_env, marker_home +): + def _run(message, **kwargs): + return {"final_response": "done"} + + def _terminal(_receipt): + raise RuntimeError("state store unavailable") + + agent = types.SimpleNamespace( + session_id="session-key", run_conversation=_run, clear_interrupt=lambda: None + ) + session = _session(agent=agent, running=True, source="bot_room") + + server._run_prompt_submit( + "rid", + "sid", + session, + "do the thing", + terminal_callback=_terminal, + ) + + assert read_turn_marker(marker_home, "session-key") is not None + + def test_continuation_turn_records_attempt_and_original_prompt( emits, turn_env, marker_home ): @@ -263,6 +317,20 @@ def test_fresh_marker_schedules_continuation(emits, schedule_env, marker_home): assert ("message.start", "sid", None) in [(e, s, p) for e, s, p in emits] +def test_hosted_room_marker_is_left_to_the_driver(schedule_env, marker_home): + record_turn_start(marker_home, "session-key", "hosted prompt") + + result = server._maybe_schedule_auto_continue( + "sid", + _session(source="bot_room"), + "session-key", + ) + + assert result is None + assert not schedule_env + assert read_turn_marker(marker_home, "session-key") is not None + + def test_stale_marker_is_cleared_not_continued(schedule_env, marker_home, monkeypatch): record_turn_start(marker_home, "session-key", "old prompt") monkeypatch.setattr( @@ -373,4 +441,3 @@ def test_failed_agent_build_leaves_marker_for_retry( # ── End to end: continuation runs a real turn and clears the marker ──── - diff --git a/tests/tui_gateway/test_groups_methods.py b/tests/tui_gateway/test_groups_methods.py index 9f58b612f5..8ca6abac5c 100644 --- a/tests/tui_gateway/test_groups_methods.py +++ b/tests/tui_gateway/test_groups_methods.py @@ -2,17 +2,24 @@ from __future__ import annotations +from types import SimpleNamespace + import pytest import tui_gateway.server as srv +from tui_gateway import methods_groups @pytest.fixture def home(tmp_path, monkeypatch): path = tmp_path / ".hermes" path.mkdir() + (path / "profiles" / "ops").mkdir(parents=True) monkeypatch.setenv("HERMES_HOME", str(path)) - return path + methods_groups.stop_hosted_room_service(timeout=1.0) + methods_groups.start_hosted_room_service() + yield path + methods_groups.stop_hosted_room_service(timeout=1.0) def _result(envelope): @@ -33,7 +40,14 @@ def _create_room(): { "room_id": "room-1", "name": "Release room", - "members": [{"profile": "ops", "handle": "ops"}], + "members": [ + { + "member_id": "default", + "profile": "default", + "handle": "hermes", + }, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ], "authority_gateway_id": "gateway-a", }, ) @@ -41,6 +55,7 @@ def _create_room(): def test_capabilities_are_honest_about_the_driver_boundary(home): + methods_groups.stop_hosted_room_service(timeout=1.0) result = _result(srv._methods["groups.capabilities"](1, {})) assert result["protocol_version"] == 2 @@ -52,6 +67,16 @@ def test_capabilities_are_honest_about_the_driver_boundary(home): assert "groups.state" in result["methods"] assert "groups.send" in result["methods"] assert "groups.send" in srv._LONG_HANDLERS + assert "groups.retry" in result["methods"] + assert "groups.approve" in result["methods"] + advertised = [ + str(value).lower() for value in (*result["features"], *result["methods"]) + ] + assert not any( + token in value + for token in ("attachment", "desktop", "messaging", "peer", "roomlink") + for value in advertised + ) def test_create_list_send_and_log_roundtrip(home): @@ -72,12 +97,12 @@ def test_create_list_send_and_log_roundtrip(home): "room_id": "room-1", "event_id": "event-1", "actor": {"kind": "user", "id": "desktop-user"}, - "payload": {"text": "hello"}, + "payload": {"text": "hello", "thread_id": "thread-1"}, }, ) ) assert sent["accepted"] is True - assert sent["driver_started"] is False + assert sent["driver_started"] is True assert sent["event"]["seq"] == 1 assert sent["event"]["kind"] == "message.user" assert sent["event"]["actor"] == {"kind": "user", "id": "desktop"} @@ -89,7 +114,10 @@ def test_create_list_send_and_log_roundtrip(home): ) ) assert replay["latest_seq"] == replay["cursor"] == 1 - assert replay["events"][0]["payload"] == {"text": "hello"} + assert replay["events"][0]["payload"] == { + "text": "hello", + "thread_id": "thread-1", + } def test_groups_list_returns_bounded_pages(home): @@ -137,7 +165,7 @@ def test_rpc_retry_is_idempotent_and_conflict_is_visible(home): "room_id": "room-1", "event_id": "event-1", "actor": {"kind": "user", "id": "desktop-user"}, - "payload": {"text": "hello"}, + "payload": {"text": "hello", "thread_id": "thread-1"}, } first = _result(srv._methods["groups.send"](2, params)) repeated = _result(srv._methods["groups.send"](3, params)) @@ -149,7 +177,10 @@ def test_rpc_retry_is_idempotent_and_conflict_is_visible(home): conflict = srv._methods["groups.send"]( 4, - {**params, "payload": {"text": "different"}}, + { + **params, + "payload": {"text": "different", "thread_id": "thread-1"}, + }, ) assert conflict["error"]["code"] == 4111 assert "different content" in conflict["error"]["message"] @@ -179,7 +210,7 @@ def test_foreign_authority_cannot_send_or_disband(home): { "room_id": "room-1", "event_id": "stale-send", - "payload": {"text": "must not land"}, + "payload": {"text": "must not land", "thread_id": "thread-1"}, }, ) disbanded = srv._methods["groups.disband"](3, {"room_id": "room-1"}) @@ -200,7 +231,7 @@ def test_client_event_id_cannot_squat_disband_receipt(home): { "room_id": "room-1", "event_id": "system:room-disbanded", - "payload": {"text": "still a user message"}, + "payload": {"text": "still a user message", "thread_id": "thread-1"}, }, ) ) @@ -221,6 +252,7 @@ def test_client_event_id_cannot_squat_disband_receipt(home): ) assert [event["kind"] for event in replay["events"]] == [ "message.user", + "room.stop_requested", "room.disbanded", ] @@ -234,7 +266,7 @@ def test_send_does_not_trust_client_supplied_actor_identity(home): "room_id": "room-1", "event_id": "event-1", "actor": {"kind": "user", "id": "spoofed-user"}, - "payload": {"text": "hello"}, + "payload": {"text": "hello", "thread_id": "thread-1"}, }, ) ) @@ -243,10 +275,14 @@ def test_send_does_not_trust_client_supplied_actor_identity(home): def test_create_ignores_client_supplied_authority_identity(home): + members = [ + {"member_id": "default", "profile": "default", "handle": "hermes"}, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ] created = _result( srv._methods["groups.create"]( 1, - {"room_id": "legacy-room", "name": "Legacy", "members": []}, + {"room_id": "legacy-room", "name": "Legacy", "members": members}, ) )["room"] retried = _result( @@ -255,7 +291,7 @@ def test_create_ignores_client_supplied_authority_identity(home): { "room_id": "legacy-room", "name": "Legacy", - "members": [], + "members": members, "authority_gateway_id": "spoofed-gateway", }, ) @@ -269,7 +305,10 @@ def test_create_ignores_client_supplied_authority_identity(home): def test_legacy_room_adoption_emits_one_lineage_receipt(home): from gateway.hosted_rooms import create_room, default_db_path - members = [{"profile": "ops", "handle": "ops"}] + members = [ + {"member_id": "default", "profile": "default", "handle": "hermes"}, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ] create_room( default_db_path(), room_id="legacy-room", @@ -285,9 +324,7 @@ def test_legacy_room_adoption_emits_one_lineage_receipt(home): {"room_id": "legacy-room", "name": "Legacy", "members": members}, ) )["room"] - state = _result( - srv._methods["groups.state"](3, {"room_id": "legacy-room"}) - )["room"] + state = _result(srv._methods["groups.state"](3, {"room_id": "legacy-room"}))["room"] assert adopted["adopted"] is True assert adopted["authority_gateway_id"] == _server_authority() @@ -327,7 +364,110 @@ def test_legacy_room_adoption_emits_one_lineage_receipt(home): ) def test_invalid_or_unknown_room_returns_contract_error(home, method_name, params): result = srv._methods[method_name](1, params) - assert result["error"]["code"] in {4110, 4111, 4112} + assert result["error"]["code"] in {4110, 4111, 4112, 5111, 5112} + + +def test_retry_and_approval_controls_forward_only_exact_local_coordinates( + home, monkeypatch +): + calls = [] + identity = SimpleNamespace( + room_id="room-1", + task_id="task-1", + thread_id="thread-1", + turn_id="turn-1", + ) + service = SimpleNamespace( + retry_room_task=lambda room_id, task_id: ( + calls.append(("retry", room_id, task_id)) + or { + "identity": identity, + "status": "queued", + "execution_generation": 1, + "cancel_generation": 0, + } + ), + approve_room_task=lambda room_id, **kwargs: ( + calls.append(("approve", room_id, kwargs)) or {"resolved": 1} + ), + ) + monkeypatch.setattr(srv, "get_hosted_room_service", lambda: service) + + retried = _result( + srv._methods["groups.retry"]( + 1, + {"room_id": "room-1", "task_id": "task-1"}, + ) + ) + approved = _result( + srv._methods["groups.approve"]( + 2, + { + "room_id": "room-1", + "member_id": "ops", + "task_id": "task-1", + "execution_generation": 1, + "request_id": "approval-1", + "choice": "once", + }, + ) + ) + + assert retried["task"] == { + "room_id": "room-1", + "task_id": "task-1", + "thread_id": "thread-1", + "turn_id": "turn-1", + "status": "queued", + "execution_generation": 1, + "cancel_generation": 0, + } + assert approved == {"approved": True, "result": {"resolved": 1}} + assert calls == [ + ("retry", "room-1", "task-1"), + ( + "approve", + "room-1", + { + "member_id": "ops", + "task_id": "task-1", + "execution_generation": 1, + "choice": "once", + "request_id": "approval-1", + }, + ), + ] + + +@pytest.mark.parametrize( + ("method_name", "params"), + [ + ("groups.create", {"room_id": "room-1", "name": "Room", "members": []}), + ("groups.send", {"room_id": "room-1", "event_id": "event-1", "payload": {}}), + ("groups.disband", {"room_id": "room-1"}), + ("groups.stop", {"room_id": "room-1"}), + ("groups.retry", {"room_id": "room-1", "task_id": "task-1"}), + ( + "groups.approve", + { + "room_id": "room-1", + "member_id": "ops", + "task_id": "task-1", + "execution_generation": 1, + "request_id": "approval-1", + "choice": "once", + }, + ), + ], +) +def test_mutating_controls_fail_closed_without_a_supervised_worker( + home, monkeypatch, method_name, params +): + monkeypatch.setattr(srv, "get_hosted_room_service", lambda: None) + + result = srv._methods[method_name](1, params) + + assert result["error"]["code"] in {4115, 4123} def test_disband_tombstones_room(home): @@ -337,9 +477,9 @@ def test_disband_tombstones_room(home): assert first["tombstone"]["idempotent"] is False assert repeated["tombstone"]["idempotent"] is True assert _result(srv._methods["groups.list"](5, {}))["rooms"] == [] - deleted = _result( - srv._methods["groups.list"](6, {"include_disbanded": True}) - )["rooms"] + deleted = _result(srv._methods["groups.list"](6, {"include_disbanded": True}))[ + "rooms" + ] assert deleted[0]["disbanded_at"] == first["tombstone"]["disbanded_at"] replay = _result( srv._methods["groups.log"]( @@ -347,12 +487,19 @@ def test_disband_tombstones_room(home): {"room_id": "room-1", "include_disbanded": True}, ) ) - assert [event["kind"] for event in replay["events"]] == ["room.disbanded"] + assert [event["kind"] for event in replay["events"]] == [ + "room.stop_requested", + "room.disbanded", + ] def test_pruned_room_send_and_log_report_expired_history(home, monkeypatch): from gateway import hosted_rooms + members = [ + {"member_id": "default", "profile": "default", "handle": "hermes"}, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ] _create_room() monkeypatch.setattr(hosted_rooms, "MAX_DISBANDED_ROOM_TOMBSTONES", 0) _result(srv._methods["groups.disband"](2, {"room_id": "room-1"})) @@ -367,7 +514,7 @@ def test_pruned_room_send_and_log_report_expired_history(home, monkeypatch): { "room_id": "room-1", "event_id": "stale-send", - "payload": {"text": "stale"}, + "payload": {"text": "stale", "thread_id": "thread-1"}, }, ) logged = srv._methods["groups.log"]( @@ -375,19 +522,21 @@ def test_pruned_room_send_and_log_report_expired_history(home, monkeypatch): {"room_id": "room-1", "include_disbanded": True}, ) + assert sent["error"]["code"] == 4111, sent + assert logged["error"]["code"] == 4112, logged assert sent["error"]["data"] == {"reason": "room_history_expired"} assert logged["error"]["data"] == {"reason": "room_history_expired"} assert "permanently retired" in sent["error"]["message"] recreated = srv._methods["groups.create"]( 6, - {"room_id": "room-1", "name": "Replacement", "members": []}, + {"room_id": "room-1", "name": "Replacement", "members": members}, ) assert recreated["error"]["code"] == 4110 created = _result( srv._methods["groups.create"]( 7, - {"room_id": "room-new", "name": "Fresh", "members": []}, + {"room_id": "room-new", "name": "Fresh", "members": members}, ) ) assert created["room"]["room_id"] == "room-new" diff --git a/tests/tui_gateway/test_hosted_room_driver_runtime.py b/tests/tui_gateway/test_hosted_room_driver_runtime.py new file mode 100644 index 0000000000..b4371ba826 --- /dev/null +++ b/tests/tui_gateway/test_hosted_room_driver_runtime.py @@ -0,0 +1,1366 @@ +"""Runtime tests for the hosted-room session adapter.""" + +from __future__ import annotations + +import threading +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +import pytest + +from gateway import hosted_room_driver as state +from gateway import hosted_rooms +from tui_gateway.hosted_room_driver import ( + MAX_TERMINAL_TEXT_BYTES, + ROOM_SESSION_SOURCE, + HostedRoomBinding, + HostedRoomRuntime, + room_session_title, +) + + +ROOM_ID = "room-1" +PROFILE = "ops" +BINDING = HostedRoomBinding( + room_id=ROOM_ID, + gateway_id="gateway-a", + authority_epoch=1, +) + + +class RecordingTurnLocks: + """Record the profile lock and expose ownership to the fake RPC.""" + + def __init__(self) -> None: + self.events: list[tuple[str, str]] = [] + self.local = threading.local() + + @contextmanager + def __call__(self, profile: str): + self.events.append(("lock-enter", profile)) + self.local.profile = profile + try: + yield + finally: + self.events.append(("lock-exit", profile)) + self.local.profile = None + + def held_for(self, profile: str) -> bool: + return getattr(self.local, "profile", None) == profile + + +class FakeSessionRPC: + """Normalized in-memory session adapter with no model or network.""" + + def __init__( + self, + *, + auto_complete: bool = True, + required_lock: RecordingTurnLocks | None = None, + ) -> None: + self.auto_complete = auto_complete + self.required_lock = required_lock + self.calls: list[tuple[str, dict[str, Any]]] = [] + self.sessions: dict[tuple[str, str], dict[str, Any]] = {} + self.states: dict[str, dict[str, Any]] = {} + self.submitted = threading.Event() + self.on_interrupt = None + self.on_info = None + self.history_failures = 0 + self._next_id = 1 + self._lock = threading.Lock() + + def _assert_lock(self, profile: str) -> None: + if self.required_lock is not None: + assert self.required_lock.held_for(profile) + + def add_session( + self, + *, + profile: str = PROFILE, + title: str = room_session_title(ROOM_ID), + active: bool = False, + task_id: str | None = None, + history: list[dict[str, Any]] | None = None, + ) -> str: + with self._lock: + session_id = f"session-{self._next_id}" + self._next_id += 1 + session = {"session_id": session_id, "title": title} + self.sessions[(profile, title)] = session + self.states[session_id] = { + "active": active, + "task_id": task_id, + "execution_generation": None, + "history": list(history or []), + "on_terminal": None, + "pending_approval": None, + } + return session_id + + def complete( + self, + task_id: str, + *, + content: str = "Finished once.", + status: str = "settled", + ) -> None: + callback = None + receipt = None + with self._lock: + for session_id, session_state in self.states.items(): + if session_state["task_id"] != task_id: + continue + receipt = { + "role": "assistant", + "task_id": task_id, + "execution_generation": session_state["execution_generation"], + "status": status, + "message_id": f"reply:{task_id}", + "content": content, + } + session_state["history"].append(receipt) + session_state["active"] = False + callback = session_state.get("on_terminal") + self.calls.append(("complete", {"session_id": session_id})) + break + if receipt is None: + raise AssertionError(f"no active session for {task_id}") + if callback is not None: + callback({ + "status": status, + "settlement_id": receipt["message_id"], + "message_id": receipt["message_id"], + "text": content, + }) + + def resolve_exact(self, *, profile: str, title: str, source: str): + self._assert_lock(profile) + params = {"profile": profile, "title": title, "source": source} + self.calls.append(("resolve_exact", params)) + with self._lock: + session = self.sessions.get((profile, title)) + return dict(session) if session is not None else None + + def create(self, *, profile: str, title: str, source: str): + self._assert_lock(profile) + params = {"profile": profile, "title": title, "source": source} + self.calls.append(("create", params)) + session_id = self.add_session(profile=profile, title=title) + return {"session_id": session_id, "title": title} + + def resume(self, *, profile: str, session_id: str, source: str): + self._assert_lock(profile) + params = { + "profile": profile, + "session_id": session_id, + "source": source, + } + self.calls.append(("resume", params)) + return {"session_id": session_id} + + def submit( + self, + *, + profile: str, + session_id: str, + prompt: str, + source: str, + task: state.TaskIdentity, + execution_generation: int, + on_terminal, + ): + self._assert_lock(profile) + params = { + "profile": profile, + "session_id": session_id, + "prompt": prompt, + "source": source, + "task": task, + "execution_generation": execution_generation, + "on_terminal": on_terminal, + } + self.calls.append(("submit", params)) + with self._lock: + self.states[session_id]["active"] = True + self.states[session_id]["task_id"] = task.task_id + self.states[session_id]["execution_generation"] = execution_generation + self.states[session_id]["on_terminal"] = on_terminal + self.submitted.set() + if self.auto_complete: + self.complete(task.task_id) + return {"accepted": True} + + def history(self, *, profile: str, session_id: str, source: str): + self._assert_lock(profile) + params = { + "profile": profile, + "session_id": session_id, + "source": source, + } + self.calls.append(("history", params)) + if self.history_failures > 0: + self.history_failures -= 1 + raise RuntimeError("transient history read failed") + with self._lock: + return [dict(message) for message in self.states[session_id]["history"]] + + def info(self, *, profile: str, session_id: str, source: str): + self._assert_lock(profile) + params = { + "profile": profile, + "session_id": session_id, + "source": source, + } + self.calls.append(("info", params)) + with self._lock: + session_state = self.states[session_id] + result = { + "active": session_state["active"], + "task_id": session_state["task_id"], + } + if session_state.get("pending_approval"): + result["status"] = "waiting_for_approval" + result["pending_approval"] = dict(session_state["pending_approval"]) + if self.on_info is not None: + self.on_info() + return result + + def interrupt( + self, + *, + profile: str, + session_id: str, + source: str, + expected_task_id: str, + ): + params = { + "profile": profile, + "session_id": session_id, + "source": source, + "expected_task_id": expected_task_id, + } + with self._lock: + current = self.states[session_id] + if not current["active"] or current["task_id"] != expected_task_id: + self.calls.append(("interrupt_skipped", params)) + return {"interrupted": False} + current["active"] = False + self.calls.append(("interrupt", params)) + if self.on_interrupt is not None: + self.on_interrupt() + return {"interrupted": True} + + +class SelectiveCompletionRPC(FakeSessionRPC): + """Keep selected local profiles running while peers complete normally.""" + + def __init__(self, *, waiting_profiles: set[str]) -> None: + super().__init__() + self.waiting_profiles = waiting_profiles + self._submit_mode_lock = threading.Lock() + + def submit(self, **kwargs): + with self._submit_mode_lock: + original = self.auto_complete + self.auto_complete = kwargs["profile"] not in self.waiting_profiles + try: + return super().submit(**kwargs) + finally: + self.auto_complete = original + + +@pytest.fixture +def db(tmp_path: Path) -> Path: + path = tmp_path / "state.db" + hosted_rooms.create_room( + path, + room_id=ROOM_ID, + name="Release room", + members=[{"profile": PROFILE, "handle": PROFILE}], + authority_gateway_id=BINDING.gateway_id, + now=time.time(), + ) + return path + + +def _identity(task_id: str = "task-1") -> state.TaskIdentity: + return state.TaskIdentity( + room_id=ROOM_ID, + task_id=task_id, + thread_id="thread-1", + turn_id=f"turn-{task_id}", + ) + + +def _admit( + db: Path, + identity: state.TaskIdentity, + *, + prompt: str = "Inspect the release candidate.", +) -> None: + state.admit_task( + db, + identity, + payload={ + "target_profile": PROFILE, + "prompt": prompt, + "source_event_seq": 1, + }, + clock=time.time, + ) + + +def _runtime( + db: Path, + rpc: FakeSessionRPC, + locks: RecordingTurnLocks | None = None, + **kwargs, +) -> HostedRoomRuntime: + return HostedRoomRuntime( + db_path=db, + rooms=[BINDING], + rpc=rpc, + turn_lock=locks or RecordingTurnLocks(), + lease_ttl_seconds=kwargs.pop("lease_ttl_seconds", 0.4), + poll_interval_seconds=kwargs.pop("poll_interval_seconds", 0.01), + **kwargs, + ) + + +def _wait_for(predicate, *, timeout: float = 2.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.01) + raise AssertionError("condition was not reached before timeout") + + +def test_runtime_uses_unique_process_generation(db: Path): + first = _runtime(db, FakeSessionRPC()) + second = _runtime(db, FakeSessionRPC()) + + assert first.process_generation != second.process_generation + assert len(first.process_generation) == 32 + + +@pytest.mark.parametrize("value", [0, True]) +def test_room_concurrency_bound_must_be_a_positive_integer(db: Path, value): + with pytest.raises(ValueError, match="max_concurrent_rooms"): + _runtime(db, FakeSessionRPC(), max_concurrent_rooms=value) + + +def test_waiting_room_does_not_block_an_independent_local_room(tmp_path: Path): + db = tmp_path / "state.db" + bindings = [ + HostedRoomBinding("room-waiting", "gateway-a", 1), + HostedRoomBinding("room-healthy", "gateway-a", 1), + ] + identities = [ + state.TaskIdentity("room-waiting", "task-waiting", "thread-a", "turn-a"), + state.TaskIdentity("room-healthy", "task-healthy", "thread-b", "turn-b"), + ] + profiles = ["profile-waiting", "profile-healthy"] + for binding, identity, profile in zip(bindings, identities, profiles): + hosted_rooms.create_room( + db, + room_id=binding.room_id, + name=binding.room_id, + members=[{"profile": profile, "handle": profile}], + authority_gateway_id=binding.gateway_id, + now=time.time(), + ) + state.admit_task( + db, + identity, + payload={ + "target_profile": profile, + "prompt": f"Run {binding.room_id}.", + "source_event_seq": 1, + }, + clock=time.time, + ) + + rpc = SelectiveCompletionRPC(waiting_profiles={"profile-waiting"}) + runtime = HostedRoomRuntime( + db_path=db, + rooms=bindings, + rpc=rpc, + turn_lock=RecordingTurnLocks(), + lease_ttl_seconds=0.4, + poll_interval_seconds=0.01, + max_concurrent_rooms=2, + ) + + runtime.start() + _wait_for(lambda: state.get_task(db, identities[1])["status"] == "settled") + _wait_for(lambda: state.get_task(db, identities[0])["status"] == "running") + assert state.get_task(db, identities[0])["status"] == "running" + _wait_for(lambda: len(runtime.status()["current_tasks"]) == 1) + assert len(runtime.status()["current_tasks"]) == 1 + assert runtime.stop(timeout=1.0) + + +def test_rotated_bounded_scheduler_eventually_runs_later_room(tmp_path: Path): + db = tmp_path / "state.db" + bindings = [ + HostedRoomBinding(f"room-{index}", "gateway-a", 1) for index in range(1, 4) + ] + for binding in bindings: + hosted_rooms.create_room( + db, + room_id=binding.room_id, + name=binding.room_id, + members=[{"profile": PROFILE, "handle": PROFILE}], + authority_gateway_id=binding.gateway_id, + now=time.time(), + ) + identity = state.TaskIdentity( + "room-3", + "task-room-3", + "thread-room-3", + "turn-room-3", + ) + state.admit_task( + db, + identity, + payload={ + "target_profile": PROFILE, + "prompt": "Run the later room.", + "source_event_seq": 1, + }, + clock=time.time, + ) + runtime = HostedRoomRuntime( + db_path=db, + rooms=bindings, + rpc=FakeSessionRPC(), + turn_lock=RecordingTurnLocks(), + lease_ttl_seconds=0.4, + poll_interval_seconds=0.01, + max_concurrent_rooms=2, + ) + + runtime.start() + _wait_for(lambda: state.get_task(db, identity)["status"] == "settled") + assert runtime.stop(timeout=1.0) + + +def test_queued_task_routes_profile_and_credentials_without_overrides(db: Path): + identity = _identity() + _admit(db, identity, prompt="Use the configured profile credentials.") + rpc = FakeSessionRPC() + runtime = _runtime(db, rpc) + + runtime.start() + _wait_for(lambda: state.get_task(db, identity)["status"] == "settled") + assert runtime.stop(timeout=1.0) + + create = next(params for method, params in rpc.calls if method == "create") + submit = next(params for method, params in rpc.calls if method == "submit") + assert create == { + "profile": PROFILE, + "title": f"Group: {ROOM_ID}", + "source": ROOM_SESSION_SOURCE, + } + assert submit["profile"] == PROFILE + assert submit["source"] == ROOM_SESSION_SOURCE + assert submit["prompt"] == "Use the configured profile credentials." + assert "model" not in create | submit + assert "provider" not in create | submit + assert state.get_task(db, identity)["result"]["text"] == "Finished once." + + +def test_worker_settles_without_any_client_transport(db: Path): + identity = _identity() + _admit(db, identity) + runtime = _runtime(db, FakeSessionRPC()) + + runtime.start() + _wait_for( + lambda: ( + state.get_task(db, identity)["status"] == "settled" + and runtime.status()["cycles"] >= 1 + ) + ) + + assert runtime.status()["running"] is True + assert runtime.status()["cycles"] >= 1 + assert runtime.stop(timeout=1.0) + + +def test_policy_hooks_prepare_and_publish_terminal_idempotently(db: Path): + identity = _identity() + _admit(db, identity) + prepared = [] + published = [] + runtime = _runtime( + db, + FakeSessionRPC(), + prepare_room=lambda binding: prepared.append(binding.room_id), + publish_terminal=lambda binding, task: published.append(( + binding.room_id, + task["identity"].task_id, + task["status"], + )), + ) + + runtime.start() + _wait_for(lambda: state.get_task(db, identity)["status"] == "settled") + assert runtime.stop(timeout=1.0) + + assert prepared + assert published == [(ROOM_ID, identity.task_id, "settled")] + + +def test_existing_canonical_session_is_resumed_not_duplicated(db: Path): + identity = _identity() + _admit(db, identity) + rpc = FakeSessionRPC() + session_id = rpc.add_session() + runtime = _runtime(db, rpc) + + runtime.start() + _wait_for(lambda: state.get_task(db, identity)["status"] == "settled") + assert runtime.stop(timeout=1.0) + + assert not [call for call in rpc.calls if call[0] == "create"] + resume = next(params for method, params in rpc.calls if method == "resume") + assert resume == { + "profile": PROFILE, + "session_id": session_id, + "source": ROOM_SESSION_SOURCE, + } + + +def test_local_crash_recovery_keeps_ambiguous_history_explicit_without_resume( + db: Path, +): + identity = _identity() + now = [100.0] + + def clock(): + return now[0] + + old_lease = state.acquire_lease( + db, + room_id=ROOM_ID, + gateway_id=BINDING.gateway_id, + authority_epoch=BINDING.authority_epoch, + process_generation="old-process", + ttl_seconds=0.2, + clock=clock, + ) + _admit(db, identity) + state.start_task( + db, + identity, + old_lease, + expected_cancel_generation=0, + clock=clock, + ) + rpc = FakeSessionRPC(auto_complete=False) + rpc.add_session( + task_id=identity.task_id, + history=[ + { + "role": "assistant", + "task_id": identity.task_id, + "execution_generation": 1, + "status": "settled", + "message_id": "reply:recovered", + "content": "Recovered durable answer.", + } + ], + ) + now[0] = 101.0 + runtime = _runtime( + db, + rpc, + clock=clock, + indeterminate_defer_seconds=5, + ) + + runtime._process_room(BINDING) + + recovered = state.get_task(db, identity) + assert recovered["status"] == "indeterminate" + assert recovered["result"] is None + assert not [call for call in rpc.calls if call[0] == "history"] + assert [call for call in rpc.calls if call[0] == "info"] + assert not [call for call in rpc.calls if call[0] == "resume"] + assert not [call for call in rpc.calls if call[0] == "submit"] + + +def test_expired_local_attempt_defers_without_hydrating_or_resubmitting(db: Path): + identity = _identity() + now = [100.0] + + def clock(): + return now[0] + + old_lease = state.acquire_lease( + db, + room_id=ROOM_ID, + gateway_id=BINDING.gateway_id, + authority_epoch=BINDING.authority_epoch, + process_generation="old-process", + ttl_seconds=0.2, + clock=clock, + ) + _admit(db, identity) + state.start_task( + db, + identity, + old_lease, + expected_cancel_generation=0, + clock=clock, + ) + rpc = FakeSessionRPC(auto_complete=False) + rpc.add_session( + task_id=identity.task_id, + history=[ + { + "role": "assistant", + "task_id": identity.task_id, + "execution_generation": 1, + "status": "settled", + "message_id": "reply:expired-recovered", + "content": "Recovered after lease expiry.", + } + ], + ) + now[0] = 101.0 + runtime = _runtime( + db, + rpc, + clock=clock, + indeterminate_defer_seconds=0.5, + ) + + runtime._process_room(BINDING) + now[0] = 102.0 + runtime._process_room(BINDING) + + recovered = state.get_task(db, identity) + assert recovered["status"] == "deferred" + assert recovered["result"] == { + "reason": "member_unavailable", + "retryable": True, + } + assert not [call for call in rpc.calls if call[0] == "history"] + assert not [call for call in rpc.calls if call[0] == "resume"] + assert not [call for call in rpc.calls if call[0] == "submit"] + + +def test_oversized_terminal_reply_is_bounded_without_waiting_for_deadline(db: Path): + identity = _identity() + _admit(db, identity) + rpc = FakeSessionRPC(auto_complete=False) + runtime = _runtime(db, rpc, turn_timeout_seconds=30) + + runtime.start() + assert rpc.submitted.wait(timeout=1.0) + rpc.complete( + identity.task_id, + content="é" * (MAX_TERMINAL_TEXT_BYTES + 100), + ) + _wait_for(lambda: state.get_task(db, identity)["status"] == "settled") + assert runtime.stop(timeout=1.0) + + result = state.get_task(db, identity)["result"] + assert result["truncated"] is True + assert len(result["text"].encode("utf-8")) <= MAX_TERMINAL_TEXT_BYTES + assert result["text"].endswith("share the full result as a file.]") + + +def test_turn_deadline_stops_exact_attempt_and_publishes_durable_failure(db: Path): + identity = _identity() + _admit(db, identity) + rpc = FakeSessionRPC(auto_complete=False) + published = [] + runtime = _runtime( + db, + rpc, + active_poll_interval_seconds=0.01, + turn_timeout_seconds=0.05, + publish_terminal=lambda _binding, task: published.append(task), + ) + + runtime.start() + _wait_for(lambda: state.get_task(db, identity)["status"] == "failed") + assert runtime.stop(timeout=1.0) + + failed = state.get_task(db, identity) + assert failed["result"] == { + "error": ( + "This Group Chat turn exceeded its configured time limit and was stopped." + ), + "reason_code": "turn_deadline_exceeded", + "timeout_seconds": 0.05, + } + assert failed["cancel_id"] == "deadline:1" + assert [call for call in rpc.calls if call[0] == "interrupt"] + assert [task["status"] for task in published] == ["failed"] + + +def test_deadline_releases_worker_capacity_for_later_room(tmp_path: Path): + db = tmp_path / "state.db" + bindings = [ + HostedRoomBinding("room-stuck", "gateway-a", 1), + HostedRoomBinding("room-healthy", "gateway-a", 1), + ] + identities = [ + state.TaskIdentity("room-stuck", "task-stuck", "thread-a", "turn-a"), + state.TaskIdentity("room-healthy", "task-healthy", "thread-b", "turn-b"), + ] + for binding, identity in zip(bindings, identities): + hosted_rooms.create_room( + db, + room_id=binding.room_id, + name=binding.room_id, + members=[{"profile": PROFILE, "handle": PROFILE}], + authority_gateway_id=binding.gateway_id, + ) + state.admit_task( + db, + identity, + payload={ + "target_profile": PROFILE, + "prompt": f"Run {binding.room_id}.", + "source_event_seq": 1, + }, + clock=time.time, + ) + + class FirstRoomStallsRPC(FakeSessionRPC): + def submit(self, **kwargs): + result = super().submit(**kwargs) + if kwargs["task"].room_id == "room-healthy": + self.complete(kwargs["task"].task_id) + return result + + rpc = FirstRoomStallsRPC(auto_complete=False) + runtime = HostedRoomRuntime( + db_path=db, + rooms=bindings, + rpc=rpc, + turn_lock=RecordingTurnLocks(), + lease_ttl_seconds=0.4, + poll_interval_seconds=0.02, + active_poll_interval_seconds=0.01, + turn_timeout_seconds=0.05, + max_concurrent_rooms=1, + ) + + runtime.start() + _wait_for(lambda: state.get_task(db, identities[0])["status"] == "failed") + _wait_for(lambda: state.get_task(db, identities[1])["status"] == "settled") + assert runtime.stop(timeout=1.0) + + assert state.get_task(db, identities[0])["result"]["reason_code"] == ( + "turn_deadline_exceeded" + ) + assert state.get_task(db, identities[1])["status"] == "settled" + + +def test_retry_ignores_late_receipt_from_prior_execution_generation(db: Path): + identity = _identity() + now = [100.0] + + def clock(): + return now[0] + + old_lease = state.acquire_lease( + db, + room_id=ROOM_ID, + gateway_id=BINDING.gateway_id, + authority_epoch=BINDING.authority_epoch, + process_generation="old-process", + ttl_seconds=0.2, + clock=clock, + ) + _admit(db, identity) + old_attempt = state.start_task( + db, + identity, + old_lease, + expected_cancel_generation=0, + clock=clock, + ) + now[0] = 101.0 + current_lease = state.acquire_lease( + db, + room_id=ROOM_ID, + gateway_id=BINDING.gateway_id, + authority_epoch=BINDING.authority_epoch, + process_generation="manual-recovery", + ttl_seconds=30, + clock=clock, + ) + state.recover_room(db, current_lease, clock=clock) + state.requeue_indeterminate_task( + db, + identity, + current_lease, + expected_execution_generation=old_attempt.execution_generation, + expected_cancel_generation=old_attempt.cancel_generation, + clock=clock, + ) + state.release_lease(db, current_lease, clock=clock) + rpc = FakeSessionRPC(auto_complete=False) + rpc.add_session( + task_id=identity.task_id, + history=[ + { + "role": "assistant", + "task_id": identity.task_id, + "execution_generation": old_attempt.execution_generation, + "status": "settled", + "message_id": "reply:late-old-attempt", + "content": "Late old result.", + } + ], + ) + runtime = _runtime(db, rpc, clock=clock) + + runtime.start() + assert rpc.submitted.wait(1.0) + time.sleep(0.04) + assert runtime.stop(timeout=1.0) + + task = state.get_task(db, identity) + assert task["status"] == "running" + assert task["execution_generation"] == old_attempt.execution_generation + 1 + + +def test_active_recovered_turn_is_never_resubmitted(db: Path): + identity = _identity() + old_lease = state.acquire_lease( + db, + room_id=ROOM_ID, + gateway_id=BINDING.gateway_id, + authority_epoch=BINDING.authority_epoch, + process_generation="old-process", + ttl_seconds=10, + clock=time.time, + ) + _admit(db, identity) + state.start_task( + db, + identity, + old_lease, + expected_cancel_generation=0, + clock=time.time, + ) + rpc = FakeSessionRPC(auto_complete=False) + rpc.add_session(active=True, task_id=identity.task_id) + runtime = _runtime(db, rpc) + + runtime.start() + time.sleep(0.08) + assert runtime.stop(timeout=1.0) + + assert state.get_task(db, identity)["status"] == "running" + assert not [call for call in rpc.calls if call[0] == "submit"] + + +def test_ambiguous_recovery_remains_indeterminate(db: Path): + identity = _identity() + now = [100.0] + + def clock(): + return now[0] + + old_lease = state.acquire_lease( + db, + room_id=ROOM_ID, + gateway_id=BINDING.gateway_id, + authority_epoch=BINDING.authority_epoch, + process_generation="old-process", + ttl_seconds=0.2, + clock=clock, + ) + _admit(db, identity) + state.start_task( + db, + identity, + old_lease, + expected_cancel_generation=0, + clock=clock, + ) + rpc = FakeSessionRPC(auto_complete=False) + rpc.add_session(active=False, task_id=identity.task_id) + now[0] = 101.0 + runtime = _runtime(db, rpc, clock=clock) + + runtime.start() + _wait_for(lambda: state.get_task(db, identity)["status"] == "indeterminate") + assert runtime.stop(timeout=1.0) + + assert not [call for call in rpc.calls if call[0] == "submit"] + + +def test_offline_member_defers_then_healthy_task_runs_and_retry_is_fenced( + db: Path, +): + now = [100.0] + + def clock(): + return now[0] + + first = _identity("task-offline") + second = state.TaskIdentity( + room_id=ROOM_ID, + task_id="task-healthy", + thread_id="thread-1", + turn_id="turn-task-healthy", + ) + state.admit_task( + db, + first, + payload={ + "target_profile": PROFILE, + "prompt": "Try the offline member.", + "source_event_seq": 1, + }, + clock=clock, + ) + old_lease = state.acquire_lease( + db, + room_id=ROOM_ID, + gateway_id=BINDING.gateway_id, + authority_epoch=BINDING.authority_epoch, + process_generation="old-process", + ttl_seconds=1, + clock=clock, + ) + old_attempt = state.start_task( + db, + first, + old_lease, + expected_cancel_generation=0, + clock=clock, + ) + state.admit_task( + db, + second, + payload={ + "target_profile": PROFILE, + "prompt": "Continue with the healthy member.", + "source_event_seq": 2, + }, + clock=clock, + ) + now[0] = 102.0 + rpc = FakeSessionRPC() + published = [] + runtime = _runtime( + db, + rpc, + clock=clock, + lease_ttl_seconds=30, + indeterminate_defer_seconds=5, + publish_terminal=lambda _binding, task: published.append(task), + ) + + runtime._process_room(BINDING) + assert state.get_task(db, first)["status"] == "indeterminate" + assert state.get_task(db, second)["status"] == "queued" + + now[0] = 108.0 + runtime._process_room(BINDING) + assert state.get_task(db, first)["status"] == "deferred" + assert state.get_task(db, second)["status"] == "settled" + assert [task["status"] for task in published] == ["deferred", "settled"] + assert ROOM_ID not in runtime.status()["blocked_rooms"] + + requeued = runtime.retry_indeterminate(first) + assert requeued["status"] == "queued" + lease = runtime._leases[ROOM_ID] + retry_attempt = state.start_task( + db, + first, + lease, + expected_cancel_generation=0, + clock=clock, + ) + assert retry_attempt.execution_generation == old_attempt.execution_generation + 1 + late_attempt = state.TaskAttempt( + identity=first, + lease=lease, + execution_generation=old_attempt.execution_generation, + cancel_generation=old_attempt.cancel_generation, + ) + with pytest.raises(state.StaleTaskError): + state.settle_task( + db, + late_attempt, + settlement_id="late-old-result", + status="settled", + result={"text": "too late"}, + clock=clock, + ) + state.settle_task( + db, + retry_attempt, + settlement_id="retry-result", + status="settled", + result={"text": "retry accepted"}, + clock=clock, + ) + assert state.get_task(db, first)["result"]["text"] == "retry accepted" + + +def test_post_submit_observation_failure_preserves_recoverable_outcome(db: Path): + identity = _identity() + _admit(db, identity) + rpc = FakeSessionRPC(auto_complete=False) + rpc.history_failures = 1 + runtime = _runtime(db, rpc) + + runtime.start() + assert rpc.submitted.wait(1.0) + _wait_for( + lambda: ( + "observation failed after submit" + in str(runtime.status()["last_error"] or "") + ) + ) + assert state.get_task(db, identity)["status"] == "running" + rpc.complete(identity.task_id, content="Recovered after a transient read.") + runtime.wakeup() + _wait_for(lambda: state.get_task(db, identity)["status"] == "settled") + assert runtime.stop(timeout=1.0) + + task = state.get_task(db, identity) + assert task["result"]["text"] == "Recovered after a transient read." + assert not [call for call in rpc.calls if call[0] == "submit"][1:] + + +def test_cancellation_is_persisted_before_interrupt_and_fences_late_result( + db: Path, +): + identity = _identity() + _admit(db, identity) + rpc = FakeSessionRPC(auto_complete=False) + runtime = _runtime(db, rpc) + observed_status: list[str] = [] + rpc.on_interrupt = lambda: observed_status.append( + state.get_task(db, identity)["status"] + ) + + runtime.start() + assert rpc.submitted.wait(1.0) + cancelled = runtime.cancel(identity, cancel_id="cancel-user") + rpc.complete(identity.task_id, content="Too late.") + runtime.wakeup() + time.sleep(0.05) + assert runtime.stop(timeout=1.0) + + assert cancelled["status"] == "cancelled" + assert observed_status == ["stopping"] + + +def test_transient_remote_stop_failure_stays_pending_and_retries(db: Path): + identity = _identity() + _admit(db, identity) + rpc = FakeSessionRPC(auto_complete=False) + runtime = _runtime(db, rpc) + original_interrupt = rpc.interrupt + attempts = 0 + retry_allowed = threading.Event() + + def flaky_interrupt(**kwargs): + nonlocal attempts + attempts += 1 + if attempts == 1: + raise RuntimeError("temporary stop transport failure") + assert retry_allowed.wait(1.0) + return original_interrupt(**kwargs) + + rpc.interrupt = flaky_interrupt + runtime.start() + assert rpc.submitted.wait(1.0) + stopping = runtime.cancel(identity, cancel_id="cancel-retry") + assert stopping["status"] == "stopping" + assert state.get_task(db, identity)["status"] == "stopping" + retry_allowed.set() + runtime.wakeup() + _wait_for(lambda: state.get_task(db, identity)["status"] == "cancelled") + assert attempts >= 2 + assert runtime.stop(timeout=1.0) + assert state.get_task(db, identity)["status"] == "cancelled" + + +def test_completion_wins_a_race_with_unacknowledged_stop(db: Path): + identity = _identity() + _admit(db, identity) + rpc = FakeSessionRPC(auto_complete=False) + runtime = _runtime(db, rpc) + + runtime.start() + assert rpc.submitted.wait(1.0) + + def finish_only_after_stop_intent(): + if state.get_task(db, identity)["status"] == "stopping": + rpc.complete(identity.task_id, content="Already done.") + + rpc.on_info = finish_only_after_stop_intent + result = runtime.cancel(identity, cancel_id="cancel-raced") + + assert result["status"] == "settled" + assert result["result"]["text"] == "Already done." + assert runtime.stop(timeout=1.0) + + +def test_restart_harvests_completion_before_retrying_durable_stop(db: Path): + identity = _identity() + _admit(db, identity) + now = [100.0] + old_lease = state.acquire_lease( + db, + room_id=ROOM_ID, + gateway_id=BINDING.gateway_id, + authority_epoch=BINDING.authority_epoch, + process_generation="old-process", + ttl_seconds=1.0, + clock=lambda: now[0], + ) + attempt = state.start_task( + db, + identity, + old_lease, + expected_cancel_generation=0, + clock=lambda: now[0], + ) + stopping = state.begin_task_cancel( + db, + identity, + cancel_id="cancel-before-restart", + expected_cancel_generation=attempt.cancel_generation, + clock=lambda: now[0], + ) + rpc = FakeSessionRPC(auto_complete=False) + rpc.add_session( + active=False, + task_id=identity.task_id, + history=[ + { + "role": "assistant", + "task_id": identity.task_id, + "execution_generation": attempt.execution_generation, + "status": "settled", + "message_id": "reply-after-stop", + "content": "Finished before Stop reached the session.", + } + ], + ) + now[0] += 2.0 + runtime = _runtime( + db, + rpc, + process_generation="new-process", + clock=lambda: now[0], + ) + + runtime.start() + _wait_for(lambda: state.get_task(db, identity)["status"] == "settled") + assert runtime.stop(timeout=1.0) + + settled = state.get_task(db, identity) + assert stopping["status"] == "stopping" + assert settled["result"]["text"] == "Finished before Stop reached the session." + assert not [call for call in rpc.calls if call[0] == "interrupt"] + + +def test_restart_acknowledges_inactive_local_stop_without_memory_marker(db: Path): + identity = _identity() + _admit(db, identity) + old_lease = state.acquire_lease( + db, + room_id=ROOM_ID, + gateway_id=BINDING.gateway_id, + authority_epoch=BINDING.authority_epoch, + process_generation="old-process", + ttl_seconds=0.05, + clock=time.time, + ) + attempt = state.start_task( + db, + identity, + old_lease, + expected_cancel_generation=0, + clock=time.time, + ) + state.begin_task_cancel( + db, + identity, + cancel_id="cancel-before-restart", + expected_cancel_generation=attempt.cancel_generation, + clock=time.time, + ) + rpc = FakeSessionRPC(auto_complete=False) + time.sleep(0.06) + runtime = _runtime(db, rpc, process_generation="new-process") + + cancelled = runtime.cancel(identity, cancel_id="cancel-before-restart") + + assert cancelled["status"] == "cancelled" + assert not [call for call in rpc.calls if call[0] == "interrupt"] + + +def test_pending_local_approval_is_reported_with_safe_choices(db: Path): + identity = _identity() + _admit(db, identity) + rpc = FakeSessionRPC(auto_complete=False) + actions = [] + runtime = _runtime( + db, + rpc, + pending_action=lambda room_id, member_id, action: actions.append(( + room_id, + member_id, + action, + )), + ) + + runtime.start() + assert rpc.submitted.wait(1.0) + session_id = next(iter(rpc.states)) + with rpc._lock: + rpc.states[session_id]["pending_approval"] = { + "request_id": "approval-1", + "command": "pytest -q tests/focused", + "choices": ["once", "session", "always", "deny"], + } + runtime.wakeup() + _wait_for(lambda: any(action for _room, _member, action in actions)) + + _room, member, action = next(item for item in actions if item[2] is not None) + assert member == PROFILE + assert action["request_id"] == "approval-1" + assert action["approval"]["choices"] == ["once", "deny"] + assert runtime.stop(timeout=1.0) + + +def test_cancel_never_interrupts_a_newer_task_in_the_same_session(db: Path): + identity = _identity() + _admit(db, identity) + rpc = FakeSessionRPC(auto_complete=False) + runtime = _runtime(db, rpc) + + runtime.start() + assert rpc.submitted.wait(1.0) + session_id = next(iter(rpc.states)) + + def switch_to_newer_task() -> None: + with rpc._lock: + rpc.states[session_id]["active"] = True + rpc.states[session_id]["task_id"] = "task-2" + + rpc.on_info = switch_to_newer_task + cancelled = runtime.cancel(identity, cancel_id="cancel-old-task") + + assert cancelled["status"] == "stopping" + assert not [call for call in rpc.calls if call[0] == "interrupt"] + assert not [call for call in rpc.calls if call[0] == "interrupt_skipped"] + assert rpc.states[session_id]["active"] is True + assert rpc.states[session_id]["task_id"] == "task-2" + assert runtime.stop(timeout=1.0) + + +def test_status_reports_room_blocked_on_unresolved_indeterminate_task(db: Path): + identity = _identity() + now = [100.0] + old_lease = state.acquire_lease( + db, + room_id=ROOM_ID, + gateway_id=BINDING.gateway_id, + authority_epoch=BINDING.authority_epoch, + process_generation="old-process", + ttl_seconds=1.0, + clock=lambda: now[0], + ) + _admit(db, identity) + state.start_task( + db, + identity, + old_lease, + expected_cancel_generation=0, + clock=lambda: now[0], + ) + rpc = FakeSessionRPC(auto_complete=False) + rpc.add_session(active=False, task_id=identity.task_id) + now[0] += 2.0 + runtime = _runtime(db, rpc, clock=lambda: now[0]) + + runtime.start() + _wait_for(lambda: ROOM_ID in runtime.status()["blocked_rooms"]) + assert runtime.stop(timeout=1.0) + + assert state.get_task(db, identity)["status"] == "indeterminate" + + +def test_authority_loss_stops_terminal_commit(db: Path): + identity = _identity() + _admit(db, identity) + rpc = FakeSessionRPC(auto_complete=False) + runtime = _runtime(db, rpc, lease_ttl_seconds=0.1) + + runtime.start() + assert rpc.submitted.wait(1.0) + hosted_rooms.claim_authority( + db, + room_id=ROOM_ID, + expected_gateway_id="gateway-a", + expected_epoch=1, + new_gateway_id="gateway-b", + event_id="claim-gateway-b", + now=time.time(), + ) + rpc.complete(identity.task_id) + runtime.wakeup() + _wait_for(lambda: runtime.status()["last_error"] is not None) + assert runtime.stop(timeout=1.0) + + assert state.get_task(db, identity)["status"] == "running" + assert "authority changed" in runtime.status()["last_error"] + + +def test_profile_turn_lock_covers_resolve_submit_and_terminal_observation(db: Path): + identity = _identity() + _admit(db, identity) + locks = RecordingTurnLocks() + rpc = FakeSessionRPC(required_lock=locks) + runtime = _runtime(db, rpc, locks) + + runtime.start() + _wait_for(lambda: state.get_task(db, identity)["status"] == "settled") + assert runtime.stop(timeout=1.0) + + assert locks.events == [("lock-enter", PROFILE), ("lock-exit", PROFILE)] + methods = [method for method, _params in rpc.calls] + assert methods.index("resolve_exact") < methods.index("submit") + assert methods.index("submit") < methods.index("complete") + assert "history" not in methods + + +def test_stop_is_bounded_and_does_not_interrupt_active_turn(db: Path): + identity = _identity() + _admit(db, identity) + rpc = FakeSessionRPC(auto_complete=False) + runtime = _runtime(db, rpc, poll_interval_seconds=0.01) + + runtime.start() + assert rpc.submitted.wait(1.0) + started = time.monotonic() + stopped = runtime.stop(timeout=0.5) + + assert stopped is True + assert time.monotonic() - started < 0.5 + assert state.get_task(db, identity)["status"] == "running" + assert not [call for call in rpc.calls if call[0] == "interrupt"] diff --git a/tests/tui_gateway/test_hosted_room_prompt_fence.py b/tests/tui_gateway/test_hosted_room_prompt_fence.py new file mode 100644 index 0000000000..0592c2f67c --- /dev/null +++ b/tests/tui_gateway/test_hosted_room_prompt_fence.py @@ -0,0 +1,151 @@ +"""Older Desktop clients cannot start a second driver for hosted rooms.""" + +from __future__ import annotations + +import sqlite3 +import time + +import pytest + +from gateway import hosted_rooms +import tui_gateway.server as server + + +def _stub_session(monkeypatch, *, title): + monkeypatch.setattr( + server, + "_sess_nowait", + lambda _params, _rid: ( + {"id": "session-1", "title": title, "source": "bot_room"}, + None, + ), + ) + + +def test_direct_prompt_to_hosted_group_session_is_rejected(tmp_path, monkeypatch): + home = tmp_path / ".hermes" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + hosted_rooms.create_room( + hosted_rooms.default_db_path(), + room_id="room-hosted", + name="Hosted room", + members=[ + {"member_id": "one", "profile": "one", "handle": "one"}, + {"member_id": "two", "profile": "two", "handle": "two"}, + ], + authority_gateway_id=hosted_rooms.local_authority_gateway_id(), + ) + _stub_session(monkeypatch, title="Group: room-hosted") + + result = server._methods["prompt.submit"]( + "request-1", {"session_id": "session-1", "text": "continue"} + ) + + assert result["error"]["code"] == 4122 + assert "managed by its gateway" in result["error"]["message"] + + +def test_direct_prompt_to_non_hosted_group_reaches_normal_admission( + tmp_path, monkeypatch +): + home = tmp_path / ".hermes" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + _stub_session(monkeypatch, title="Group: local-only") + monkeypatch.setattr( + server, + "_ensure_active_session_slot", + lambda _sid, _session: "normal admission reached", + ) + + result = server._methods["prompt.submit"]( + "request-2", {"session_id": "session-1", "text": "continue"} + ) + + assert result["error"] == {"code": 4090, "message": "normal admission reached"} + assert not hosted_rooms.default_db_path().exists() + + +@pytest.mark.parametrize( + "legacy_name", + ( + "Launch room", + "Ceo, Product Designer, Cfo", + "Équipe", + "Alpha/Beta", + ), +) +def test_direct_prompt_to_legacy_named_group_reaches_normal_admission( + tmp_path, monkeypatch, legacy_name +): + home = tmp_path / ".hermes" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + _stub_session(monkeypatch, title=f"Group: {legacy_name}") + monkeypatch.setattr( + server, + "_ensure_active_session_slot", + lambda _sid, _session: "normal admission reached", + ) + + result = server._methods["prompt.submit"]( + "request-legacy", {"session_id": "session-1", "text": "continue"} + ) + + assert result["error"] == {"code": 4090, "message": "normal admission reached"} + + +def test_direct_prompt_is_refused_when_room_authority_cannot_be_verified( + tmp_path, monkeypatch +): + home = tmp_path / ".hermes" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + _stub_session(monkeypatch, title="Group: room-unknown") + monkeypatch.setattr( + hosted_rooms, + "probe_hosted_room", + lambda *_args, **_kwargs: (_ for _ in ()).throw(OSError("disk busy")), + ) + + result = server._methods["prompt.submit"]( + "request-3", {"session_id": "session-1", "text": "continue"} + ) + + assert result["error"]["code"] == 5122 + assert result["error"]["message"] == ( + "Could not verify this group. Try again after the gateway recovers." + ) + + +def test_contended_ownership_probe_fails_quickly_without_blocking_socket( + tmp_path, monkeypatch +): + home = tmp_path / ".hermes" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + db = hosted_rooms.default_db_path() + hosted_rooms.create_room( + db, + room_id="room-busy", + name="Busy room", + members=[], + authority_gateway_id=hosted_rooms.local_authority_gateway_id(), + ) + _stub_session(monkeypatch, title="Group: room-busy") + + blocker = sqlite3.connect(db) + blocker.execute("PRAGMA journal_mode=DELETE") + blocker.execute("BEGIN EXCLUSIVE") + started = time.monotonic() + try: + result = server._methods["prompt.submit"]( + "request-busy", {"session_id": "session-1", "text": "continue"} + ) + finally: + blocker.rollback() + blocker.close() + + assert time.monotonic() - started < 0.5 + assert result["error"]["code"] == 5122 diff --git a/tests/tui_gateway/test_hosted_room_server_rpc.py b/tests/tui_gateway/test_hosted_room_server_rpc.py new file mode 100644 index 0000000000..10577bda0c --- /dev/null +++ b/tests/tui_gateway/test_hosted_room_server_rpc.py @@ -0,0 +1,177 @@ +"""Tests for the in-process hosted room session adapter.""" + +from __future__ import annotations + +import threading +from types import SimpleNamespace + +import pytest + +from gateway.hosted_room_driver import TaskIdentity +from tui_gateway.hosted_room_server_rpc import ( + HostedRoomServerRPC, + HostedRoomSessionError, +) + + +def _server(): + sessions = {} + calls = [] + + def method(name, result): + def handler(rid, params): + calls.append((name, params)) + value = result(params) if callable(result) else result + return {"id": rid, **value} + + return handler + + methods = { + "session.list": method( + "session.list", + {"result": {"sessions": [{"id": "stored", "resolved_id": "tip", "title": "Group: room"}]}}, + ), + "session.create": method("session.create", {"result": {"session_id": "runtime"}}), + "session.resume": method("session.resume", {"result": {"session_id": "runtime"}}), + "session.history": method("session.history", {"result": {"messages": [{"role": "assistant"}]}}), + "session.interrupt": method("session.interrupt", {"result": {"interrupted": True}}), + "approval.respond": method("approval.respond", {"result": {"resolved": 1}}), + "prompt.submit": method("prompt.submit", {"result": {"status": "streaming"}}), + } + server = SimpleNamespace( + _methods=methods, + _sessions=sessions, + _sessions_lock=threading.Lock(), + _pending_approval_request_payload=lambda _session_key: None, + ) + return server, calls + + +def test_routes_exact_hidden_session_and_internal_task_proof(): + server, calls = _server() + rpc = HostedRoomServerRPC(server) + task = TaskIdentity("room", "task", "thread", "turn") + callback = lambda _receipt: None + + assert rpc.resolve_exact(profile="ops", title="Group: room", source="bot_room")["session_id"] == "tip" + assert rpc.create(profile="ops", title="Group: room", source="bot_room")["session_id"] == "runtime" + rpc.submit( + profile="ops", + session_id="runtime", + prompt="Do the work", + source="bot_room", + task=task, + execution_generation=2, + on_terminal=callback, + ) + + create = next(params for method, params in calls if method == "session.create") + submit = next(params for method, params in calls if method == "prompt.submit") + assert create["hidden"] is True + assert create["room_plumbing"] is True + assert create["follow_profile_config"] is True + assert create["close_on_disconnect"] is False + assert submit["_hosted_task"] == { + "room_id": "room", + "task_id": "task", + "thread_id": "thread", + "turn_id": "turn", + "execution_generation": 2, + } + assert submit["_hosted_terminal_callback"] is callback + + rpc.resume(profile="ops", session_id="stored", source="bot_room") + resume = next(params for method, params in calls if method == "session.resume") + assert resume["source"] == "bot_room" + + +def test_info_and_interrupt_are_exact_task_scoped(): + server, calls = _server() + lock = threading.Lock() + server._sessions["runtime"] = { + "history_lock": lock, + "running": True, + "_hosted_room_task": {"task_id": "task-a"}, + } + rpc = HostedRoomServerRPC(server) + + assert rpc.info(profile="ops", session_id="runtime", source="bot_room") == { + "active": True, + "task_id": "task-a", + } + rpc.interrupt( + profile="ops", + session_id="runtime", + source="bot_room", + expected_task_id="task-a", + ) + params = next(params for method, params in calls if method == "session.interrupt") + assert params["expected_hosted_task_id"] == "task-a" + + +def test_local_approval_snapshot_and_response_use_exact_request(): + server, calls = _server() + server._pending_approval_request_payload = lambda session_key: { + "request_id": "approval-1", + "command": "pytest -q tests/focused", + "choices": ["once", "deny"], + } if session_key == "stored-session" else None + server._sessions["runtime"] = { + "history_lock": threading.Lock(), + "running": True, + "session_key": "stored-session", + "_hosted_room_task": {"task_id": "task-a"}, + } + rpc = HostedRoomServerRPC(server) + + info = rpc.info(profile="ops", session_id="runtime", source="bot_room") + assert info["status"] == "waiting_for_approval" + assert info["pending_approval"]["request_id"] == "approval-1" + assert rpc.approve( + session_id="runtime", + request_id="approval-1", + choice="once", + ) == {"resolved": 1} + params = next(params for method, params in calls if method == "approval.respond") + assert params == { + "session_id": "runtime", + "request_id": "approval-1", + "choice": "once", + "all": False, + } + + +def test_rpc_errors_are_typed(): + server, _calls = _server() + server._methods["session.list"] = lambda rid, _params: { + "id": rid, + "error": {"code": 4007, "message": "not found"}, + } + rpc = HostedRoomServerRPC(server) + + with pytest.raises(HostedRoomSessionError) as exc: + rpc.resolve_exact(profile="ops", title="Group: room", source="bot_room") + assert exc.value.code == 4007 + + +def test_prompt_rejection_is_proven_not_admitted(): + server, _calls = _server() + server._methods["prompt.submit"] = lambda rid, _params: { + "id": rid, + "error": {"code": 4121, "message": "session is already busy"}, + } + rpc = HostedRoomServerRPC(server) + + with pytest.raises(HostedRoomSessionError) as exc: + rpc.submit( + profile="ops", + session_id="runtime", + prompt="Do the work", + source="bot_room", + task=TaskIdentity("room", "task", "thread", "turn"), + execution_generation=1, + on_terminal=lambda _receipt: None, + ) + + assert exc.value.code == 4121 + assert exc.value.not_admitted is True diff --git a/tests/tui_gateway/test_hosted_room_service.py b/tests/tui_gateway/test_hosted_room_service.py new file mode 100644 index 0000000000..3c3664ae36 --- /dev/null +++ b/tests/tui_gateway/test_hosted_room_service.py @@ -0,0 +1,748 @@ +"""Integration tests for the hosted Discussion coordinator.""" + +from __future__ import annotations + +import json +import sqlite3 +import threading +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from gateway import hosted_room_driver as driver +from gateway import hosted_room_discussion as discussion +from gateway import hosted_rooms +from gateway.hosted_room_policy_checkpoint import MAX_ACTIVE_POLICY_EVENTS +from tui_gateway.hosted_room_service import HostedRoomService + + +def _append_room_event(db, **kwargs): + if kwargs.get("kind") == "message.user": + room = hosted_rooms.room_state(db, room_id=kwargs["room_id"]) + kwargs.setdefault( + "authority_gateway_id", str(room["authority_gateway_id"]) + ) + kwargs.setdefault("authority_epoch", int(room["authority_epoch"])) + return hosted_rooms.append_event(db, **kwargs) + + +class _FakeRPC: + def __init__(self) -> None: + self.sessions = {} + + def resolve_exact(self, *, profile, title, source): + return self.sessions.get((profile, title)) + + def create(self, *, profile, title, source): + session = {"session_id": f"{profile}-session", "title": title} + self.sessions[(profile, title)] = session + return session + + def resume(self, *, profile, session_id, source): + return {"session_id": session_id} + + def submit( + self, + *, + profile, + session_id, + prompt, + source, + task, + execution_generation, + on_terminal, + ): + on_terminal({"status": "settled", "text": f"reply from {profile}"}) + return {"accepted": True} + + def history(self, *, profile, session_id, source): + return [] + + def info(self, *, profile, session_id, source): + return {"active": False, "task_id": None} + + def interrupt(self, *, profile, session_id, source, expected_task_id): + return {"interrupted": True} + + +class _PromptRecordingRPC(_FakeRPC): + def __init__(self) -> None: + super().__init__() + self.prompts: list[tuple[str, str]] = [] + + def submit( + self, + *, + profile, + session_id, + prompt, + source, + task, + execution_generation, + on_terminal, + ): + self.prompts.append((profile, prompt)) + on_terminal({"status": "settled", "text": f"reply from {profile}"}) + return {"accepted": True} + + +class _BlockingFirstRPC(_PromptRecordingRPC): + def __init__(self) -> None: + super().__init__() + self.first_started = threading.Event() + self.release_first = threading.Event() + + def submit(self, **kwargs): + self.prompts.append((kwargs["profile"], kwargs["prompt"])) + if len(self.prompts) == 1: + self.first_started.set() + assert self.release_first.wait(timeout=2) + kwargs["on_terminal"]( + {"status": "settled", "text": f"reply from {kwargs['profile']}"} + ) + return {"accepted": True} + + +def _server(): + return SimpleNamespace(_methods={}, _sessions={}, _sessions_lock=threading.Lock()) + + +def _wait_for(predicate, timeout=2.0): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.01) + raise AssertionError("condition was not reached") + + +def test_create_send_drive_publish_and_replay_without_client_transport(tmp_path: Path): + db = tmp_path / "state.db" + service = HostedRoomService(_server(), db_path=db) + service.rpc = _FakeRPC() + service.runtime.rpc = service.rpc + service.local_profiles = lambda: ("default", "ops") + room = service.create_room( + room_id="room-1", + name="Release room", + members=[ + {"member_id": "default", "profile": "default", "handle": "hermes"}, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ], + ) + assert room["room_id"] == "room-1" + + service.start() + service.send( + room_id="room-1", + event_id="user-1", + payload={"text": "@ops inspect the release", "thread_id": "thread-1"}, + ) + _wait_for( + lambda: any( + event["kind"] == "message.member" for event in service._events("room-1") + ) + ) + assert service.stop(timeout=1.0) + + events = service._events("room-1") + assert [event["kind"] for event in events][:3] == [ + "message.user", + "message.member", + "turn.settled", + ] + assert events[1]["payload"]["text"] == "reply from ops" + assert service.status("room-1")["working"] is False + + +def test_restart_republishes_terminal_task_before_admitting_more(tmp_path: Path): + db = tmp_path / "state.db" + service = HostedRoomService(_server(), db_path=db) + service.local_profiles = lambda: ("default", "ops") + service.create_room( + room_id="room-1", + name="Release room", + members=[ + {"member_id": "default", "profile": "default", "handle": "hermes"}, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ], + ) + event = _append_room_event( + db, + room_id="room-1", + event_id="user-1", + kind="message.user", + actor={"kind": "user", "id": "desktop"}, + payload={"text": "@ops inspect", "thread_id": "thread-1"}, + ) + binding = service.bindings()[0] + service.prepare_room(binding) + task = driver.list_tasks(db, room_id="room-1", status="queued")[0] + lease = driver.acquire_lease( + db, + room_id="room-1", + gateway_id=binding.gateway_id, + authority_epoch=binding.authority_epoch, + process_generation="crashed", + ttl_seconds=30, + clock=time.time, + ) + attempt = driver.start_task( + db, + task["identity"], + lease, + expected_cancel_generation=0, + clock=time.time, + ) + driver.settle_task( + db, + attempt, + settlement_id="reply-1", + status="settled", + result={"text": "done"}, + clock=time.time, + ) + + service.prepare_room(binding) + events = service._events("room-1") + assert event["seq"] == 1 + assert sum(row["kind"] == "message.member" for row in events) == 1 + assert sum(row["kind"] == "turn.settled" for row in events) == 1 + service.prepare_room(binding) + replayed = service._events("room-1") + assert replayed == events + + +def test_policy_checkpoint_bounds_replay_after_completed_room_history( + tmp_path: Path, + monkeypatch, +): + db = tmp_path / "state.db" + service = HostedRoomService(_server(), db_path=db) + service.local_profiles = lambda: ("default", "ops") + room = service.create_room( + room_id="room-1", + name="Long-running room", + members=[ + {"member_id": "default", "profile": "default", "handle": "default"}, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ], + ) + authority = str(room["authority_gateway_id"]) + rows = [] + for index in range(200): + user_seq = index * 2 + 1 + activity_seq = user_seq + 1 + thread_id = f"thread-{index}" + event_id = f"user-{index}" + rows.extend(( + ( + "room-1", + user_seq, + event_id, + "message.user", + json.dumps({"kind": "user", "id": "load-test"}), + None, + json.dumps({"text": "done", "thread_id": thread_id}), + float(user_seq), + ), + ( + "room-1", + activity_seq, + f"activity-{index}", + "room.activity", + json.dumps({"kind": "gateway", "id": authority}), + 1, + json.dumps({ + "status": "settled", + "reason_code": "silent_round", + "thread_id": thread_id, + "discussion_event_id": event_id, + }), + float(activity_seq), + ), + )) + with sqlite3.connect(db) as conn: + conn.executemany( + """INSERT INTO hosted_room_events( + room_id, seq, event_id, kind, actor_json, + authority_epoch, payload_json, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", + rows, + ) + conn.execute( + """UPDATE hosted_rooms + SET next_seq=401, revision=revision+400, updated_at=400 + WHERE room_id='room-1'""" + ) + _append_room_event( + db, + room_id="room-1", + event_id="user-active", + kind="message.user", + actor={"kind": "user", "id": "desktop"}, + payload={"text": "Review this", "thread_id": "thread-active"}, + now=401, + ) + + original_read_events = hosted_rooms.read_events + reads = {"calls": 0, "rows": 0} + + def counted_read_events(*args, **kwargs): + page = original_read_events(*args, **kwargs) + reads["calls"] += 1 + reads["rows"] += len(page["events"]) + return page + + monkeypatch.setattr(hosted_rooms, "read_events", counted_read_events) + binding = service.bindings()[0] + service.prepare_room(binding) + assert reads["rows"] == 401 + snapshot = service._policy_snapshot(hosted_rooms.room_state(db, room_id="room-1")) + assert len(snapshot.events) == 1 + assert len(snapshot.events) <= MAX_ACTIVE_POLICY_EVENTS + with sqlite3.connect(db) as conn: + assert ( + conn.execute("SELECT COUNT(*) FROM hosted_room_policy_events").fetchone()[0] + == 1 + ) + assert ( + conn.execute("SELECT COUNT(*) FROM hosted_room_policy_threads").fetchone()[ + 0 + ] + == 1 + ) + + reads.update(calls=0, rows=0) + service.prepare_room(binding) + assert reads == {"calls": 0, "rows": 0} + + +def test_same_thread_followup_migrates_and_delivers_committed_peer_reply( + tmp_path: Path, +): + db = tmp_path / "state.db" + service = HostedRoomService(_server(), db_path=db) + service.rpc = _PromptRecordingRPC() + service.runtime.rpc = service.rpc + service.local_profiles = lambda: ("default", "ops") + service.create_room( + room_id="room-1", + name="Shared context room", + members=[ + {"member_id": "default", "profile": "default", "handle": "hermes"}, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ], + ) + + service.start() + service.send( + room_id="room-1", + event_id="user-1", + payload={"text": "@ops provide the marker", "thread_id": "thread-1"}, + ) + _wait_for(lambda: len(service.rpc.prompts) == 1) + _wait_for( + lambda: any( + event["kind"] == "room.activity" + and event["payload"]["discussion_event_id"] == "user-1" + for event in service._events("room-1") + ) + ) + with sqlite3.connect(db) as conn: + assert conn.execute( + """SELECT COUNT(*) FROM hosted_room_policy_transcript + WHERE room_id='room-1' AND thread_id='thread-1'""" + ).fetchone()[0] == 2 + conn.execute("DELETE FROM hosted_room_policy_transcript") + conn.execute( + """DELETE FROM hosted_room_policy_transcript_state + WHERE room_id='room-1'""" + ) + service.send( + room_id="room-1", + event_id="user-2", + payload={"text": "@hermes continue", "thread_id": "thread-1"}, + ) + _wait_for(lambda: len(service.rpc.prompts) == 2) + assert service.stop(timeout=1.0) + + profile, prompt = service.rpc.prompts[1] + assert profile == "default" + assert "@ops: reply from ops" in prompt + assert "User (user): @hermes continue" in prompt + + +def test_active_same_thread_followup_waits_for_current_task(tmp_path: Path): + db = tmp_path / "state.db" + service = HostedRoomService(_server(), db_path=db) + service.rpc = _BlockingFirstRPC() + service.runtime.rpc = service.rpc + service.local_profiles = lambda: ("default", "ops") + service.create_room( + room_id="room-1", + name="Serialized room", + members=[ + {"member_id": "default", "profile": "default", "handle": "hermes"}, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ], + ) + + service.start() + service.send( + room_id="room-1", + event_id="user-1", + payload={"text": "@ops start", "thread_id": "thread-1"}, + ) + assert service.rpc.first_started.wait(timeout=2) + service.send( + room_id="room-1", + event_id="user-2", + payload={"text": "@hermes follow up", "thread_id": "thread-1"}, + ) + assert len(service.rpc.prompts) == 1 + service.rpc.release_first.set() + _wait_for(lambda: len(service.rpc.prompts) == 2) + _wait_for( + lambda: any( + event["kind"] == "room.activity" + and event["payload"]["discussion_event_id"] == "user-2" + for event in service._events("room-1") + ) + ) + assert service.stop(timeout=1.0) + assert "User (user): @hermes follow up" in service.rpc.prompts[1][1] + + +def test_thread_transcript_prunes_committed_message_and_settlement_together( + tmp_path: Path, +): + db = tmp_path / "state.db" + service = HostedRoomService(_server(), db_path=db) + service.rpc = _FakeRPC() + service.runtime.rpc = service.rpc + service.local_profiles = lambda: ("default", "ops") + service.create_room( + room_id="room-1", + name="Bounded room", + members=[ + {"member_id": "default", "profile": "default", "handle": "hermes"}, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ], + ) + service.start() + service.send( + room_id="room-1", + event_id="user-first", + payload={"text": "@ops old", "thread_id": "thread-1"}, + ) + _wait_for( + lambda: any( + event["kind"] == "room.activity" + for event in service._events("room-1") + ) + ) + assert service.stop(timeout=1.0) + for index in range(24): + _append_room_event( + db, + room_id="room-1", + event_id=f"user-tail-{index}", + kind="message.user", + actor={"kind": "user", "id": "desktop"}, + payload={"text": f"tail {index}", "thread_id": "thread-1"}, + ) + + room = hosted_rooms.room_state(db, room_id="room-1") + snapshot = service._policy_snapshot(room) + assert len(snapshot.events) == 24 + assert {event["kind"] for event in snapshot.events} == {"message.user"} + discussion.plan_next_task( + room, + snapshot.events, + local_profiles=service.local_profiles(), + initial_watermarks=snapshot.watermarks, + ) + + +def test_service_uses_low_idle_poll_with_immediate_wakeup(tmp_path: Path): + service = HostedRoomService(_server(), db_path=tmp_path / "state.db") + + assert service.runtime.poll_interval_seconds == 5.0 + assert service.runtime.active_poll_interval_seconds == 0.25 + assert service.runtime.turn_timeout_seconds == 1830.0 + service.runtime._wake.clear() + service.wakeup() + assert service.runtime._wake.is_set() + + +def test_service_derives_room_deadline_from_agent_timeout(tmp_path: Path, monkeypatch): + monkeypatch.setenv("HERMES_AGENT_TIMEOUT", "90") + + service = HostedRoomService(_server(), db_path=tmp_path / "state.db") + + assert service.runtime.turn_timeout_seconds == 120.0 + + +def test_service_publishes_deferred_turn_continues_and_retries_new_generation( + tmp_path: Path, +): + now = [100.0] + + def clock(): + return now[0] + + db = tmp_path / "state.db" + service = HostedRoomService(_server(), db_path=db) + service.rpc = _FakeRPC() + service.runtime.rpc = service.rpc + service.runtime.clock = clock + service.runtime.lease_ttl_seconds = 30 + service.runtime.indeterminate_defer_seconds = 5 + service.local_profiles = lambda: ("default", "ops") + service.create_room( + room_id="room-1", + name="Resilient room", + members=[ + {"member_id": "default", "profile": "default", "handle": "default"}, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ], + ) + service.send( + room_id="room-1", + event_id="user-resilience", + payload={"text": "Check this", "thread_id": "thread-1"}, + ) + first = driver.list_tasks(db, room_id="room-1", status="queued")[0] + old_lease = driver.acquire_lease( + db, + room_id="room-1", + gateway_id=service.bindings()[0].gateway_id, + authority_epoch=1, + process_generation="offline-member", + ttl_seconds=1, + clock=clock, + ) + old_attempt = driver.start_task( + db, + first["identity"], + old_lease, + expected_cancel_generation=0, + clock=clock, + ) + + now[0] = 102.0 + binding = service.bindings()[0] + service.runtime._process_room(binding) + now[0] = 108.0 + service.runtime._process_room(binding) + + events = service._events("room-1") + deferred = next(event for event in events if event["kind"] == "turn.deferred") + assert deferred["payload"]["task_id"] == first["identity"].task_id + assert deferred["payload"]["execution_generation"] == 1 + assert any( + event["kind"] == "message.member" and event["payload"]["member_id"] == "ops" + for event in events + ) + + requeued = service.retry_room_task( + "room-1", + task_id=first["identity"].task_id, + ) + assert requeued["status"] == "queued" + lease = service.runtime._leases["room-1"] + retried = driver.start_task( + db, + first["identity"], + lease, + expected_cancel_generation=0, + clock=clock, + ) + assert retried.execution_generation == old_attempt.execution_generation + 1 + + +def test_stop_fence_prevents_the_next_room_member_from_starting( + tmp_path: Path, monkeypatch +): + db = tmp_path / "state.db" + service = HostedRoomService(_server(), db_path=db) + monkeypatch.setattr(service, "local_profiles", lambda: ("default", "ops")) + service.create_room( + room_id="room-1", + name="Release room", + members=[ + {"member_id": "default", "profile": "default", "handle": "hermes"}, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ], + ) + service.send( + room_id="room-1", + event_id="user-1", + payload={"text": "Inspect the release", "thread_id": "thread-1"}, + ) + assert len(driver.list_tasks(db, room_id="room-1")) == 1 + + assert service.stop_room("room-1", cancel_id="stop-1") == 1 + service.prepare_room(service.bindings()[0]) + + tasks = driver.list_tasks(db, room_id="room-1") + assert len(tasks) == 1 + assert tasks[0]["status"] == "cancelled" + assert any( + event["kind"] == "room.stop_requested" for event in service._events("room-1") + ) + + +def test_acknowledged_stop_refuses_to_disband_while_exact_turn_is_still_running( + tmp_path: Path, +): + class PendingStopRPC(_FakeRPC): + def __init__(self) -> None: + super().__init__() + self.active_task_id = None + + def info(self, *, profile, session_id, source): + return {"active": True, "task_id": self.active_task_id} + + def interrupt(self, *, profile, session_id, source, expected_task_id): + return None + + db = tmp_path / "state.db" + service = HostedRoomService(_server(), db_path=db) + rpc = PendingStopRPC() + service.rpc = rpc + service.runtime.rpc = rpc + service.local_profiles = lambda: ("default", "ops") + service.create_room( + room_id="room-1", + name="Release room", + members=[ + {"member_id": "default", "profile": "default", "handle": "hermes"}, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ], + ) + service.send( + room_id="room-1", + event_id="user-1", + payload={"text": "@ops inspect", "thread_id": "thread-1"}, + ) + task = driver.list_tasks(db, room_id="room-1", status="queued")[0] + binding = service.bindings()[0] + lease = driver.acquire_lease( + db, + room_id="room-1", + gateway_id=binding.gateway_id, + authority_epoch=binding.authority_epoch, + process_generation="worker", + ttl_seconds=30, + clock=time.time, + ) + driver.start_task( + db, + task["identity"], + lease, + expected_cancel_generation=0, + clock=time.time, + ) + rpc.sessions[("ops", "Group: room-1")] = {"session_id": "ops-session"} + rpc.active_task_id = task["identity"].task_id + + with pytest.raises(RuntimeError, match="still stopping"): + service.stop_room( + "room-1", + cancel_id="stop-1", + require_acknowledged=True, + ) + + stopping = driver.get_task(db, task["identity"]) + assert stopping["status"] == "stopping" + assert stopping["cancel_id"] == "stop-1" + + +def test_local_pending_approval_requires_exact_task_generation_and_request( + tmp_path: Path, +): + class ApprovalRPC(_FakeRPC): + def __init__(self) -> None: + super().__init__() + self.approvals = [] + + def approve(self, *, session_id, request_id, choice): + self.approvals.append((session_id, request_id, choice)) + return {"resolved": 1} + + db = tmp_path / "state.db" + service = HostedRoomService(_server(), db_path=db) + rpc = ApprovalRPC() + service.rpc = rpc + service.runtime.rpc = rpc + service.local_profiles = lambda: ("default", "ops") + service.create_room( + room_id="room-1", + name="Release room", + members=[ + {"member_id": "default", "profile": "default", "handle": "hermes"}, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ], + ) + service.send( + room_id="room-1", + event_id="user-1", + payload={"text": "@ops inspect", "thread_id": "thread-1"}, + ) + task = driver.list_tasks(db, room_id="room-1", status="queued")[0] + binding = service.bindings()[0] + lease = driver.acquire_lease( + db, + room_id="room-1", + gateway_id=binding.gateway_id, + authority_epoch=binding.authority_epoch, + process_generation="worker", + ttl_seconds=30, + clock=time.time, + ) + driver.start_task( + db, + task["identity"], + lease, + expected_cancel_generation=0, + clock=time.time, + ) + task = driver.get_task(db, task["identity"]) + service.runtime._report_pending_action( + task, + session_id="ops-session", + info={ + "pending_approval": { + "request_id": "approval-1", + "choices": ["once", "always", "deny"], + } + }, + ) + + action = service.status("room-1")["pending_actions"][0] + assert action["member_id"] == "ops" + assert action["approval"]["choices"] == ["once", "deny"] + with pytest.raises(RuntimeError, match="no longer pending"): + service.approve_room_task( + "room-1", + member_id="ops", + task_id=task["identity"].task_id, + execution_generation=1, + choice="once", + request_id="wrong-request", + ) + + assert service.approve_room_task( + "room-1", + member_id="ops", + task_id=task["identity"].task_id, + execution_generation=1, + choice="once", + request_id="approval-1", + ) == {"resolved": 1} + assert rpc.approvals == [("ops-session", "approval-1", "once")] + assert service.status("room-1")["pending_actions"] == [] diff --git a/tui_gateway/hosted_room_driver.py b/tui_gateway/hosted_room_driver.py new file mode 100644 index 0000000000..9fdeede8f2 --- /dev/null +++ b/tui_gateway/hosted_room_driver.py @@ -0,0 +1,1276 @@ +"""Runtime adapter for gateway-owned hosted room turns. + +The durable state machine lives in :mod:`gateway.hosted_room_driver`. This +module owns the process-local worker and a deliberately small, injected session +adapter. It does not import the gateway server, construct agents, or depend on +any client transport. + +The adapter normalizes existing internal session RPCs into seven methods. A +future server integration can implement those methods with the in-process +handlers while tests use deterministic fakes and no models or network. + +One bounded supervisor schedules independent room workers. Profile turn locks +still serialize Bots that share one profile, while a room waiting for approval +cannot stop unrelated rooms from progressing. Hosted member sessions +intentionally reuse ``Group: `` so a local-to-hosted migration +preserves the same canonical transcript instead of forking a second conversation. +""" + +from __future__ import annotations + +import contextlib +import threading +import time +import uuid +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any, ContextManager, Protocol, cast + +from gateway import hosted_room_driver as state + + +ROOM_SESSION_SOURCE = "bot_room" +MAX_TERMINAL_TEXT_BYTES = 64 * 1024 +_TERMINAL_TRUNCATION_NOTICE = ( + "\n\n[Reply truncated. Ask the Bot to share the full result as a file.]" +) + + +class InternalSessionRPC(Protocol): + """Normalized in-process session operations required by the room driver.""" + + def resolve_exact( + self, *, profile: str, title: str, source: str + ) -> Mapping[str, Any] | None: + """Return the exact titled session under ``profile``, if it exists.""" + + def create(self, *, profile: str, title: str, source: str) -> Mapping[str, Any]: + """Create a session without model or provider overrides.""" + + def resume( + self, *, profile: str, session_id: str, source: str + ) -> Mapping[str, Any]: + """Resume the canonical room session.""" + + def submit( + self, + *, + profile: str, + session_id: str, + prompt: str, + source: str, + task: state.TaskIdentity, + execution_generation: int, + on_terminal: Callable[[Mapping[str, Any]], None], + ) -> Mapping[str, Any]: + """Submit one fenced room turn and durably report its terminal result.""" + + def history( + self, *, profile: str, session_id: str, source: str + ) -> Sequence[Mapping[str, Any]]: + """Return normalized session messages in durable order.""" + + def info(self, *, profile: str, session_id: str, source: str) -> Mapping[str, Any]: + """Return normalized live status for a session.""" + + def interrupt( + self, + *, + profile: str, + session_id: str, + source: str, + expected_task_id: str, + ) -> Mapping[str, Any] | None: + """Interrupt only when the current turn still matches the expected task.""" + + +@dataclass(frozen=True) +class HostedRoomBinding: + """Current server-issued authority coordinate for one hosted room.""" + + room_id: str + gateway_id: str + authority_epoch: int + + +@dataclass(frozen=True) +class _TerminalReceipt: + status: state.TerminalStatus + settlement_id: str + result: dict[str, Any] + + +@dataclass(frozen=True) +class _RecoveryInspection: + terminal: _TerminalReceipt | None + active: bool + status: str | None + + +class HostedRoomRuntime: + """Run queued hosted-room tasks independently of Desktop connections.""" + + def __init__( + self, + *, + db_path: Path | str, + rooms: Iterable[HostedRoomBinding] | Callable[[], Iterable[HostedRoomBinding]], + turn_lock: Callable[[str], ContextManager[Any]], + rpc: InternalSessionRPC, + prepare_room: Callable[[HostedRoomBinding], None] | None = None, + publish_terminal: Callable[[HostedRoomBinding, Mapping[str, Any]], None] + | None = None, + pending_action: Callable[[str, str, Mapping[str, Any] | None], None] + | None = None, + clock: Callable[[], float] = time.time, + lease_ttl_seconds: float = 30.0, + poll_interval_seconds: float = 5.0, + active_poll_interval_seconds: float = 0.25, + turn_timeout_seconds: float = 1830.0, + indeterminate_defer_seconds: float = 60.0, + max_concurrent_rooms: int = 4, + process_generation: str | None = None, + ) -> None: + if lease_ttl_seconds <= 0: + raise ValueError("lease_ttl_seconds must be positive") + if poll_interval_seconds <= 0: + raise ValueError("poll_interval_seconds must be positive") + if active_poll_interval_seconds <= 0: + raise ValueError("active_poll_interval_seconds must be positive") + if turn_timeout_seconds <= 0: + raise ValueError("turn_timeout_seconds must be positive") + if indeterminate_defer_seconds <= 0: + raise ValueError("indeterminate_defer_seconds must be positive") + if ( + isinstance(max_concurrent_rooms, bool) + or not isinstance(max_concurrent_rooms, int) + or max_concurrent_rooms < 1 + ): + raise ValueError("max_concurrent_rooms must be a positive integer") + self.db_path = Path(db_path) + self.rpc = rpc + self.turn_lock = turn_lock + self.prepare_room = prepare_room + self.publish_terminal = publish_terminal + self.pending_action = pending_action + self.clock = clock + self.lease_ttl_seconds = float(lease_ttl_seconds) + self.poll_interval_seconds = float(poll_interval_seconds) + self.active_poll_interval_seconds = float(active_poll_interval_seconds) + self.turn_timeout_seconds = float(turn_timeout_seconds) + self.indeterminate_defer_seconds = float(indeterminate_defer_seconds) + self.max_concurrent_rooms = max_concurrent_rooms + self.process_generation = process_generation or uuid.uuid4().hex + self._rooms_provider: Callable[[], Iterable[HostedRoomBinding]] + if callable(rooms): + self._rooms_provider = cast( + Callable[[], Iterable[HostedRoomBinding]], rooms + ) + else: + room_bindings = tuple(rooms) + self._rooms_provider = lambda: room_bindings + + self._stop = threading.Event() + self._wake = threading.Event() + self._thread: threading.Thread | None = None + self._room_threads: dict[str, threading.Thread] = {} + self._rooms_needing_reschedule: set[str] = set() + self._leases: dict[str, state.DriverLease] = {} + self._recovered_leases: set[tuple[str, int]] = set() + self._inspected_indeterminate_attempts: set[tuple[str, str, int]] = set() + self._ambiguous_rooms: dict[str, float] = {} + self._blocked_rooms: set[str] = set() + self._status_lock = threading.Lock() + self._current_tasks: dict[str, state.TaskIdentity] = {} + self._room_schedule_cursor = 0 + self._last_error: str | None = None + self._cycles = 0 + + def start(self) -> None: + """Start the bounded room-worker supervisor idempotently.""" + with self._status_lock: + if self._thread is not None and self._thread.is_alive(): + return + self._stop.clear() + self._wake.set() + self._thread = threading.Thread( + target=self._worker_loop, + name="hosted-room-driver-supervisor", + daemon=True, + ) + self._thread.start() + + def stop(self, *, timeout: float = 5.0) -> bool: + """Request a bounded clean stop without interrupting accepted turns.""" + self._stop.set() + self._wake.set() + with self._status_lock: + thread = self._thread + if thread is None: + return True + deadline = time.monotonic() + max(0.0, timeout) + thread.join(max(0.0, deadline - time.monotonic())) + with self._status_lock: + room_threads = tuple(self._room_threads.values()) + for room_thread in room_threads: + room_thread.join(max(0.0, deadline - time.monotonic())) + return not thread.is_alive() and all( + not room_thread.is_alive() for room_thread in room_threads + ) + + def wakeup(self) -> None: + """Wake the worker after task admission or a room-state change.""" + with self._status_lock: + # If the supervisor observes this signal while a room still owns a + # worker slot, remember to revisit it once that thread exits. This + # closes the race between terminal publication/route repair and the + # longer idle fallback without turning idle rooms into a busy loop. + self._rooms_needing_reschedule.update(self._room_threads) + self._wake.set() + + def status(self) -> dict[str, Any]: + """Return a transport-neutral snapshot of runtime health.""" + with self._status_lock: + thread = self._thread + current_tasks = tuple(self._current_tasks.values()) + return { + "running": bool(thread and thread.is_alive()), + "stopping": self._stop.is_set(), + "process_generation": self.process_generation, + "current_task": current_tasks[0] if current_tasks else None, + "current_tasks": current_tasks, + "leased_rooms": tuple(sorted(self._leases)), + "blocked_rooms": tuple(sorted(self._blocked_rooms)), + "last_error": self._last_error, + "cycles": self._cycles, + } + + def cancel( + self, + identity: state.TaskIdentity, + *, + cancel_id: str, + ) -> dict[str, Any]: + """Persist a stop intent, then commit cancellation after acknowledgement.""" + before = state.get_task(self.db_path, identity) + if before["status"] in {"queued", "deferred"}: + cancelled = state.cancel_task( + self.db_path, + identity, + cancel_id=cancel_id, + expected_cancel_generation=before["cancel_generation"], + clock=self.clock, + ) + self.wakeup() + return cancelled + + stopping = state.begin_task_cancel( + self.db_path, + identity, + cancel_id=cancel_id, + expected_cancel_generation=before["cancel_generation"], + clock=self.clock, + ) + binding = self._binding_for_room(identity.room_id) + try: + if binding is not None and self._interrupt_stopping_task(binding, stopping): + stopping = state.complete_task_cancel( + self.db_path, + identity, + cancel_id=cancel_id, + expected_cancel_generation=stopping["cancel_generation"], + clock=self.clock, + ) + except Exception as exc: + self._record_error(f"stop remains pending: {exc}") + stopping = state.get_task(self.db_path, identity) + self.wakeup() + return stopping + + def retry_indeterminate(self, identity: state.TaskIdentity) -> dict[str, Any]: + """Explicitly retry one uncertain attempt under the current room lease.""" + task = state.get_task(self.db_path, identity) + if task["status"] not in {"indeterminate", "deferred"}: + raise state.InvalidTaskTransitionError( + f"cannot retry task in state '{task['status']}'" + ) + binding = self._binding_for_room(identity.room_id) + if binding is None: + raise state.RoomUnavailableError("hosted room is unavailable") + lease = self._ensure_lease(binding) + if task["status"] == "deferred": + retried = state.requeue_deferred_task( + self.db_path, + identity, + lease, + expected_execution_generation=task["execution_generation"], + expected_cancel_generation=task["cancel_generation"], + clock=self.clock, + ) + with self._status_lock: + self._blocked_rooms.discard(identity.room_id) + self.wakeup() + return retried + inspection = self._inspect_local_recovery_session(task) + if inspection.active: + with self._status_lock: + self._blocked_rooms.add(identity.room_id) + raise state.InvalidTaskTransitionError( + "cannot retry while the original task attempt is still active" + ) + retried = state.requeue_indeterminate_task( + self.db_path, + identity, + lease, + expected_execution_generation=task["execution_generation"], + expected_cancel_generation=task["cancel_generation"], + clock=self.clock, + ) + with self._status_lock: + self._blocked_rooms.discard(identity.room_id) + self.wakeup() + return retried + + def _interrupt_stopping_task( + self, + binding: HostedRoomBinding, + task: Mapping[str, Any], + ) -> bool: + transport = self.rpc + if transport is None: + return False + profile = task["payload"]["target_profile"] + session = transport.resolve_exact( + profile=profile, + title=room_session_title(binding.room_id), + source=ROOM_SESSION_SOURCE, + ) + if session is None: + # A local accepted turn cannot survive without its canonical + # session. Resolution errors raise; an authoritative absence is a + # safe Stop acknowledgement. + return True + resumed = transport.resume( + profile=profile, + session_id=_session_id(session), + source=ROOM_SESSION_SOURCE, + ) + session_id = _session_id(resumed) + info = transport.info( + profile=profile, + session_id=session_id, + source=ROOM_SESSION_SOURCE, + ) + active = bool(info.get("active", info.get("running", False))) + if not active: + # History was checked immediately before this probe. An exact + # local session that is no longer active cannot keep executing, and + # after a restart its process-local task marker is expected to be + # absent. + return True + if not _info_is_active_for(info, task["identity"], require_exact=True): + return False + result = transport.interrupt( + profile=profile, + session_id=session_id, + source=ROOM_SESSION_SOURCE, + expected_task_id=task["identity"].task_id, + ) + if result is None: + return False + return result.get("interrupted") is True or str(result.get("status") or "") in { + "cancelled", + "interrupted", + "stopping", + } + + def _settle_stopping_completion( + self, + binding: HostedRoomBinding, + task: Mapping[str, Any], + lease: state.DriverLease, + ) -> bool: + """Publish a terminal receipt that arrived before Stop was acknowledged.""" + transport = self.rpc + if transport is None: + return False + profile = task["payload"]["target_profile"] + session = transport.resolve_exact( + profile=profile, + title=room_session_title(binding.room_id), + source=ROOM_SESSION_SOURCE, + ) + if session is None: + return False + resumed = transport.resume( + profile=profile, + session_id=_session_id(session), + source=ROOM_SESSION_SOURCE, + ) + receipt = _find_terminal_receipt( + transport.history( + profile=profile, + session_id=_session_id(resumed), + source=ROOM_SESSION_SOURCE, + ), + task["identity"], + int(task["execution_generation"]), + ) + if receipt is None: + return False + settled = state.settle_stopping_task( + self.db_path, + task["identity"], + lease, + expected_execution_generation=int(task["execution_generation"]), + expected_cancel_generation=int(task["cancel_generation"]), + settlement_id=receipt.settlement_id, + status=receipt.status, + result=receipt.result, + clock=self.clock, + ) + if self.publish_terminal is not None: + self.publish_terminal(binding, settled) + return True + + def _report_pending_action( + self, + task: Mapping[str, Any], + *, + session_id: str, + info: Mapping[str, Any], + ) -> None: + if self.pending_action is None: + return + payload = task.get("payload") or {} + member_id = str( + payload.get("target_member_id") or payload.get("target_profile") or "" + ) + approval = info.get("pending_approval") or info.get("approval") + action = None + if isinstance(approval, Mapping): + safe_approval = dict(approval) + choices = [ + choice + for choice in safe_approval.get("choices") or () + if choice in {"once", "deny"} + ] + safe_approval["choices"] = choices or ["once", "deny"] + action = { + "kind": "approval", + "task_id": task["identity"].task_id, + "execution_generation": int(task["execution_generation"]), + "run_id": info.get("run_id"), + "session_id": session_id, + "request_id": safe_approval.get("request_id"), + "approval": safe_approval, + } + self.pending_action(task["identity"].room_id, member_id, action) + + def _retry_stopping_tasks( + self, binding: HostedRoomBinding, lease: state.DriverLease + ) -> bool: + pending = state.list_tasks( + self.db_path, + room_id=binding.room_id, + status="stopping", + ) + for task in pending: + try: + lease = self._renew_lease_if_needed(binding, lease) + if self._settle_stopping_completion(binding, task, lease): + continue + if not self._interrupt_stopping_task(binding, task): + return True + self._complete_acknowledged_stop(binding, task, lease) + except Exception as exc: + self._record_error(f"stop retry remains pending: {exc}") + return True + return False + + def _worker_loop(self) -> None: + try: + while not self._stop.is_set(): + # Clear before work so a write racing the cycle remains set and + # causes an immediate follow-up pass rather than being lost. + self._wake.clear() + try: + self._run_cycle() + except Exception as exc: # keep independent rooms serviceable + self._record_error(f"worker cycle failed: {exc}") + with self._status_lock: + self._cycles += 1 + self._wake.wait(self.poll_interval_seconds) + finally: + while True: + with self._status_lock: + room_threads = tuple( + thread + for thread in self._room_threads.values() + if thread.is_alive() + ) + if not room_threads: + break + for room_thread in room_threads: + room_thread.join(self.active_poll_interval_seconds) + self._release_idle_leases() + + def _run_cycle(self) -> None: + with self._status_lock: + supervisor = self._thread + if threading.current_thread() is not supervisor: + for binding in tuple(self._rooms_provider()): + if self._stop.is_set(): + return + self._run_room_once(binding) + return + + with self._status_lock: + self._room_threads = { + room_id: thread + for room_id, thread in self._room_threads.items() + if thread.is_alive() + } + available = self.max_concurrent_rooms - len(self._room_threads) + active_rooms = set(self._room_threads) + if available <= 0: + return + + bindings = tuple(self._rooms_provider()) + if not bindings: + return + start = self._room_schedule_cursor % len(bindings) + ordered_bindings = bindings[start:] + bindings[:start] + self._room_schedule_cursor = (start + 1) % len(bindings) + + for binding in ordered_bindings: + if self._stop.is_set() or available <= 0: + return + if binding.room_id in active_rooms: + continue + room_thread = threading.Thread( + target=self._run_room_once, + args=(binding,), + name=f"hosted-room-{binding.room_id[:24]}", + daemon=True, + ) + with self._status_lock: + self._room_threads[binding.room_id] = room_thread + active_rooms.add(binding.room_id) + available -= 1 + room_thread.start() + + def _run_room_once(self, binding: HostedRoomBinding) -> None: + try: + self._process_room(binding) + except state.LeaseHeldError: + return + except (state.RoomUnavailableError, state.StaleLeaseError) as exc: + self._drop_lease(binding.room_id) + with self._status_lock: + self._blocked_rooms.discard(binding.room_id) + self._record_error(f"room {binding.room_id}: {exc}") + except Exception as exc: + self._record_error(f"room {binding.room_id}: {exc}") + finally: + current = threading.current_thread() + with self._status_lock: + if self._room_threads.get(binding.room_id) is current: + self._room_threads.pop(binding.room_id, None) + should_wake = binding.room_id in self._rooms_needing_reschedule + self._rooms_needing_reschedule.discard(binding.room_id) + if should_wake: + self.wakeup() + + def _process_room(self, binding: HostedRoomBinding) -> None: + if self.prepare_room is not None: + self.prepare_room(binding) + self._inspect_abandoned_attempts(binding) + deferred_until = self._ambiguous_rooms.get(binding.room_id) + if deferred_until is not None: + running = state.list_tasks( + self.db_path, + room_id=binding.room_id, + status="running", + ) + if not running: + self._ambiguous_rooms.pop(binding.room_id, None) + elif self.clock() < deferred_until: + return + else: + self._ambiguous_rooms.pop(binding.room_id, None) + lease = self._ensure_lease(binding) + recovery_key = (lease.room_id, lease.lease_generation) + if recovery_key not in self._recovered_leases: + state.recover_room(self.db_path, lease, clock=self.clock) + self._recovered_leases.add(recovery_key) + if self._retry_stopping_tasks(binding, lease): + with self._status_lock: + self._blocked_rooms.add(binding.room_id) + return + if self._reconcile_indeterminate(binding, lease): + return + + queued = state.list_tasks( + self.db_path, + room_id=binding.room_id, + status="queued", + ) + for task in queued: + if self._stop.is_set(): + return + lease = self._renew_lease_if_needed(binding, lease) + attempt = state.start_task( + self.db_path, + task["identity"], + lease, + expected_cancel_generation=task["cancel_generation"], + clock=self.clock, + ) + self._execute_attempt(binding, task, attempt) + + def _ensure_lease(self, binding: HostedRoomBinding) -> state.DriverLease: + with self._status_lock: + current = self._leases.get(binding.room_id) + if current is not None: + try: + renewed = self._renew_lease_if_needed(binding, current) + except state.StaleLeaseError: + self._drop_lease(binding.room_id) + else: + return renewed + + lease = state.acquire_lease( + self.db_path, + room_id=binding.room_id, + gateway_id=binding.gateway_id, + authority_epoch=binding.authority_epoch, + process_generation=self.process_generation, + ttl_seconds=self.lease_ttl_seconds, + clock=self.clock, + ) + with self._status_lock: + self._leases[binding.room_id] = lease + self._recovered_leases = { + key for key in self._recovered_leases if key[0] != binding.room_id + } + return lease + + def _renew_lease_if_needed( + self, + binding: HostedRoomBinding, + lease: state.DriverLease, + *, + force: bool = False, + ) -> state.DriverLease: + del binding + renew_at = lease.expires_at - (self.lease_ttl_seconds / 2) + if not force and self.clock() < renew_at: + return lease + renewed = state.renew_lease( + self.db_path, + lease, + ttl_seconds=self.lease_ttl_seconds, + clock=self.clock, + ) + with self._status_lock: + self._leases[lease.room_id] = renewed + return renewed + + def _execute_attempt( + self, + binding: HostedRoomBinding, + task: Mapping[str, Any], + attempt: state.TaskAttempt, + ) -> None: + profile = task["payload"]["target_profile"] + transport = self.rpc + submit_attempted = False + with self._status_lock: + self._current_tasks[binding.room_id] = attempt.identity + try: + with self.turn_lock(profile): + session = self._resolve_or_create(transport, profile, binding.room_id) + # An in-process submit should fail before admission or return + # after it, but an unexpected exception at that boundary is + # still ambiguous. Never terminalize it as a proven failure. + submit_attempted = True + + def on_terminal(receipt: Mapping[str, Any]) -> None: + status = receipt.get("status") + if status == "cancelled": + self.wakeup() + return + terminal_status: state.TerminalStatus = ( + "settled" if status == "settled" else "failed" + ) + try: + settled = state.settle_task( + self.db_path, + attempt, + settlement_id=( + receipt.get("settlement_id") + or f"reply:{attempt.identity.task_id}:{attempt.execution_generation}" + ), + status=terminal_status, + result=_bounded_terminal_result(receipt), + clock=self.clock, + ) + if self.publish_terminal is not None: + self.publish_terminal(binding, settled) + except state.StaleTaskError: + try: + current = state.get_task(self.db_path, attempt.identity) + if current["status"] == "stopping": + settled = state.settle_stopping_task( + self.db_path, + attempt.identity, + attempt.lease, + expected_execution_generation=attempt.execution_generation, + expected_cancel_generation=int( + current["cancel_generation"] + ), + settlement_id=( + receipt.get("settlement_id") + or f"reply:{attempt.identity.task_id}:{attempt.execution_generation}" + ), + status=terminal_status, + result=_bounded_terminal_result(receipt), + clock=self.clock, + ) + if self.publish_terminal is not None: + self.publish_terminal(binding, settled) + except (state.StaleLeaseError, state.StaleTaskError): + pass + except state.StaleLeaseError: + # Cancellation, disband, or authority transfer won the + # durable race. The model result is intentionally + # discarded; never turn a correct fence into a worker + # thread exception. + pass + except state.DriverStateError as exc: + # A malformed terminal receipt must not escape the + # callback and hold the profile lock until the deadline. + self._settle_failure_if_current( + attempt, + RuntimeError( + f"terminal result could not be committed: {exc}" + ), + ) + self.wakeup() + + deadline_monotonic = time.monotonic() + self.turn_timeout_seconds + transport.submit( + profile=profile, + session_id=_session_id(session), + prompt=task["payload"]["prompt"], + source=ROOM_SESSION_SOURCE, + task=attempt.identity, + execution_generation=attempt.execution_generation, + on_terminal=on_terminal, + ) + receipt = self._wait_for_terminal( + binding, + task=task, + profile=profile, + session_id=_session_id(session), + attempt=attempt, + transport=transport, + deadline_monotonic=deadline_monotonic, + ) + if receipt is None: + return + state.settle_task( + self.db_path, + attempt, + settlement_id=receipt.settlement_id, + status=receipt.status, + result=receipt.result, + clock=self.clock, + ) + except (state.StaleLeaseError, state.StaleTaskError) as exc: + self._drop_lease(binding.room_id) + self._record_error(f"task {attempt.identity.task_id} fenced: {exc}") + except Exception as exc: + if submit_attempted: + self._drop_lease(binding.room_id) + self._ambiguous_rooms[binding.room_id] = attempt.lease.expires_at + self._record_error( + f"task {attempt.identity.task_id} observation failed after submit: {exc}" + ) + else: + self._settle_failure_if_current(attempt, exc) + finally: + with self._status_lock: + self._current_tasks.pop(binding.room_id, None) + # The task may have published a reply, deferred a member, or + # exposed the next turn while this room thread still occupied + # its slot. Schedule exactly one immediate follow-up after the + # thread leaves; idle room scans never set this marker. + self._rooms_needing_reschedule.add(binding.room_id) + + def _wait_for_terminal( + self, + binding: HostedRoomBinding, + *, + task: Mapping[str, Any], + profile: str, + session_id: str, + attempt: state.TaskAttempt, + transport: InternalSessionRPC, + deadline_monotonic: float, + ) -> _TerminalReceipt | None: + lease = attempt.lease + while not self._stop.is_set(): + task = state.get_task(self.db_path, attempt.identity) + if task["status"] in state.TERMINAL_STATUSES: + return None + if task["status"] == "stopping": + try: + lease = self._renew_lease_if_needed(binding, lease) + if self._settle_stopping_completion(binding, task, lease): + return None + if self._interrupt_stopping_task(binding, task): + self._complete_acknowledged_stop(binding, task, lease) + return None + except Exception as exc: + self._record_error(f"stop retry remains pending: {exc}") + self._wake.wait(self.active_poll_interval_seconds) + self._wake.clear() + continue + + if time.monotonic() >= deadline_monotonic: + self._expire_attempt_deadline(binding, task, lease) + return None + + lease = self._renew_lease_if_needed(binding, lease) + receipt = _find_terminal_receipt( + transport.history( + profile=profile, + session_id=session_id, + source=ROOM_SESSION_SOURCE, + ), + attempt.identity, + attempt.execution_generation, + ) + if receipt is not None: + return receipt + + info = transport.info( + profile=profile, + session_id=session_id, + source=ROOM_SESSION_SOURCE, + ) + self._report_pending_action(task, session_id=session_id, info=info) + remaining = max(0.0, deadline_monotonic - time.monotonic()) + self._wake.wait(min(self.active_poll_interval_seconds, remaining)) + self._wake.clear() + return None + + @staticmethod + def _deadline_cancel_id(task: Mapping[str, Any]) -> str: + return f"deadline:{int(task['execution_generation'])}" + + @staticmethod + def _is_deadline_stop(task: Mapping[str, Any]) -> bool: + return str(task.get("cancel_id") or "").startswith("deadline:") + + def _settle_deadline_failure( + self, + binding: HostedRoomBinding, + task: Mapping[str, Any], + lease: state.DriverLease, + ) -> dict[str, Any]: + """Publish the explicit terminal outcome after exact Stop acknowledgement.""" + + execution_generation = int(task["execution_generation"]) + settled = state.settle_stopping_task( + self.db_path, + task["identity"], + lease, + expected_execution_generation=execution_generation, + expected_cancel_generation=int(task["cancel_generation"]), + settlement_id=f"deadline:{execution_generation}", + status="failed", + result={ + "error": ( + "This Group Chat turn exceeded its configured time limit and " + "was stopped." + ), + "reason_code": "turn_deadline_exceeded", + "timeout_seconds": self.turn_timeout_seconds, + }, + clock=self.clock, + ) + if self.publish_terminal is not None: + self.publish_terminal(binding, settled) + return settled + + def _complete_acknowledged_stop( + self, + binding: HostedRoomBinding, + task: Mapping[str, Any], + lease: state.DriverLease, + ) -> dict[str, Any]: + if self._is_deadline_stop(task): + return self._settle_deadline_failure(binding, task, lease) + return state.complete_task_cancel( + self.db_path, + task["identity"], + cancel_id=task["cancel_id"], + expected_cancel_generation=task["cancel_generation"], + clock=self.clock, + ) + + def _expire_attempt_deadline( + self, + binding: HostedRoomBinding, + task: Mapping[str, Any], + lease: state.DriverLease, + ) -> None: + """Fence, stop, and terminalize one exact attempt at its deadline.""" + + if task["status"] == "running": + task = state.begin_task_cancel( + self.db_path, + task["identity"], + cancel_id=self._deadline_cancel_id(task), + expected_cancel_generation=int(task["cancel_generation"]), + clock=self.clock, + ) + elif task["status"] != "stopping": + return + + # A user Stop that won the race keeps its own cancellation semantics. + if not self._is_deadline_stop(task): + return + lease = self._renew_lease_if_needed(binding, lease, force=True) + if self._settle_stopping_completion(binding, task, lease): + return + if self._interrupt_stopping_task(binding, task): + self._complete_acknowledged_stop(binding, task, lease) + return + self._record_error( + f"task {task['identity'].task_id} exceeded its deadline; stop remains pending" + ) + + def _inspect_abandoned_attempts(self, binding: HostedRoomBinding) -> None: + running = state.list_tasks( + self.db_path, + room_id=binding.room_id, + status="running", + ) + for task in running: + if task["run_process_generation"] == self.process_generation: + continue + inspection = self._inspect_local_recovery_session(task) + if inspection.terminal is not None: + self._harvest_previous_attempt(binding, task, inspection.terminal) + elif inspection.active: + # The prior session still owns the turn. Do not contend for its + # lease or submit a duplicate prompt. + raise state.LeaseHeldError("recovered session turn is still active") + + def _inspect_local_recovery_session( + self, + task: Mapping[str, Any], + ) -> _RecoveryInspection: + """Check only live process state before explicit local recovery. + + A restart loses the in-process terminal callback identity. The ordinary + session history is a display projection and cannot prove which durable + task attempt authored a row, so never hydrate or infer completion from + it. An inactive abandoned attempt remains indeterminate until the user + explicitly retries it under a new fenced generation. + """ + + profile = task["payload"]["target_profile"] + with self.turn_lock(profile): + session = self.rpc.resolve_exact( + profile=profile, + title=room_session_title(task["identity"].room_id), + source=ROOM_SESSION_SOURCE, + ) + if session is None: + return _RecoveryInspection(terminal=None, active=False, status=None) + session_id = _session_id(session) + info = self.rpc.info( + profile=profile, + session_id=session_id, + source=ROOM_SESSION_SOURCE, + ) + self._report_pending_action(task, session_id=session_id, info=info) + return _RecoveryInspection( + terminal=None, + active=_info_is_active_for(info, task["identity"]), + status=str(info.get("status") or "") or None, + ) + + def _reconcile_indeterminate( + self, + binding: HostedRoomBinding, + lease: state.DriverLease, + ) -> bool: + unresolved = state.list_tasks( + self.db_path, + room_id=binding.room_id, + status="indeterminate", + ) + if not unresolved: + with self._status_lock: + self._blocked_rooms.discard(binding.room_id) + return False + for task in unresolved: + generation = int(task["execution_generation"]) + attempt_key = ( + binding.room_id, + task["identity"].task_id, + generation, + ) + if attempt_key not in self._inspected_indeterminate_attempts: + inspection = self._inspect_local_recovery_session(task) + self._inspected_indeterminate_attempts.add(attempt_key) + if inspection.terminal is not None: + resolved = state.resolve_indeterminate_task( + self.db_path, + task["identity"], + lease, + expected_execution_generation=generation, + expected_cancel_generation=task["cancel_generation"], + settlement_id=inspection.terminal.settlement_id, + status=inspection.terminal.status, + result=inspection.terminal.result, + clock=self.clock, + ) + self._inspected_indeterminate_attempts.discard(attempt_key) + if self.publish_terminal is not None: + self.publish_terminal(binding, resolved) + continue + if inspection.active: + with self._status_lock: + self._blocked_rooms.add(binding.room_id) + return True + deferred_at = float( + task.get("indeterminate_at") + or task.get("updated_at") + or task.get("created_at") + or self.clock() + ) + if self.clock() < deferred_at + self.indeterminate_defer_seconds: + with self._status_lock: + self._blocked_rooms.add(binding.room_id) + return True + deferred = state.defer_indeterminate_task( + self.db_path, + task["identity"], + lease, + expected_execution_generation=task["execution_generation"], + expected_cancel_generation=task["cancel_generation"], + reason="member_unavailable", + clock=self.clock, + ) + self._report_pending_action(task, session_id="", info={}) + if self.publish_terminal is not None: + self.publish_terminal(binding, deferred) + with self._status_lock: + self._blocked_rooms.discard(binding.room_id) + return False + + def _harvest_previous_attempt( + self, + binding: HostedRoomBinding, + task: Mapping[str, Any], + receipt: _TerminalReceipt, + ) -> None: + previous_lease = state.DriverLease( + room_id=binding.room_id, + gateway_id=task["run_gateway_id"], + authority_epoch=binding.authority_epoch, + process_generation=task["run_process_generation"], + lease_generation=task["run_lease_generation"], + expires_at=0.0, + ) + previous_attempt = state.TaskAttempt( + identity=task["identity"], + lease=previous_lease, + execution_generation=task["execution_generation"], + cancel_generation=task["cancel_generation"], + ) + try: + state.settle_task( + self.db_path, + previous_attempt, + settlement_id=receipt.settlement_id, + status=receipt.status, + result=receipt.result, + clock=self.clock, + ) + except (state.StaleLeaseError, state.StaleTaskError): + # Once the previous proof has expired, the current state contract + # deliberately offers no unsafe "trust this historical output" + # escape hatch. The subsequent fenced recovery leaves it + # indeterminate for explicit user action. + return + + def _binding_for_room(self, room_id: str) -> HostedRoomBinding | None: + return next( + ( + binding + for binding in self._rooms_provider() + if binding.room_id == room_id + ), + None, + ) + + def _resolve_or_create( + self, + transport: InternalSessionRPC, + profile: str, + room_id: str, + ) -> Mapping[str, Any]: + title = room_session_title(room_id) + session = transport.resolve_exact( + profile=profile, + title=title, + source=ROOM_SESSION_SOURCE, + ) + if session is None: + return transport.create( + profile=profile, + title=title, + source=ROOM_SESSION_SOURCE, + ) + return transport.resume( + profile=profile, + session_id=_session_id(session), + source=ROOM_SESSION_SOURCE, + ) + + def _settle_failure_if_current( + self, attempt: state.TaskAttempt, exc: Exception + ) -> None: + try: + state.settle_task( + self.db_path, + attempt, + settlement_id=f"failure:{attempt.identity.task_id}:{attempt.execution_generation}", + status="failed", + result={"error": str(exc)}, + clock=self.clock, + ) + except (state.DriverStateError, state.RoomUnavailableError): + pass + self._record_error(f"task {attempt.identity.task_id} failed: {exc}") + + def _record_error(self, message: str) -> None: + with self._status_lock: + self._last_error = message + + def _drop_lease(self, room_id: str) -> None: + with self._status_lock: + self._leases.pop(room_id, None) + + def _release_idle_leases(self) -> None: + for room_id, lease in tuple(self._leases.items()): + try: + state.release_lease(self.db_path, lease, clock=self.clock) + except state.DriverStateError: + continue + self._drop_lease(room_id) + + +def room_session_title(room_id: str) -> str: + """Return the canonical hidden session title for one hosted room.""" + return f"Group: {room_id}" + + +def _session_id(session: Mapping[str, Any]) -> str: + value = session.get("session_id", session.get("id")) + if not isinstance(value, str) or not value: + raise ValueError("session adapter returned no session_id") + return value + + +def _truncate_utf8(value: Any, *, max_bytes: int) -> tuple[str, bool]: + text = str(value or "") + encoded = text.encode("utf-8") + if len(encoded) <= max_bytes: + return text, False + suffix = _TERMINAL_TRUNCATION_NOTICE.encode("utf-8") + prefix = encoded[: max(0, max_bytes - len(suffix))] + while prefix: + try: + return prefix.decode("utf-8") + _TERMINAL_TRUNCATION_NOTICE, True + except UnicodeDecodeError: + prefix = prefix[:-1] + return _TERMINAL_TRUNCATION_NOTICE.strip(), True + + +def _bounded_terminal_result(receipt: Mapping[str, Any]) -> dict[str, Any]: + text, truncated = _truncate_utf8( + receipt.get("text", ""), + max_bytes=MAX_TERMINAL_TEXT_BYTES, + ) + error, error_truncated = _truncate_utf8( + receipt.get("error", ""), + max_bytes=4096, + ) + return { + "message_id": receipt.get("message_id"), + "text": text, + **({"error": error} if error else {}), + **({"truncated": True} if truncated or error_truncated else {}), + } + + +def _find_terminal_receipt( + history: Sequence[Mapping[str, Any]], + identity: state.TaskIdentity, + execution_generation: int, +) -> _TerminalReceipt | None: + for message in reversed(history): + if message.get("task_id") != identity.task_id: + continue + if message.get("execution_generation") != execution_generation: + continue + if message.get("role") != "assistant": + continue + status = message.get("status") + if status not in {"settled", "failed"}: + continue + terminal_status = cast(state.TerminalStatus, status) + receipt_id = message.get("message_id") + if not isinstance(receipt_id, str) or not receipt_id: + receipt_id = f"reply:{identity.task_id}:{execution_generation}" + return _TerminalReceipt( + status=terminal_status, + settlement_id=receipt_id, + result=_bounded_terminal_result( + { + "message_id": receipt_id, + "text": message.get("content", ""), + } + ), + ) + return None + + +def _info_is_active_for( + info: Mapping[str, Any], + identity: state.TaskIdentity, + *, + require_exact: bool = False, +) -> bool: + if not bool(info.get("active", info.get("running", False))): + return False + active_task_id = info.get("task_id") + if require_exact: + return active_task_id == identity.task_id + return active_task_id in {None, identity.task_id} + + +@contextlib.contextmanager +def null_turn_lock(_profile: str) -> Any: + """Provide an explicit no-op lock for narrow embedding tests.""" + yield diff --git a/tui_gateway/hosted_room_server_rpc.py b/tui_gateway/hosted_room_server_rpc.py new file mode 100644 index 0000000000..a28dcd65a1 --- /dev/null +++ b/tui_gateway/hosted_room_server_rpc.py @@ -0,0 +1,213 @@ +"""In-process session adapter for the hosted room driver. + +The room worker must not depend on a Desktop/WebSocket transport, but it should +still use the same session handlers as every other TUI/Desktop turn. This +adapter calls the installed handler registry directly and keeps the extra +task proof as an in-process-only Python object that JSON clients cannot forge. +""" + +from __future__ import annotations + +import itertools +import threading +from collections.abc import Mapping, Sequence +from types import ModuleType +from typing import Any, Callable + +from gateway import hosted_room_driver as state + + +class HostedRoomSessionError(RuntimeError): + """Raised when an in-process session operation is rejected.""" + + def __init__(self, method: str, code: int, message: str) -> None: + super().__init__(f"{method} failed: {message}") + self.method = method + self.code = code + + +class HostedRoomServerRPC: + """Normalize the installed server handlers for :class:`HostedRoomRuntime`.""" + + def __init__(self, server: ModuleType) -> None: + self.server = server + self._ids = itertools.count(1) + + def _call(self, method: str, params: dict[str, Any]) -> dict[str, Any]: + handler = self.server._methods[method] + envelope = handler(f"hosted-room-{next(self._ids)}", params) + error = envelope.get("error") if isinstance(envelope, dict) else None + if isinstance(error, dict): + raise HostedRoomSessionError( + method, + int(error.get("code") or 5000), + str(error.get("message") or "gateway rejected the request"), + ) + result = envelope.get("result") if isinstance(envelope, dict) else None + if not isinstance(result, dict): + raise HostedRoomSessionError(method, 5000, "gateway returned no result") + return result + + def resolve_exact( + self, *, profile: str, title: str, source: str + ) -> Mapping[str, Any] | None: + del source + result = self._call( + "session.list", + {"profile": profile, "title": title, "include_hidden": True}, + ) + rows = result.get("sessions") + if not isinstance(rows, list) or not rows: + return None + row = rows[0] + if not isinstance(row, dict): + return None + session_id = row.get("resolved_id") or row.get("id") + return {"session_id": session_id, "title": row.get("title") or title} + + def create(self, *, profile: str, title: str, source: str) -> Mapping[str, Any]: + return self._call( + "session.create", + { + "profile": profile, + "title": title, + "source": source, + "hidden": True, + "room_plumbing": True, + "follow_profile_config": True, + "close_on_disconnect": False, + }, + ) + + def resume( + self, *, profile: str, session_id: str, source: str + ) -> Mapping[str, Any]: + return self._call( + "session.resume", + { + "profile": profile, + "session_id": session_id, + "omit_messages": True, + "source": source, + }, + ) + + def submit( + self, + *, + profile: str, + session_id: str, + prompt: str, + source: str, + task: state.TaskIdentity, + execution_generation: int, + on_terminal: Callable[[Mapping[str, Any]], None], + ) -> Mapping[str, Any]: + try: + return self._call( + "prompt.submit", + { + "profile": profile, + "session_id": session_id, + "text": prompt, + "source": source, + "_hosted_task": { + "room_id": task.room_id, + "task_id": task.task_id, + "thread_id": task.thread_id, + "turn_id": task.turn_id, + "execution_generation": execution_generation, + }, + "_hosted_terminal_callback": on_terminal, + }, + ) + except HostedRoomSessionError as exc: + # In-process prompt.submit error envelopes are returned before the + # background turn is admitted. Preserve that proof so the driver + # can defer or requeue without waiting out an ambiguity lease. + exc.not_admitted = True + raise + + def history( + self, *, profile: str, session_id: str, source: str + ) -> Sequence[Mapping[str, Any]]: + del source + result = self._call( + "session.history", + {"profile": profile, "session_id": session_id}, + ) + rows = result.get("messages") + return tuple(row for row in rows if isinstance(row, dict)) if isinstance(rows, list) else () + + def _session_record(self, session_id: str) -> dict[str, Any] | None: + with self.server._sessions_lock: + record = self.server._sessions.get(session_id) + if record is not None: + return record + for candidate in self.server._sessions.values(): + if str(candidate.get("session_key") or "") == session_id: + return candidate + return None + + def info(self, *, profile: str, session_id: str, source: str) -> Mapping[str, Any]: + del profile, source + record = self._session_record(session_id) + if record is None: + return {"active": False, "task_id": None} + lock = record.get("history_lock") + if not isinstance(lock, type(threading.Lock())): + return {"active": bool(record.get("running")), "task_id": None} + with lock: + task = record.get("_hosted_room_task") + result = { + "active": bool(record.get("running")), + "task_id": task.get("task_id") if isinstance(task, dict) else None, + } + pending_reader = getattr( + self.server, "_pending_approval_request_payload", None + ) + pending = ( + pending_reader(str(record.get("session_key") or "")) + if callable(pending_reader) + else None + ) + if pending: + result["status"] = "waiting_for_approval" + result["pending_approval"] = pending + return result + + def approve( + self, + *, + session_id: str, + request_id: str, + choice: str, + ) -> Mapping[str, Any]: + """Resolve one exact local room approval without broad policy changes.""" + return self._call( + "approval.respond", + { + "session_id": session_id, + "request_id": request_id, + "choice": choice, + "all": False, + }, + ) + + def interrupt( + self, + *, + profile: str, + session_id: str, + source: str, + expected_task_id: str, + ) -> Mapping[str, Any] | None: + del source + return self._call( + "session.interrupt", + { + "profile": profile, + "session_id": session_id, + "expected_hosted_task_id": expected_task_id, + }, + ) diff --git a/tui_gateway/hosted_room_service.py b/tui_gateway/hosted_room_service.py new file mode 100644 index 0000000000..49dba1d2c6 --- /dev/null +++ b/tui_gateway/hosted_room_service.py @@ -0,0 +1,518 @@ +"""Production coordinator for same-gateway hosted Discussion rooms.""" + +from __future__ import annotations + +import contextlib +import os +import threading +import time +from collections import Counter +from collections.abc import Iterator, Mapping +from pathlib import Path +from types import ModuleType +from typing import Any + +from gateway import hosted_room_discussion as discussion +from gateway import hosted_room_driver as driver +from gateway import hosted_rooms +from gateway.hosted_room_policy_checkpoint import ( + HostedRoomPolicyCheckpoint, + PolicySnapshot, +) +from tui_gateway.hosted_room_driver import HostedRoomBinding, HostedRoomRuntime +from tui_gateway.hosted_room_server_rpc import HostedRoomServerRPC + + +_HOSTED_ROOM_IDLE_FALLBACK_SECONDS = 5.0 +_HOSTED_ROOM_ACTIVE_POLL_SECONDS = 0.25 +_HOSTED_ROOM_TERMINAL_GRACE_SECONDS = 30.0 + + +def _hosted_room_turn_timeout_seconds() -> float: + try: + agent_timeout = float(os.getenv("HERMES_AGENT_TIMEOUT", "1800")) + except (TypeError, ValueError): + agent_timeout = 1800.0 + if agent_timeout <= 0: + agent_timeout = 1800.0 + return agent_timeout + _HOSTED_ROOM_TERMINAL_GRACE_SECONDS + + +class HostedRoomService: + """Own the hosted Discussion policy and its transport-free worker.""" + + def __init__( + self, server: ModuleType, *, db_path: Path | str | None = None + ) -> None: + self.server = server + self.db_path = Path(db_path or hosted_rooms.default_db_path()) + hosted_rooms.prune_disbanded_rooms(self.db_path) + self._policy_lock = threading.RLock() + self._pending_actions: dict[tuple[str, str], dict[str, Any]] = {} + self.policy_checkpoint = HostedRoomPolicyCheckpoint(self.db_path) + self.rpc = HostedRoomServerRPC(server) + self.runtime = HostedRoomRuntime( + db_path=self.db_path, + rooms=self.bindings, + rpc=self.rpc, + turn_lock=self._turn_lock, + prepare_room=self.prepare_room, + publish_terminal=self.publish_terminal, + pending_action=self._set_pending_action, + poll_interval_seconds=_HOSTED_ROOM_IDLE_FALLBACK_SECONDS, + active_poll_interval_seconds=_HOSTED_ROOM_ACTIVE_POLL_SECONDS, + turn_timeout_seconds=_hosted_room_turn_timeout_seconds(), + ) + + @property + def root(self) -> Path: + return self.db_path.parent + + def local_profiles(self) -> tuple[str, ...]: + profiles = {"default"} + profiles_dir = self.root / "profiles" + if profiles_dir.is_dir(): + profiles.update( + path.name for path in profiles_dir.iterdir() if path.is_dir() + ) + return tuple(sorted(profiles)) + + def bindings(self) -> tuple[HostedRoomBinding, ...]: + local_gateway_id = hosted_rooms.local_authority_gateway_id() + return tuple( + HostedRoomBinding( + room_id=str(room["room_id"]), + gateway_id=str(room["authority_gateway_id"]), + authority_epoch=int(room["authority_epoch"]), + ) + for room in hosted_rooms.list_rooms(self.db_path) + if str(room["authority_gateway_id"]) == local_gateway_id + ) + + def _owned_room(self, room_id: str) -> dict[str, Any]: + room = hosted_rooms.room_state(self.db_path, room_id=room_id) + if str(room["authority_gateway_id"]) != ( + hosted_rooms.local_authority_gateway_id() + ): + raise hosted_rooms.AuthorityConflictError( + "This Group Chat is managed by another gateway." + ) + return room + + @contextlib.contextmanager + def _turn_lock(self, profile: str) -> Iterator[None]: + from tools.bot_relay import acquire_turn_lock + + with acquire_turn_lock(self.root, profile): + yield + + def start(self) -> None: + self.runtime.start() + + def stop(self, *, timeout: float = 5.0) -> bool: + return self.runtime.stop(timeout=timeout) + + def wakeup(self) -> None: + self.runtime.wakeup() + + def _set_pending_action( + self, + room_id: str, + member_id: str, + action: Mapping[str, Any] | None, + ) -> None: + key = (room_id, member_id) + with self._policy_lock: + if action is None: + self._pending_actions.pop(key, None) + else: + self._pending_actions[key] = {**action, "member_id": member_id} + + def _events(self, room_id: str) -> list[dict[str, Any]]: + events: list[dict[str, Any]] = [] + cursor = 0 + while True: + page = hosted_rooms.read_events( + self.db_path, + room_id=room_id, + since_seq=cursor, + limit=hosted_rooms.MAX_LOG_LIMIT, + ) + rows = page.get("events") + if isinstance(rows, list): + events.extend(row for row in rows if isinstance(row, dict)) + next_cursor = int(page.get("cursor") or cursor) + if not page.get("has_more"): + return events + if next_cursor <= cursor: + raise RuntimeError("hosted room replay cursor did not advance") + cursor = next_cursor + + def _append_plan(self, room_id: str, plan: discussion.PublicationPlan) -> None: + for event in plan.events: + hosted_rooms.append_event( + self.db_path, + **event.append_kwargs(room_id), + ) + + def _policy_snapshot(self, room: Mapping[str, Any]) -> PolicySnapshot: + return self.policy_checkpoint.snapshot( + room_id=str(room["room_id"]), + latest_seq=int(room["latest_seq"]), + ) + + def _publish_terminal_tasks( + self, + room: Mapping[str, Any], + ) -> bool: + changed = False + local_profiles = self.local_profiles() + for status in ("deferred", "settled", "failed", "cancelled"): + for task in driver.list_tasks( + self.db_path, + room_id=str(room["room_id"]), + status=status, + ): + identity = task["identity"] + if self.policy_checkpoint.publication_exists( + room_id=str(room["room_id"]), + task_id=identity.task_id, + status=status, + execution_generation=int(task["execution_generation"]), + ): + continue + task_events = self.policy_checkpoint.events_for_task( + room_id=str(room["room_id"]), + source_event_seq=int(task["payload"]["source_event_seq"]), + ) + plan = discussion.reconstruct_task_plan( + room, + task_events, + task, + local_profiles=local_profiles, + ) + publication = discussion.plan_publication( + room, + task_events, + plan, + status=status, + result=task.get("result"), + execution_generation=( + int(task["execution_generation"]) + if status == "deferred" + else None + ), + local_profiles=local_profiles, + ) + self._append_plan(str(room["room_id"]), publication) + changed = True + return changed + + def _append_room_status( + self, + room: Mapping[str, Any], + decision: discussion.DiscussionDecision, + ) -> None: + if decision.discussion_event_id is None: + return + hosted_rooms.append_event( + self.db_path, + room_id=str(room["room_id"]), + event_id=f"dactivity:{decision.discussion_event_id}:{decision.reason}", + kind="room.activity", + actor={"kind": "gateway", "id": str(room["authority_gateway_id"])}, + payload={ + "status": decision.status, + "reason_code": decision.reason, + "thread_id": decision.thread_id, + "discussion_event_id": decision.discussion_event_id, + }, + authority_gateway_id=str(room["authority_gateway_id"]), + authority_epoch=int(room["authority_epoch"]), + ) + + def prepare_room(self, binding: HostedRoomBinding) -> None: + with self._policy_lock: + room = hosted_rooms.room_state(self.db_path, room_id=binding.room_id) + snapshot = self._policy_snapshot(room) + events = list(snapshot.events) + if self._publish_terminal_tasks(room): + room = hosted_rooms.room_state( + self.db_path, + room_id=binding.room_id, + ) + snapshot = self._policy_snapshot(room) + events = list(snapshot.events) + self.policy_checkpoint.compact_completed(room_id=binding.room_id) + driver.prune_published_terminal_tasks( + self.db_path, + room_id=binding.room_id, + clock=self.runtime.clock, + ) + if any( + driver.list_tasks( + self.db_path, + room_id=binding.room_id, + status=status, + ) + for status in ("queued", "running", "stopping") + ): + return + decision = discussion.plan_next_task( + room, + events, + local_profiles=self.local_profiles(), + initial_watermarks=snapshot.watermarks, + ) + if decision.status == "task" and decision.task is not None: + driver.admit_task( + self.db_path, + decision.task.identity, + payload=decision.task.payload, + clock=time.time, + ) + # A stop can race the policy read from another process. Re-read + # after admission and cancel before the runtime can execute a + # task whose source event is now behind the room stop fence. + fresh_room = hosted_rooms.room_state( + self.db_path, + room_id=binding.room_id, + ) + stopped_through_seq = self._policy_snapshot( + fresh_room + ).stopped_through_seq + if ( + decision.source_event_seq is not None + and decision.source_event_seq < stopped_through_seq + ): + self.runtime.cancel( + decision.task.identity, + cancel_id=f"stop-fence:{stopped_through_seq}", + ) + elif decision.status in {"settled", "bounded"}: + self._append_room_status(room, decision) + + def publish_terminal( + self, + binding: HostedRoomBinding, + _task: Mapping[str, Any], + ) -> None: + self.prepare_room(binding) + self.runtime.wakeup() + + def create_room(self, *, room_id: str, name: str, members: Any) -> dict[str, Any]: + normalized = discussion.validate_roster( + members, + local_profiles=self.local_profiles(), + ) + room = hosted_rooms.create_room( + self.db_path, + room_id=room_id, + name=name, + members=[ + { + "member_id": member.member_id, + "profile": member.profile, + "handle": member.handle, + **( + {"display_name": member.display_name} + if member.display_name + else {} + ), + } + for member in normalized + ], + authority_gateway_id=hosted_rooms.local_authority_gateway_id(), + ) + self.runtime.wakeup() + return room + + def send( + self, + *, + room_id: str, + event_id: str, + payload: Any, + ) -> dict[str, Any]: + normalized = discussion.validate_user_payload(payload) + room = self._owned_room(room_id) + event = hosted_rooms.append_event( + self.db_path, + room_id=room_id, + event_id=event_id, + kind="message.user", + actor={"kind": "user", "id": "desktop"}, + payload=normalized, + authority_gateway_id=str(room["authority_gateway_id"]), + authority_epoch=int(room["authority_epoch"]), + ) + binding = next( + ( + candidate + for candidate in self.bindings() + if candidate.room_id == room_id + ), + None, + ) + if binding is None: + raise hosted_rooms.RoomNotFoundError("hosted room not found") + self.prepare_room(binding) + self.runtime.wakeup() + return event + + def stop_room( + self, + room_id: str, + *, + cancel_id: str, + require_acknowledged: bool = False, + ) -> int: + room = self._owned_room(room_id) + hosted_rooms.request_room_stop( + self.db_path, + room_id=room_id, + cancel_id=cancel_id, + expected_gateway_id=str(room["authority_gateway_id"]), + expected_epoch=int(room["authority_epoch"]), + ) + cancelled = 0 + pending = 0 + with self._policy_lock: + tasks = {} + for status in ( + "queued", + "running", + "indeterminate", + "deferred", + "stopping", + ): + for task in driver.list_tasks( + self.db_path, + room_id=room_id, + status=status, + ): + identity = task["identity"] + tasks[(identity.room_id, identity.task_id)] = task + for task in tasks.values(): + task_cancel_id = ( + str(task.get("cancel_id") or "") + if task.get("status") == "stopping" + else "" + ) + result = self.runtime.cancel( + task["identity"], + cancel_id=task_cancel_id or cancel_id, + ) + cancelled += 1 + if result["status"] == "stopping": + pending += 1 + if require_acknowledged and pending: + raise RuntimeError( + "room work is still stopping; retry deletion after Stop completes" + ) + self.runtime.wakeup() + return cancelled + + def retry_room_task(self, room_id: str, *, task_id: str) -> dict[str, Any]: + """Retry one uncertain or deferred task only after explicit user action.""" + + task = next( + ( + candidate + for status in ("indeterminate", "deferred") + for candidate in driver.list_tasks( + self.db_path, room_id=room_id, status=status + ) + if candidate["identity"].task_id == task_id + ), + None, + ) + if task is None: + raise driver.InvalidTaskTransitionError( + "no retryable room task matches task_id" + ) + return self.runtime.retry_indeterminate(task["identity"]) + + def approve_room_task( + self, + room_id: str, + *, + member_id: str, + task_id: str, + execution_generation: int, + choice: str, + request_id: str | None = None, + ) -> Mapping[str, Any]: + """Resolve one exact local approval and wake room observation.""" + + key = (room_id, member_id) + with self._policy_lock: + action = self._pending_actions.get(key) + requested_approval_id = str(request_id or "") + pending_approval_id = str((action or {}).get("request_id") or "") + if ( + action is None + or action.get("task_id") != task_id + or int(action.get("execution_generation") or 0) != execution_generation + or not requested_approval_id + or requested_approval_id != pending_approval_id + ): + raise RuntimeError("room approval is no longer pending") + if choice not in {"once", "deny"}: + raise RuntimeError("room approval choice must be once or deny") + session_id = str(action.get("session_id") or "") + if not session_id: + raise RuntimeError("local room approval identity is unavailable") + result = self.rpc.approve( + session_id=session_id, + request_id=requested_approval_id, + choice=choice, + ) + if result is None: + raise RuntimeError("room approval target is unavailable") + with self._policy_lock: + current = self._pending_actions.get(key) + if ( + current is not None + and str(current.get("request_id") or "") == requested_approval_id + and current.get("task_id") == task_id + and int(current.get("execution_generation") or 0) + == execution_generation + ): + self._pending_actions.pop(key, None) + self.runtime.wakeup() + return result + + def status(self, room_id: str | None = None) -> dict[str, Any]: + runtime = self.runtime.status() + if room_id is None: + return runtime + tasks = driver.list_tasks(self.db_path, room_id=room_id) + counts = Counter(str(task["status"]) for task in tasks) + pending_actions = [ + { + "kind": "retry", + "task_id": task["identity"].task_id, + } + for task in tasks + if task["status"] in {"indeterminate", "deferred"} + ] + with self._policy_lock: + pending_actions.extend( + dict(action) + for ( + action_room_id, + _member_id, + ), action in self._pending_actions.items() + if action_room_id == room_id + ) + return { + "running": runtime["running"], + "working": bool( + counts.get("running") or counts.get("queued") or counts.get("stopping") + ), + "blocked": room_id in runtime["blocked_rooms"] + or bool(counts.get("indeterminate") or counts.get("stopping")), + "counts": dict(counts), + "pending_actions": pending_actions, + } diff --git a/tui_gateway/methods_bot_relay.py b/tui_gateway/methods_bot_relay.py index cc5552a43c..1c5781b687 100644 --- a/tui_gateway/methods_bot_relay.py +++ b/tui_gateway/methods_bot_relay.py @@ -214,5 +214,12 @@ def register(server) -> None: _registry.install(server) from . import methods_groups + server._LONG_HANDLERS = server._LONG_HANDLERS | methods_groups.LONG_HANDLERS + server.get_hosted_room_service = methods_groups.get_hosted_room_service + server._WORKER_UNAVAILABLE = methods_groups._WORKER_UNAVAILABLE + methods_groups.bind_server(server) + methods_groups.register(server) + from . import methods_groups + methods_groups.register(server) server._LONG_HANDLERS = server._LONG_HANDLERS | methods_groups.LONG_HANDLERS diff --git a/tui_gateway/methods_groups.py b/tui_gateway/methods_groups.py index 9c014d797c..c0ce55f24c 100644 --- a/tui_gateway/methods_groups.py +++ b/tui_gateway/methods_groups.py @@ -1,13 +1,15 @@ """Hosted-room JSON-RPC contract. -These methods expose durable room identity and an append-only, monotonic room -log. They deliberately do not drive Bot turns yet. ``groups.capabilities`` -makes that boundary machine-readable so a hosted-aware Desktop cannot mistake -the log prototype for a complete gateway-side orchestrator. +These methods expose durable room identity, replay, and the process-owned +same-gateway Discussion driver. ``groups.capabilities`` keeps that boundary +machine-readable so older clients stay on the renderer-owned room path. """ from .method_ctx import HandlerRegistry +import os +import threading + _registry = HandlerRegistry() method = _registry.method @@ -23,8 +25,74 @@ LONG_HANDLERS = frozenset({ "groups.replica_state", "groups.promote", "groups.demote", + "groups.stop", + "groups.retry", + "groups.approve", }) +_service_lock = threading.Lock() +_bound_server = None +_service = None + + +def bind_server(server) -> None: + """Bind the fully initialized server module without starting a worker.""" + + global _bound_server + _bound_server = server + + +def start_hosted_room_service(): + """Start one process-owned hosted room service idempotently.""" + + global _service + if _bound_server is None: + return None + from gateway.hosted_rooms import default_db_path + from tui_gateway.hosted_room_service import HostedRoomService + + db_path = default_db_path() + with _service_lock: + if _service is not None and _service.db_path != db_path: + _service.stop(timeout=1.0) + _service = None + if _service is None: + _service = HostedRoomService(_bound_server, db_path=db_path) + _service.start() + return _service + + +def stop_hosted_room_service(*, timeout: float = 5.0) -> bool: + """Stop the process-owned worker without interrupting accepted turns.""" + + global _service + with _service_lock: + service = _service + if service is None: + return True + stopped = service.stop(timeout=timeout) + if stopped and _service is service: + _service = None + return stopped + + +def get_hosted_room_service(): + """Return the active service, if its lifecycle owner started it.""" + + service = _service + if service is None: + return None + try: + status = service.runtime.status() + except Exception: + return None + return service if status.get("running") and not status.get("stopping") else None + + +_WORKER_UNAVAILABLE = ( + "Group Chat worker is unavailable. Restart the Hermes gateway and try again." +) + @method("groups.capabilities") def _(rid, params: dict) -> dict: @@ -35,11 +103,14 @@ def _(rid, params: dict) -> dict: local_authority_gateway_id, ) + service = get_hosted_room_service() + driver_ready = bool(service and service.runtime.status()["running"]) return _ok( rid, { "protocol_version": PROTOCOL_VERSION, - "driver": False, + "driver": driver_ready, + "persistent_process": os.getenv("HERMES_DESKTOP") != "1", "authority_gateway_id": local_authority_gateway_id(), "features": [ "authority_epoch", @@ -65,6 +136,9 @@ def _(rid, params: dict) -> dict: "groups.replica_state", "groups.promote", "groups.demote", + "groups.stop", + "groups.retry", + "groups.approve", ], "max_log_limit": MAX_LOG_LIMIT, }, @@ -108,20 +182,16 @@ def _(rid, params: dict) -> dict: Required params: ``room_id``, ``name``, and ``members``. Authority is derived from this gateway's stable install identity, never from the client. """ - from gateway.hosted_rooms import ( - HostedRoomError, - create_room, - default_db_path, - local_authority_gateway_id, - ) + from gateway.hosted_rooms import HostedRoomError try: - room = create_room( - default_db_path(), + service = get_hosted_room_service() + if service is None: + return _err(rid, 4123, _WORKER_UNAVAILABLE) + room = service.create_room( room_id=params.get("room_id"), name=params.get("name"), members=params.get("members"), - authority_gateway_id=local_authority_gateway_id(), ) return _ok(rid, {"room": room}) except HostedRoomError as exc: @@ -137,14 +207,21 @@ def _(rid, params: dict) -> dict: from gateway.hosted_rooms import HostedRoomError, default_db_path, room_state try: + room = room_state( + default_db_path(), + room_id=params.get("room_id"), + include_disbanded=params.get("include_disbanded") is True, + ) + service = get_hosted_room_service() return _ok( rid, { - "room": room_state( - default_db_path(), - room_id=params.get("room_id"), - include_disbanded=params.get("include_disbanded") is True, - ) + "room": room, + **( + {"driver_status": service.status(str(room["room_id"]))} + if service is not None and room.get("disbanded_at") is None + else {} + ), }, ) except HostedRoomError as exc: @@ -163,33 +240,17 @@ def _(rid, params: dict) -> dict: method. The actor is server-owned rather than trusted from params. Admission is durable; no Bot turn is started by this slice. """ - from gateway.hosted_rooms import ( - AuthorityConflictError, - HostedRoomError, - append_event, - default_db_path, - local_authority_gateway_id, - room_state, - user_event_id, - ) + from gateway.hosted_rooms import HostedRoomError, user_event_id try: - room = room_state(default_db_path(), room_id=params.get("room_id")) - local_gateway_id = local_authority_gateway_id() - if str(room["authority_gateway_id"]) != local_gateway_id: - raise AuthorityConflictError( - "This Group Chat is managed by another gateway." - ) client_event_id = params.get("event_id") - event = append_event( - default_db_path(), + service = get_hosted_room_service() + if service is None: + return _err(rid, 4123, _WORKER_UNAVAILABLE) + event = service.send( room_id=params.get("room_id"), event_id=user_event_id(client_event_id), - kind="message.user", - actor={"kind": "user", "id": "desktop"}, payload=params.get("payload"), - authority_gateway_id=local_gateway_id, - authority_epoch=int(room["authority_epoch"]), ) return _ok( rid, @@ -197,7 +258,7 @@ def _(rid, params: dict) -> dict: "event": event, "client_event_id": client_event_id, "accepted": True, - "driver_started": False, + "driver_started": True, }, ) except HostedRoomError as exc: @@ -214,35 +275,53 @@ def _(rid, params: dict) -> dict: AuthorityConflictError, HostedRoomError, RoomHistoryExpiredError, - default_db_path, disband_room, local_authority_gateway_id, room_state, ) try: - local_gateway_id = local_authority_gateway_id() + service = get_hosted_room_service() + if service is None: + return _err(rid, 4123, _WORKER_UNAVAILABLE) + + def disband_with_state(state: dict | None = None) -> dict: + local_gateway_id = local_authority_gateway_id() + if state is not None and ( + str(state["authority_gateway_id"]) != local_gateway_id + ): + raise AuthorityConflictError( + "This Group Chat is managed by another gateway." + ) + return disband_room( + service.db_path, + room_id=params.get("room_id"), + expected_gateway_id=str( + local_gateway_id + ), + expected_epoch=int( + state["authority_epoch"] if state is not None else 1 + ), + ) + try: - room = room_state( - default_db_path(), + existing = room_state( + service.db_path, room_id=params.get("room_id"), include_disbanded=True, ) except RoomHistoryExpiredError: - room = { - "authority_gateway_id": local_gateway_id, - "authority_epoch": 1, - } - if str(room["authority_gateway_id"]) != local_gateway_id: - raise AuthorityConflictError( - "This Group Chat is managed by another gateway." - ) - tombstone = disband_room( - default_db_path(), - room_id=params.get("room_id"), - expected_gateway_id=local_gateway_id, - expected_epoch=int(room["authority_epoch"]), + tombstone = disband_with_state() + return _ok(rid, {"tombstone": tombstone}) + if existing.get("disbanded_at") is not None: + tombstone = disband_with_state(existing) + return _ok(rid, {"tombstone": tombstone}) + service.stop_room( + str(params.get("room_id") or ""), + cancel_id=str(params.get("cancel_id") or "room-disbanded"), + require_acknowledged=True, ) + tombstone = disband_with_state(existing) return _ok(rid, {"tombstone": tombstone}) except HostedRoomError as exc: reason = getattr(exc, "reason", None) @@ -251,6 +330,75 @@ def _(rid, params: dict) -> dict: return _err(rid, 5114, str(exc)) +@method("groups.stop") +def _(rid, params: dict) -> dict: + """Durably cancel queued or running work for one hosted room.""" + + service = get_hosted_room_service() + if service is None: + return _err(rid, 4115, "hosted room driver is unavailable") + try: + count = service.stop_room( + str(params.get("room_id") or ""), + cancel_id=str(params.get("cancel_id") or "desktop-stop"), + ) + return _ok(rid, {"cancelled": count}) + except Exception as exc: + return _err(rid, 5116, str(exc)) + + +@method("groups.approve") +def _(rid, params: dict) -> dict: + """Resolve one exact approval requested by a local room member.""" + + service = get_hosted_room_service() + if service is None: + return _err(rid, 4115, "hosted room driver is unavailable") + try: + result = service.approve_room_task( + str(params.get("room_id") or ""), + member_id=str(params.get("member_id") or ""), + task_id=str(params.get("task_id") or ""), + execution_generation=int(params.get("execution_generation") or 0), + choice=str(params.get("choice") or ""), + request_id=str(params.get("request_id") or ""), + ) + return _ok(rid, {"approved": True, "result": result}) + except Exception as exc: + return _err(rid, 5119, str(exc)) + + +@method("groups.retry") +def _(rid, params: dict) -> dict: + """Retry one indeterminate room task after explicit user confirmation.""" + + service = get_hosted_room_service() + if service is None: + return _err(rid, 4115, "hosted room driver is unavailable") + try: + task = service.retry_room_task( + str(params.get("room_id") or ""), + task_id=str(params.get("task_id") or ""), + ) + identity = task.get("identity") if isinstance(task, dict) else None + receipt = { + "room_id": str(getattr(identity, "room_id", "") or ""), + "task_id": str(getattr(identity, "task_id", "") or ""), + "thread_id": str(getattr(identity, "thread_id", "") or ""), + "turn_id": str(getattr(identity, "turn_id", "") or ""), + "status": str(task.get("status") or "") if isinstance(task, dict) else "", + "execution_generation": int(task.get("execution_generation") or 0) + if isinstance(task, dict) + else 0, + "cancel_generation": int(task.get("cancel_generation") or 0) + if isinstance(task, dict) + else 0, + } + return _ok(rid, {"retried": True, "task": receipt}) + except Exception as exc: + return _err(rid, 5118, str(exc)) + + @method("groups.log") def _(rid, params: dict) -> dict: """Return a monotonic room-log delta after ``since_seq``.""" diff --git a/tui_gateway/methods_prompt.py b/tui_gateway/methods_prompt.py index 34e38aacf9..66d2153b26 100644 --- a/tui_gateway/methods_prompt.py +++ b/tui_gateway/methods_prompt.py @@ -335,6 +335,67 @@ def _(rid, params: dict) -> dict: session, err = _sess_nowait(params, rid) if err: return err + hosted_task = params.get("_hosted_task") + hosted_terminal_callback = params.get("_hosted_terminal_callback") + internal_hosted_submit = hosted_task is not None or hosted_terminal_callback is not None + if internal_hosted_submit: + if session.get("source") != "bot_room": + return _err(rid, 4120, "hosted room turns require a bot_room session") + if not isinstance(hosted_task, dict) or not callable(hosted_terminal_callback): + return _err(rid, 4120, "invalid hosted room turn proof") + required_hosted_fields = { + "room_id", + "task_id", + "thread_id", + "turn_id", + "execution_generation", + } + if set(hosted_task) != required_hosted_fields or not all( + isinstance(hosted_task.get(field), str) and hosted_task[field] + for field in required_hosted_fields - {"execution_generation"} + ) or not isinstance(hosted_task.get("execution_generation"), int): + return _err(rid, 4120, "invalid hosted room turn proof") + else: + # Older Desktop builds know the `Group: ` session title but + # not the hosted authority marker. Once a gateway owns that room, a + # direct prompt into its member session would start a second renderer + # driver. Fence it server-side instead of trusting client awareness. + title = str(session.get("title") or "") + if title.startswith("Group: "): + room_id = title.removeprefix("Group: ").strip() + if room_id: + try: + from gateway.hosted_rooms import ( + HostedRoomError, + RoomProbeUnavailableError, + default_db_path, + probe_hosted_room, + ) + + hosted = probe_hosted_room(default_db_path(), room_id=room_id) + except RoomProbeUnavailableError: + return _err( + rid, + 5122, + "Could not verify this group. Try again after the gateway recovers.", + ) + except HostedRoomError: + # Legacy Desktop sessions used the display name after + # "Group: "; those names are not hosted room ids. + pass + except Exception: + return _err( + rid, + 5122, + "Could not verify this group. Try again after the gateway recovers.", + ) + else: + if hosted: + return _err( + rid, + 4122, + "This room is managed by its gateway. Update Hermes Desktop to continue it.", + ) if (limit_message := _ensure_active_session_slot(sid, session)) is not None: return _err(rid, 4090, limit_message) # Which desktop window this message was typed into. Rewritten on every @@ -357,6 +418,12 @@ def _(rid, params: dict) -> dict: ) isolation_cfg = _load_dashboard_process_isolation_config() turn_isolation = _session_uses_compute_host(session, isolation_cfg) + if internal_hosted_submit and turn_isolation: + return _err( + rid, + 4121, + "hosted room turns do not support isolated compute workers yet", + ) # Re-bind to the current client transport for this request. This keeps # streaming events on the active websocket even if an earlier disconnect # or fallback moved the session transport to stdio. @@ -366,6 +433,8 @@ def _(rid, params: dict) -> dict: busy_transport = None with session["history_lock"]: if session.get("running"): + if internal_hosted_submit: + return _err(rid, 4091, "hosted room member session is busy") # Don't reject a mid-turn prompt — queue it (and, by default, # interrupt the live turn) so it runs as the next turn. The # provider interrupt itself must happen after this lock is @@ -812,6 +881,8 @@ def _(rid, params: dict) -> dict: session["running"] = True session["_turn_cancel_requested"] = False session["last_active"] = time.time() + if internal_hosted_submit: + session["_hosted_room_task"] = dict(hosted_task) _start_inflight_turn(session, text) if turn_isolation: @@ -908,7 +979,14 @@ def _(rid, params: dict) -> dict: }, ) return - _run_prompt_submit(rid, sid, session, text, display_kind=display_kind) + _run_prompt_submit( + rid, + sid, + session, + text, + display_kind=display_kind, + terminal_callback=hosted_terminal_callback, + ) run_thread = threading.Thread(target=run_after_agent_ready, daemon=True) # Keep a handle so session.interrupt can tell a live turn from a stuck diff --git a/tui_gateway/methods_session.py b/tui_gateway/methods_session.py index 37844381f8..c3a9d7c86c 100644 --- a/tui_gateway/methods_session.py +++ b/tui_gateway/methods_session.py @@ -3361,6 +3361,18 @@ def _(rid, params: dict) -> dict: session, err = _sess_nowait(params, rid) if err: return err + expected_hosted_task_id = str( + params.get("expected_hosted_task_id") or "" + ).strip() + if expected_hosted_task_id: + with session["history_lock"]: + active_task = session.get("_hosted_room_task") + if ( + not session.get("running") + or not isinstance(active_task, dict) + or active_task.get("task_id") != expected_hosted_task_id + ): + return _ok(rid, {"status": "not_interrupted", "interrupted": False}) if _session_uses_compute_host(session): sid = str(params.get("session_id") or "") try: diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 23a36b6248..46be96ee92 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -9801,6 +9801,12 @@ def _maybe_schedule_auto_continue(sid: str, session: dict, session_key: str) -> same _run_prompt_submit machinery as every other synthesized turn — so the client that just resumed streams it live. """ + # Hosted room turns are recovered by their durable task/lease state + # machine. Generic session auto-continue would bypass its execution + # generation and can duplicate work after a process restart. + if session.get("source") == "bot_room": + return None + home = _session_home(session) marker = read_turn_marker(home, session_key) if marker is None: @@ -10280,7 +10286,12 @@ def _inflight_snapshot(session: dict) -> dict | None: def _emit_terminal_turn_error( - sid: str, session: dict, error: Any, error_surface: Optional[dict] = None + sid: str, + session: dict, + error: Any, + error_surface: Optional[dict] = None, + *, + retire_marker: bool = True, ) -> None: """Close a failed turn with a terminal ``message.complete`` frame. @@ -10333,7 +10344,8 @@ def _emit_terminal_turn_error( rendered = "" if rendered: payload["rendered"] = rendered - _retire_turn_marker(session) + if retire_marker: + _retire_turn_marker(session) _emit("message.complete", sid, payload) @@ -12579,6 +12591,7 @@ def _run_prompt_submit( display_metadata: dict | None = None, image_paths: list[str] | None = None, queued_prompt_generation: int | None = None, + terminal_callback: Callable[[dict[str, Any]], None] | None = None, ) -> bool: with session["history_lock"]: if session.get("_closing"): @@ -12628,6 +12641,8 @@ def _run_prompt_submit( _emit("message.start", sid) def run(): + terminal_receipt_attempted = False + terminal_receipt_committed = terminal_callback is None # The conversation runs on a fresh thread, so ContextVars from the RPC # dispatcher do not follow automatically. Rebind the exact transport # stored on this session generation before any tool can commission a @@ -13168,7 +13183,26 @@ def _run_prompt_submit( payload["recoverable"] = True if _error_surface: payload["error_surface"] = _error_surface - _retire_turn_marker(session, marker_key) + if terminal_callback is not None: + terminal_receipt_attempted = True + terminal_callback( + { + "status": ( + "cancelled" + if status == "interrupted" + else "failed" if status == "error" else "settled" + ), + "text": raw if isinstance(raw, str) else str(raw), + **( + {"error": str(result.get("error") or raw)} + if status == "error" and isinstance(result, dict) + else {} + ), + } + ) + terminal_receipt_committed = True + if terminal_receipt_committed: + _retire_turn_marker(session, marker_key) _emit("message.complete", sid, payload) # ── /goal continuation (Ralph-style loop) ───────────────── @@ -13348,11 +13382,25 @@ def _run_prompt_submit( # Keep the partial turn available to the next prompt; the durable # inflight record still carries the recoverable error state. _restore_agent_history_after_turn_error(session, agent) + if terminal_callback is not None and not terminal_receipt_attempted: + terminal_receipt_attempted = True + try: + terminal_callback( + {"status": "failed", "text": "", "error": str(e)} + ) + terminal_receipt_committed = True + except Exception: + logger.exception("hosted room terminal receipt commit failed") try: # Close the turn with the same terminal error frame shape as # the returned-error path (uniform client handling), retaining # the failed turn for resume replay. - _emit_terminal_turn_error(sid, session, e) + _emit_terminal_turn_error( + sid, + session, + e, + retire_marker=terminal_receipt_committed, + ) turn_error_retained = True except Exception as emit_exc: print( @@ -13447,7 +13495,10 @@ def _run_prompt_submit( ) # Backstop for turns that never reached a terminal frame (the # frame paths retire the marker as they emit). - _retire_turn_marker(session, marker_key) + if terminal_receipt_committed: + _retire_turn_marker(session, marker_key) + with session["history_lock"]: + session.pop("_hosted_room_task", None) session.pop("_auto_continue_scheduled", None) _emit_settled_session_info(sid, session, agent) From 5a3edc7467558f68a1a66ed32ea9ea458503a680 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:17:52 -0700 Subject: [PATCH 043/634] fix(bot-mode): cancel re-routes on worker races instead of failing disband Follow-ups on the salvaged #97744 runner: - tui_gateway/hosted_room_driver.py: HostedRoomRuntime.cancel() treated its initial status read as truth, so a task transitioning queued->running (or settling) between the read and the state call surfaced a transient 'running work requires acknowledged two-phase cancellation' / StaleTaskError to the caller and failed groups.disband. Deterministic repro on the PR head: test_client_event_id_cannot_squat_disband_receipt failed 5/5 locally. cancel() now re-reads and re-routes on every race-shaped failure (bounded retries), returns already-cancelled tasks idempotently, and rejects truly terminal states honestly. - methods_groups conflict resolution keeps both method sets: the replication surface from #99047 (groups.replicate/replica_state/promote/demote) and the runner surface from this layer (groups.stop/retry/approve). - test_groups_replication_methods.py updated to the runner's stricter create contract (2-6 profile-backed members, live worker service). --- .../test_groups_replication_methods.py | 20 +++- tui_gateway/hosted_room_driver.py | 92 +++++++++++++------ 2 files changed, 81 insertions(+), 31 deletions(-) diff --git a/tests/tui_gateway/test_groups_replication_methods.py b/tests/tui_gateway/test_groups_replication_methods.py index 9101a8cbbe..642a0561fa 100644 --- a/tests/tui_gateway/test_groups_replication_methods.py +++ b/tests/tui_gateway/test_groups_replication_methods.py @@ -6,6 +6,7 @@ from __future__ import annotations import pytest import tui_gateway.server as srv +from tui_gateway import methods_groups MEMBERS = [{"kind": "bot", "id": "planner"}] @@ -14,8 +15,12 @@ MEMBERS = [{"kind": "bot", "id": "planner"}] def home(tmp_path, monkeypatch): path = tmp_path / ".hermes" path.mkdir() + (path / "profiles" / "ops").mkdir(parents=True) monkeypatch.setenv("HERMES_HOME", str(path)) - return path + methods_groups.stop_hosted_room_service(timeout=1.0) + methods_groups.start_hosted_room_service() + yield path + methods_groups.stop_hosted_room_service(timeout=1.0) def _result(envelope): @@ -124,7 +129,18 @@ def test_demote_fences_local_room_against_newer_epoch(home): _result( srv._methods["groups.create"]( 1, - {"room_id": "room-1", "name": "Local room", "members": MEMBERS}, + { + "room_id": "room-1", + "name": "Local room", + "members": [ + { + "member_id": "default", + "profile": "default", + "handle": "hermes", + }, + {"member_id": "ops", "profile": "ops", "handle": "ops"}, + ], + }, ) ) observed_gateway = "install:" + "b" * 32 diff --git a/tui_gateway/hosted_room_driver.py b/tui_gateway/hosted_room_driver.py index 9fdeede8f2..8f95a2cd9f 100644 --- a/tui_gateway/hosted_room_driver.py +++ b/tui_gateway/hosted_room_driver.py @@ -29,6 +29,8 @@ from typing import Any, ContextManager, Protocol, cast from gateway import hosted_room_driver as state +_CANCEL_ROUTE_RETRIES = 8 + ROOM_SESSION_SOURCE = "bot_room" MAX_TERMINAL_TEXT_BYTES = 64 * 1024 @@ -252,41 +254,73 @@ class HostedRoomRuntime: *, cancel_id: str, ) -> dict[str, Any]: - """Persist a stop intent, then commit cancellation after acknowledgement.""" - before = state.get_task(self.db_path, identity) - if before["status"] in {"queued", "deferred"}: - cancelled = state.cancel_task( - self.db_path, - identity, - cancel_id=cancel_id, - expected_cancel_generation=before["cancel_generation"], - clock=self.clock, - ) - self.wakeup() - return cancelled + """Persist a stop intent, then commit cancellation after acknowledgement. - stopping = state.begin_task_cancel( - self.db_path, - identity, - cancel_id=cancel_id, - expected_cancel_generation=before["cancel_generation"], - clock=self.clock, - ) - binding = self._binding_for_room(identity.room_id) - try: - if binding is not None and self._interrupt_stopping_task(binding, stopping): - stopping = state.complete_task_cancel( + The worker thread transitions tasks concurrently with cancellation + (queued -> running -> terminal), so the status read below is only a + routing hint. Every fast-path failure caused by a concurrent + transition re-reads and re-routes instead of surfacing a transient + `InvalidTaskTransitionError`/`StaleTaskError` to the caller. + """ + for _ in range(_CANCEL_ROUTE_RETRIES): + before = state.get_task(self.db_path, identity) + if before["status"] == "cancelled": + return before + if before["status"] in state.TERMINAL_STATUSES: + raise state.InvalidTaskTransitionError( + f"cannot cancel task in state '{before['status']}'" + ) + if before["status"] in {"queued", "deferred"}: + try: + cancelled = state.cancel_task( + self.db_path, + identity, + cancel_id=cancel_id, + expected_cancel_generation=before["cancel_generation"], + clock=self.clock, + ) + except (state.InvalidTaskTransitionError, state.StaleTaskError): + # Lost the race with the worker; re-read and re-route. + continue + self.wakeup() + return cancelled + try: + stopping = state.begin_task_cancel( self.db_path, identity, cancel_id=cancel_id, - expected_cancel_generation=stopping["cancel_generation"], + expected_cancel_generation=before["cancel_generation"], clock=self.clock, ) - except Exception as exc: - self._record_error(f"stop remains pending: {exc}") - stopping = state.get_task(self.db_path, identity) - self.wakeup() - return stopping + except (state.InvalidTaskTransitionError, state.StaleTaskError): + # Task settled or re-queued mid-flight; re-read and re-route. + continue + binding = self._binding_for_room(identity.room_id) + try: + if binding is not None and self._interrupt_stopping_task( + binding, stopping + ): + stopping = state.complete_task_cancel( + self.db_path, + identity, + cancel_id=cancel_id, + expected_cancel_generation=stopping["cancel_generation"], + clock=self.clock, + ) + except Exception as exc: + self._record_error(f"stop remains pending: {exc}") + stopping = state.get_task(self.db_path, identity) + self.wakeup() + return stopping + # Exhausted routing retries under sustained contention: surface the + # live status honestly rather than a transient transition error. + final = state.get_task(self.db_path, identity) + if final["status"] == "cancelled": + return final + raise state.InvalidTaskTransitionError( + f"cancel kept losing races with task transitions " + f"(last observed state '{final['status']}')" + ) def retry_indeterminate(self, identity: state.TaskIdentity) -> dict[str, Any]: """Explicitly retry one uncertain attempt under the current room lease.""" From 1b6ea1a2c2b4e48a15ea392e1f3f94331fba1c57 Mon Sep 17 00:00:00 2001 From: itskaism Date: Sun, 16 Aug 2026 16:49:42 +0900 Subject: [PATCH 044/634] fix(delegation): report failed children as failed, not completed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A subagent whose loop gave up on a structured failure (e.g. "API call failed after 3 retries: HTTP 524") returns that error message as final_response together with completed=False / failed=True / failure_reason. _run_single_child derived the batch-entry status from the summary alone (`elif summary and not _empty_sentinel: status = "completed"`), so the non-empty error text made the batch report show the task as "✓ status=completed" — the `failed` flag was never consulted anywhere in delegate_tool.py. Only the "(empty)" sentinel was mapped to failed. Fix, at the single status-determination choke point both the single-task and batch paths share: - `failed=True` on the child result now wins over a non-empty summary: status = "failed". - The child's classified failure_reason (rate_limit / billing / server_error / ...) is propagated onto the batch entry so the parent can tell a quota wall from a real task error without parsing prose. - exit_reason for a structured failure is "error" instead of falling through to "max_iterations" (which also wrongly set truncated=True). Successful children (completed=True, no failed flag) are untouched — covered by an explicit control test alongside the regression test, which is red on the old code and green with the fix. --- tests/tools/test_delegate.py | 69 ++++++++++++++++++++++++++++++++++++ tools/delegate_tool.py | 6 ++++ 2 files changed, 75 insertions(+) diff --git a/tests/tools/test_delegate.py b/tests/tools/test_delegate.py index 598819cf6e..63d28cc121 100644 --- a/tests/tools/test_delegate.py +++ b/tests/tools/test_delegate.py @@ -675,6 +675,75 @@ class TestDelegateObservability(unittest.TestCase): result = json.loads(delegate_task(goal="Test empty sentinel", parent_agent=parent)) self.assertEqual(result["results"][0]["status"], "failed") + def test_failed_child_with_error_summary_marks_status_failed(self): + """Regression: a child whose loop gave up on a structured failure + (``failed=True``, ``completed=False``, e.g. "API call failed after 3 + retries: HTTP 524") returns that error message as final_response. + Status was derived from summary alone, so the non-empty error text + made the batch report show the task as ✓ status=completed. The + ``failed`` flag must win over a non-empty summary.""" + parent = _make_mock_parent(depth=0) + + with patch("run_agent.AIAgent") as MockAgent: + mock_child = MagicMock() + mock_child.model = "claude-sonnet-4-6" + mock_child.session_prompt_tokens = 0 + mock_child.session_completion_tokens = 0 + mock_child.run_conversation.return_value = { + "final_response": ( + "API call failed after 3 retries: HTTP 524 — origin timeout" + ), + "completed": False, + "failed": True, + "error": "HTTP 524 — origin timeout", + "failure_reason": "server_error", + "interrupted": False, + "api_calls": 3, + "messages": [], + } + MockAgent.return_value = mock_child + + result = json.loads( + delegate_task(goal="Test failed child", parent_agent=parent) + ) + entry = result["results"][0] + self.assertEqual(entry["status"], "failed") + # The classified reason must survive into the batch entry so the + # parent can tell a quota wall from a real task error. + self.assertEqual(entry["failure_reason"], "server_error") + self.assertEqual(entry["error"], "HTTP 524 — origin timeout") + # A structured failure is not budget truncation. + self.assertEqual(entry["exit_reason"], "error") + self.assertFalse(entry["truncated"]) + + def test_successful_child_still_completed(self): + """Control for the failed-flag check: a child that succeeds + (``completed=True``, no ``failed`` flag) must keep reporting + status=completed — the fix must not change success behavior.""" + parent = _make_mock_parent(depth=0) + + with patch("run_agent.AIAgent") as MockAgent: + mock_child = MagicMock() + mock_child.model = "claude-sonnet-4-6" + mock_child.session_prompt_tokens = 0 + mock_child.session_completion_tokens = 0 + mock_child.run_conversation.return_value = { + "final_response": "All done.", + "completed": True, + "interrupted": False, + "api_calls": 2, + "messages": [], + } + MockAgent.return_value = mock_child + + result = json.loads( + delegate_task(goal="Test success control", parent_agent=parent) + ) + entry = result["results"][0] + self.assertEqual(entry["status"], "completed") + self.assertEqual(entry["exit_reason"], "completed") + self.assertNotIn("failure_reason", entry) + class TestDelegateFailedChildStatus(unittest.TestCase): """Honest status / exit_reason for failed subagents (issue #97655). diff --git a/tools/delegate_tool.py b/tools/delegate_tool.py index 89cab103c5..4177f74d22 100644 --- a/tools/delegate_tool.py +++ b/tools/delegate_tool.py @@ -3317,6 +3317,12 @@ def _run_single_child( ) if status == "failed": entry["error"] = result.get("error", "Subagent did not produce a response.") + # Classified reason from the child loop (e.g. "rate_limit", + # "billing", "server_error") — lets the parent distinguish a + # quota wall from a real task error without parsing prose. + _failure_reason = result.get("failure_reason") + if isinstance(_failure_reason, str) and _failure_reason: + entry["failure_reason"] = _failure_reason # T1-24: schema-validation outcome — emitted ONLY when a schema was # requested, so legacy (schema-less) payloads keep their exact shape. From 8557e0a4804d424799d07266c0b01d6877f9ba56 Mon Sep 17 00:00:00 2001 From: liuhao1024 Date: Sat, 29 Aug 2026 12:31:13 +0800 Subject: [PATCH 045/634] fix(tools): surface config-level model_not_found notices in delegation batch reports When a typo'd delegation.model slug is rejected by the provider, every subagent in the batch dies within a second carrying the provider's rejection text as its summary while the per-task blocks keep labelling it status=completed + TRUNCATED. The config-level root cause stays buried in the batch dump (#97654). Detect the rejection in the batch render path (summary/error text matching a model_not_found pattern from agent.error_classifier AND naming the configured delegation model id) and prepend a single config-level notice with the model id, hit count, and the setting to fix, before the per-task blocks. --- tests/tools/test_async_delegation.py | 83 ++++++++++++++++++++++++++++ tools/process_registry.py | 33 ++++++----- 2 files changed, 102 insertions(+), 14 deletions(-) diff --git a/tests/tools/test_async_delegation.py b/tests/tools/test_async_delegation.py index dd8c07ab9f..ac2d56e372 100644 --- a/tests/tools/test_async_delegation.py +++ b/tests/tools/test_async_delegation.py @@ -881,3 +881,86 @@ def test_batch_truncation_banner_marks_only_truncated_task(): # The header banner for task 2 appears after task 1's summary. assert banner_pos > clean_pos + +def test_batch_model_rejection_notice_prepended(): + """A rejected delegation model must surface ONE config-level notice above + the per-task blocks instead of staying buried in each summary (#97654).""" + rejection = "HTTP 400: upstage/solar-pro-4 is not a valid model ID" + evt = _make_async_evt( + is_batch=True, + model="upstage/solar-pro-4", + goals=["task a", "task b"], + results=[ + { + "task_index": 0, + "status": "completed", + "summary": rejection, + "api_calls": 1, + "duration_seconds": 0.74, + "exit_reason": "max_iterations", + "truncated": True, + }, + { + "task_index": 1, + "status": "completed", + "summary": rejection, + "api_calls": 1, + "duration_seconds": 0.71, + "exit_reason": "max_iterations", + "truncated": True, + }, + ], + ) + text = format_process_notification(evt) + assert text is not None + assert "SUBAGENT MODEL REJECTED" in text + assert "upstage/solar-pro-4" in text + assert "2/2" in text + assert "delegation.model" in text + # The notice precedes the per-task blocks, not just trails them. + assert text.index("SUBAGENT MODEL REJECTED") < text.index("TASK 1/2") + + +def test_batch_model_rejection_notice_absent_when_clean(): + """Ordinary summaries must not grow a model-rejection notice.""" + evt = _make_async_evt( + is_batch=True, + model="upstage/solar-pro4", + goals=["task a"], + results=[ + { + "task_index": 0, + "status": "completed", + "summary": "did the work", + "api_calls": 3, + "exit_reason": "completed", + "truncated": False, + }, + ], + ) + text = format_process_notification(evt) + assert text is not None + assert "SUBAGENT MODEL REJECTED" not in text + + +def test_batch_model_rejection_notice_requires_configured_model_in_text(): + """A model_not_found pattern naming a DIFFERENT model than the configured + delegation model is task-level noise, not a config-level rejection.""" + evt = _make_async_evt( + is_batch=True, + model="upstage/solar-pro4", + goals=["task a"], + results=[ + { + "task_index": 0, + "status": "completed", + "summary": "HTTP 400: other/model-x is not a valid model ID", + "api_calls": 1, + "exit_reason": "max_iterations", + "truncated": True, + }, + ], + ) + text = format_process_notification(evt) + assert text is not None + assert "SUBAGENT MODEL REJECTED" not in text diff --git a/tools/process_registry.py b/tools/process_registry.py index 5d25bf7384..926fb55cfa 100644 --- a/tools/process_registry.py +++ b/tools/process_registry.py @@ -2960,19 +2960,21 @@ def _format_age(seconds: float) -> str: return f"{h}h" if m == 0 else f"{h}h{m}m" -# Model-not-found phrases lifted from agent/error_classifier.py so the -# delegation batch renderer can spot a config-level rejection without pulling -# the classifier's failover machinery. Kept in sync by hand. -_MODEL_NOT_FOUND_PATTERNS = ( - "is not a valid model", - "invalid model", - "model not found", - "model_not_found", - "does not exist", - "no such model", - "unknown model", - "unsupported model", -) +def _model_not_found_patterns() -> "list[str]": + """Model-not-found phrases from the failover classifier. + + Imported from ``agent.error_classifier`` so the batch renderer applies + the SAME classification the failover path consumes — no hand-copied + pattern list to drift. Fails open to a minimal built-in set so a + classifier import problem never hides the per-task blocks. + (Import approach from PR #97667 by @liuhao1024.) + """ + try: + from agent.error_classifier import _MODEL_NOT_FOUND_PATTERNS + + return list(_MODEL_NOT_FOUND_PATTERNS) + except Exception: + return ["is not a valid model", "model not found", "model_not_found"] def _delegation_config() -> dict: @@ -3009,7 +3011,7 @@ def _delegation_model_not_found(results, config) -> bool: ).lower() if not text or model not in text: continue - if any(p in text for p in _MODEL_NOT_FOUND_PATTERNS): + if any(p in text for p in _model_not_found_patterns()): return True return False @@ -3105,6 +3107,9 @@ def _format_async_delegation(evt: dict) -> str: lines.append("--- ERROR ---") lines.append(f"The batch did not complete successfully: {error}") return "\n".join(lines) + # Config-level rejection notice BEFORE the per-task wall — a rejected + # delegation model fails every task identically before doing any + # work, and that signal must not stay buried in the task blocks. _notice = _delegation_model_not_found_notice(results) if _notice: lines.append("") From abdc7b952c30ce0270384c15b92c4e9ee2c0c98d Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:18:22 -0700 Subject: [PATCH 046/634] test: adapt #97667 rejection-notice tests to the config-reading implementation on main --- tests/tools/test_async_delegation.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/tests/tools/test_async_delegation.py b/tests/tools/test_async_delegation.py index ac2d56e372..5d0a01c22a 100644 --- a/tests/tools/test_async_delegation.py +++ b/tests/tools/test_async_delegation.py @@ -882,10 +882,22 @@ def test_batch_truncation_banner_marks_only_truncated_task(): assert banner_pos > clean_pos -def test_batch_model_rejection_notice_prepended(): +def _patch_delegation_cfg(monkeypatch, model="upstage/solar-pro-4", provider="openrouter"): + """Pin the delegation config the notice renderer reads (adapts the + #97667 tests to the shipped implementation, which reads the configured + model from config rather than the event's model field).""" + import tools.process_registry as _pr + + monkeypatch.setattr( + _pr, "_delegation_config", lambda: {"model": model, "provider": provider} + ) + + +def test_batch_model_rejection_notice_prepended(monkeypatch): """A rejected delegation model must surface ONE config-level notice above the per-task blocks instead of staying buried in each summary (#97654).""" rejection = "HTTP 400: upstage/solar-pro-4 is not a valid model ID" + _patch_delegation_cfg(monkeypatch) evt = _make_async_evt( is_batch=True, model="upstage/solar-pro-4", @@ -915,14 +927,14 @@ def test_batch_model_rejection_notice_prepended(): assert text is not None assert "SUBAGENT MODEL REJECTED" in text assert "upstage/solar-pro-4" in text - assert "2/2" in text assert "delegation.model" in text # The notice precedes the per-task blocks, not just trails them. assert text.index("SUBAGENT MODEL REJECTED") < text.index("TASK 1/2") -def test_batch_model_rejection_notice_absent_when_clean(): +def test_batch_model_rejection_notice_absent_when_clean(monkeypatch): """Ordinary summaries must not grow a model-rejection notice.""" + _patch_delegation_cfg(monkeypatch, model="upstage/solar-pro4") evt = _make_async_evt( is_batch=True, model="upstage/solar-pro4", @@ -943,9 +955,10 @@ def test_batch_model_rejection_notice_absent_when_clean(): assert "SUBAGENT MODEL REJECTED" not in text -def test_batch_model_rejection_notice_requires_configured_model_in_text(): +def test_batch_model_rejection_notice_requires_configured_model_in_text(monkeypatch): """A model_not_found pattern naming a DIFFERENT model than the configured delegation model is task-level noise, not a config-level rejection.""" + _patch_delegation_cfg(monkeypatch, model="upstage/solar-pro4") evt = _make_async_evt( is_batch=True, model="upstage/solar-pro4", From b215e8d5f9345ace732bbf5602b02c7edc440b87 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:19:13 -0700 Subject: [PATCH 047/634] chore: map contributor email for itskaism --- contributors/emails/itskaism@users.noreply.github.com | 1 + 1 file changed, 1 insertion(+) create mode 100644 contributors/emails/itskaism@users.noreply.github.com diff --git a/contributors/emails/itskaism@users.noreply.github.com b/contributors/emails/itskaism@users.noreply.github.com new file mode 100644 index 0000000000..08da9b467f --- /dev/null +++ b/contributors/emails/itskaism@users.noreply.github.com @@ -0,0 +1 @@ +itskaism From f9908e2ed641b0d071fed13f5008cad5c8da9392 Mon Sep 17 00:00:00 2001 From: Lime-oss-hash Date: Sat, 29 Aug 2026 19:30:54 +1200 Subject: [PATCH 048/634] fix(bot-mode): avoid inherited stdin on Windows Query-file DM transports do not consume stdin. Use DEVNULL for both the initial attempt and policy-gated retry so Git Bash cannot pass an invalid pseudo-handle to Windows subprocess creation. --- tests/tools/test_bot_mode_dm.py | 30 ++++++++++++++++++++++++++++++ tools/bot_mode_dm.py | 2 ++ 2 files changed, 32 insertions(+) diff --git a/tests/tools/test_bot_mode_dm.py b/tests/tools/test_bot_mode_dm.py index 61d53dc189..cc196c2982 100644 --- a/tests/tools/test_bot_mode_dm.py +++ b/tests/tools/test_bot_mode_dm.py @@ -390,6 +390,36 @@ def test_delivery_runner_preserves_child_failure_and_unlinks(tmp_path): assert not dm_file.exists() +def test_query_file_delivery_closes_stdin_for_initial_attempt_and_retry( + tmp_path, monkeypatch +): + dm_file = tmp_path / "message.txt" + dm_file.write_text("secret", encoding="utf-8") + calls = [] + responses = [ + subprocess.CompletedProcess([], 1, stdout="", stderr="HTTP 429 rate limit"), + subprocess.CompletedProcess([], 0, stdout="", stderr=""), + ] + + def fake_run(argv, **kwargs): + calls.append((argv, kwargs)) + return responses.pop(0) + + monkeypatch.setattr(subprocess, "run", fake_run) + + returncode = bot_mode_dm._run_delivery( + ["hermes", "-p", "researcher"], str(dm_file), stdin_file=False + ) + + assert returncode == 0 + assert len(calls) == 2 + assert [kwargs["stdin"] for _argv, kwargs in calls] == [ + subprocess.DEVNULL, + subprocess.DEVNULL, + ] + assert not dm_file.exists() + + @pytest.mark.parametrize("args", [[], ["--run-delivery"], ["--run-delivery", "bad", "x"]]) def test_delivery_main_rejects_invalid_cli(args): assert bot_mode_dm._delivery_main(args) == 2 diff --git a/tools/bot_mode_dm.py b/tools/bot_mode_dm.py index 0cce0cfee6..46879344c5 100644 --- a/tools/bot_mode_dm.py +++ b/tools/bot_mode_dm.py @@ -572,6 +572,7 @@ def _run_delivery(argv: list[str], dm_file: str, *, stdin_file: bool) -> int: proc = subprocess.run( [*argv, "--query-file", dm_file], check=False, + stdin=subprocess.DEVNULL, capture_output=True, text=True, ) @@ -587,6 +588,7 @@ def _run_delivery(argv: list[str], dm_file: str, *, stdin_file: bool) -> int: proc = subprocess.run( [*argv, "--query-file", dm_file], check=False, + stdin=subprocess.DEVNULL, capture_output=True, text=True, ) From 6cfadafb767eb2bb8ab69485273bd6ab871ddfaf Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:07:53 -0700 Subject: [PATCH 049/634] chore: map contributor email for @sambai-dev --- contributors/emails/yozamono@gmail.com | 1 + 1 file changed, 1 insertion(+) create mode 100644 contributors/emails/yozamono@gmail.com diff --git a/contributors/emails/yozamono@gmail.com b/contributors/emails/yozamono@gmail.com new file mode 100644 index 0000000000..5395a2f2b5 --- /dev/null +++ b/contributors/emails/yozamono@gmail.com @@ -0,0 +1 @@ +sambai-dev From a9c783f21995723c812dcb2f8ae58bc6a4323e2f Mon Sep 17 00:00:00 2001 From: Lime-oss-hash Date: Sat, 29 Aug 2026 21:38:09 +1200 Subject: [PATCH 050/634] fix(desktop): surface persistent group holds Render durable per-member hold state in Group Chat with canonical resume guidance and accessible, theme-safe status copy.\n\nVerified by independent pre-commit review. --- .../plugins/hermes-bots/group-chat-view.tsx | 6 + .../hermes-bots/group-hold-status.test.tsx | 166 ++++++++++++++++++ .../plugins/hermes-bots/group-hold-status.tsx | 60 +++++++ apps/desktop/src/plugins/hermes-bots/i18n.ts | 15 ++ 4 files changed, 247 insertions(+) create mode 100644 apps/desktop/src/plugins/hermes-bots/group-hold-status.test.tsx create mode 100644 apps/desktop/src/plugins/hermes-bots/group-hold-status.tsx diff --git a/apps/desktop/src/plugins/hermes-bots/group-chat-view.tsx b/apps/desktop/src/plugins/hermes-bots/group-chat-view.tsx index eaf10ade53..e9749dc3c3 100644 --- a/apps/desktop/src/plugins/hermes-bots/group-chat-view.tsx +++ b/apps/desktop/src/plugins/hermes-bots/group-chat-view.tsx @@ -72,6 +72,7 @@ import { import type { GroupChatRoom } from './group-chat' import { GroupClarifyCard, GroupImageControls, GroupMentionInput } from './group-chat-parts' import type { GroupRoomPrompt } from './group-chat-parts' +import { GroupHoldStatus } from './group-hold-status' import { botGroups, groupChatMemberBots, @@ -1233,6 +1234,11 @@ export function GroupChatWorkspace({ group, members, onBack, visible = true }: G ) : null} {header} + displayName(member, botRosterMeta(member, allMeta))} + members={members} + /> {activityPanel}
diff --git a/apps/desktop/src/plugins/hermes-bots/group-hold-status.test.tsx b/apps/desktop/src/plugins/hermes-bots/group-hold-status.test.tsx new file mode 100644 index 0000000000..352d88638e --- /dev/null +++ b/apps/desktop/src/plugins/hermes-bots/group-hold-status.test.tsx @@ -0,0 +1,166 @@ +import { cleanup, render, screen } from '@testing-library/react' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { translateBots } from './i18n-test-helper' +import type { GroupMember } from './types' + +const { host } = vi.hoisted(() => ({ host: {} as Record })) + +vi.mock('@hermes/plugin-sdk', async () => { + const { pluginSdkMock } = await import('./group-test-utils') + const base = await pluginSdkMock(host) + + return { + ...base, + Button: (props: React.ComponentProps<'button'>) =>