diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 0000000000..d327e5befc --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,18 @@ +# CodeRabbit configuration. +# +# Auto-review is DISABLED repo-wide: the app was enabled at the org level on +# 2026-08-14 and immediately began reviewing every opened PR. This repo's +# merge gate is CI ("All required checks pass") plus maintainer review — +# CodeRabbit reviews carry no merge weight (protect-main ruleset requires +# 0 approving reviews), so auto-firing on the full PR firehose adds comment +# noise without gating value. +# +# The bot stays installed and summonable on demand: comment +# `@coderabbitai review` on any PR to request a one-off review, or +# `@coderabbitai ignore` to mute it on a PR it has already joined. +# +# To re-enable auto-review, flip `enabled: true` below (or delete this file — +# the app default is on). +reviews: + auto_review: + enabled: false diff --git a/.gitattributes b/.gitattributes index 553e3cd21b..80824c6758 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,3 +8,27 @@ web/package-lock.json linguist-generated=true Dockerfile text eol=lf *.dockerfile text eol=lf docker/entrypoint.sh text eol=lf + +# Enforce LF for all source/text files. Windows editors and tools default to +# CRLF; without normalization a Windows contributor's edit turns into a +# whole-file phantom diff (every line "changed" by its ending), breaks +# string-match patch tooling, and pollutes review. `text` normalizes to LF +# at check-in; `eol=lf` also checks out as LF so the working tree matches +# the index on every platform. PowerShell files are the deliberate +# exception (PS 5.1 tooling expects CRLF). +*.py text eol=lf +*.ts text eol=lf +*.tsx text eol=lf +*.js text eol=lf +*.mjs text eol=lf +*.cjs text eol=lf +*.jsx text eol=lf +*.json text eol=lf +*.yaml text eol=lf +*.yml text eol=lf +*.toml text eol=lf +*.md text eol=lf +*.css text eol=lf +*.html text eol=lf +*.svg text eol=lf +*.ps1 text eol=crlf diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 9119e0c7a9..9631cdd7fa 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -20,9 +20,33 @@ jobs: - name: grab npm 12 run: | - npm i -g npm@12 + # No-op once the bundled npm is already 12.x — saves ~5-15s/job and + # keeps the installed major aligned with the npm12 cache-key tag. + npm --version | grep -q '^12\.' || npm i -g npm@12 + + # ``setup-node``'s ``cache: npm`` only caches the ~/.npm tarball cache; + # every job still re-extracts the full workspace node_modules and reruns + # postinstalls (including the Electron binary fetch). Cache the installed + # tree itself, keyed on the lockfile, and skip ``npm ci`` on an exact + # hit. No restore-keys: a partial hit would leave a stale tree, so + # anything but an exact lockfile match reinstalls from scratch. + # The discovery job installs with --ignore-scripts, so its tree differs + # from the check jobs' — hence the distinct ``-noscripts`` key. + - name: Restore node_modules + id: node-modules-cache + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + with: + path: | + node_modules + apps/*/node_modules + ui-tui/node_modules + ui-tui/packages/*/node_modules + tests-js/node_modules + web/node_modules + key: node-modules-noscripts-${{ runner.os }}-node26-npm12-${{ hashFiles('package-lock.json') }} - uses: ./.github/actions/retry + if: steps.node-modules-cache.outputs.cache-hit != 'true' with: command: npm ci --ignore-scripts - id: set-matrix @@ -71,9 +95,31 @@ jobs: - name: grab npm 12 run: | - npm i -g npm@12 + # No-op once the bundled npm is already 12.x — saves ~5-15s/job and + # keeps the installed major aligned with the npm12 cache-key tag. + npm --version | grep -q '^12\.' || npm i -g npm@12 + + # Same rationale as the discovery job's cache above, but this ``npm ci`` + # runs WITH install scripts, so the tree includes postinstall artifacts + # (electron's postinstall unpacks its binary into node_modules/electron/ + # dist, which lives inside the cached tree — the ~/.cache/electron + # download cache is deliberately NOT cached: with npm ci skipped on hit + # it would never be read, only inflate the archive). + - name: Restore node_modules + id: node-modules-cache + uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 + with: + path: | + node_modules + apps/*/node_modules + ui-tui/node_modules + ui-tui/packages/*/node_modules + tests-js/node_modules + web/node_modules + key: node-modules-scripts-${{ runner.os }}-node26-npm12-${{ hashFiles('package-lock.json') }} - uses: ./.github/actions/retry + if: steps.node-modules-cache.outputs.cache-hit != 'true' with: command: npm ci - run: npm run --prefix ${{ matrix.package }} ${{ matrix.script }} diff --git a/AGENTS.md b/AGENTS.md index a31642ad09..5b2c944533 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1224,9 +1224,10 @@ detects process completion and triggers a new agent turn. Control verbosity of b messages with `display.background_process_notifications` in config.yaml (or `HERMES_BACKGROUND_NOTIFICATIONS` env var): -- `all` — running-output updates + final message (default) -- `result` — only the final completion message -- `error` — only the final message when exit code != 0 +- `concise` — one-line status message on completion; failures append a short output tail (default) +- `all` — running-output updates + final raw-output message +- `result` — only the final raw-output completion message +- `error` — only the final raw-output message when exit code != 0 - `off` — no watcher messages at all --- diff --git a/agent/agent_init.py b/agent/agent_init.py index 68fa17b0df..bf90925a06 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -2272,7 +2272,11 @@ def init_agent( # overrides consistent with them and let provider metadata resolve the # active model's window instead. if _config_context_length is not None and isinstance(_model_cfg, dict): - _configured_default_model = str(_model_cfg.get("default") or "").strip() + _default = _model_cfg.get("default") + if isinstance(_default, dict): + from hermes_cli.config import split_model_config_default + _default, _ = split_model_config_default(_default) + _configured_default_model = str(_default or "").strip() _configured_default_runtime_model = _configured_default_model _active_runtime_model = agent.model if _configured_default_model: diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 94694db158..502e6fdfc3 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -650,6 +650,22 @@ def repair_message_sequence(agent, messages: List[Dict]) -> int: prev["tool_calls"] = prev_calls + new_calls elif prev_calls: prev["tool_calls"] = prev_calls + else: + # Neither turn carries tool calls, but the surviving turn may + # still carry a stale ``tool_calls: []`` from the earlier + # message. An empty array is semantically "no tool calls", + # yet strict OpenAI-compatible providers (DeepSeek v4, + # Moonshot/Kimi) reject it with HTTP 400 ("Invalid + # 'messages[N].tool_calls': empty array..."). Drop the key + # HERE, at the source: ``sanitize_api_messages`` only fixes + # the per-call wire copy, so a ``[]`` left on the repaired + # turn survives in the live/persisted trajectory returned to + # callers (gateway/WebUI transcripts, session resume, + # subagents, cron) and is replayed on the next turn — which + # is how #58755 kept reproducing after the chokepoint fix + # (#77921). Popping is non-destructive: an empty array + # carries no information. + prev.pop("tool_calls", None) # Concatenate plain-text content; leave multimodal (list) # content on either side alone to avoid mangling attachment # blocks — fall back to keeping the existing content. @@ -943,6 +959,7 @@ def recover_with_credential_pool( has_retried_429: bool, classified_reason: Optional[FailoverReason] = None, error_context: Optional[Dict[str, Any]] = None, + billing_unverified: bool = False, ) -> tuple[bool, bool]: """Attempt credential recovery via pool rotation. @@ -957,6 +974,12 @@ def recover_with_credential_pool( providers that surface billing/rate-limit/auth conditions under a different status code, such as Anthropic returning HTTP 400 for "out of extra usage". + + `billing_unverified` marks a billing verdict that rests on an ambiguous + body (``ClassifiedError.billing_unverified``, #82154): the pool persists + it as ``billing_unverified`` so the exhausted entry gets a short cooldown + instead of the one-hour billing bench — the same 400 can be a + content-filter rejection that leaves the credential healthy. """ pool = agent._credential_pool if pool is None: @@ -1049,7 +1072,13 @@ def recover_with_credential_pool( # cooldowns — the pool can only tell them apart if we say which. # ``effective_reason`` is resolved below; this closure runs after. if effective_reason is not None: - kwargs["failure_reason"] = effective_reason.value + _failure_reason = effective_reason.value + if effective_reason == FailoverReason.billing and billing_unverified: + # Ambiguous billing body (#82154): persist the ambiguity so + # the cooldown is sized as transient, not a 1-hour bench. + from agent.credential_pool import FAILURE_REASON_BILLING_UNVERIFIED + _failure_reason = FAILURE_REASON_BILLING_UNVERIFIED + kwargs["failure_reason"] = _failure_reason return pool.mark_exhausted_and_rotate(**kwargs) effective_reason = classified_reason @@ -2220,6 +2249,9 @@ def anthropic_prompt_cache_policy( logger.debug("MoA aggregator cache-policy resolution failed: %s", _moa_exc) return False, False + if isinstance(eff_model, dict): + eff_model = eff_model.get('model') or eff_model.get('default') or '' + eff_model = eff_model if isinstance(eff_model, str) else str(eff_model or '') model_lower = eff_model.lower() provider_lower = eff_provider.lower() is_claude = "claude" in model_lower @@ -2238,13 +2270,38 @@ def anthropic_prompt_cache_policy( # Nous Portal proxies to OpenRouter behind the scenes — identical # OpenAI-wire envelope cache_control semantics. Treat it as an # OpenRouter-equivalent endpoint for caching layout purposes. - is_nous_portal = "nousresearch" in eff_base_url.lower() + is_nous_portal = base_url_host_matches(eff_base_url, "nousresearch.com") is_anthropic_wire = eff_api_mode == "anthropic_messages" is_native_anthropic = ( is_anthropic_wire and (eff_provider == "anthropic" or base_url_hostname(eff_base_url) == "api.anthropic.com") ) + # A custom Anthropic-compatible route may use a bare model alias that is + # canonicalized only after Hermes sends the request. In that case model + # spelling cannot prove cache support. Honor an exact route+model + # capability declaration instead; explicit false is authoritative too. + # This preserves the runtime model id (and therefore request/cache keys) + # while avoiding unsafe alias-name guesses. + custom_prompt_caching = None + if is_anthropic_wire: + try: + from hermes_cli.config import get_custom_provider_model_capability + + custom_prompt_caching = get_custom_provider_model_capability( + model=eff_model, + base_url=eff_base_url, + capability="prompt_caching", + custom_providers=getattr(agent, "_custom_providers", None), + ) + except Exception as _cap_exc: + logger.debug( + "custom-provider prompt_caching capability lookup failed: %s", + _cap_exc, + ) + if custom_prompt_caching is not None: + return custom_prompt_caching, custom_prompt_caching + # MiniMax-M3 rides MiniMax's server-side automatic prefix cache on the # Anthropic wire (content-keyed, no marker needed); explicit cache_control # is documented for M2.7/M2.5/M2.1/M2 only, so markers on M3 are dead @@ -2343,6 +2400,17 @@ def create_openai_client(agent, client_kwargs: dict, *, reason: str, shared: boo # copy locks the contract so future transport/keepalive work can't reintroduce # the same class of bug. client_kwargs = dict(client_kwargs) + # The MoA virtual provider has no real OpenAI wire endpoint - the facade + # *is* the client. Rebuilding a native OpenAI client while + # agent.provider == "moa" (client replacement, stream-retry pool cleanup, + # credential rotation, fallback+restore) drops the facade: the next primary + # call either raises a `_moa_prepared_request` TypeError (#78382) or, when + # _client_kwargs carry an unrelated relay base_url, leaks the request to a + # foreign gateway. Rebuild the facade instead (build_moa_facade also + # re-wires the reference relay, see #53802). + if (getattr(agent, "provider", "") or "").strip().lower() == "moa": + from agent.moa_loop import build_moa_facade + return build_moa_facade(agent, getattr(agent, "model", None) or "default") ssl_ca_cert = client_kwargs.pop("ssl_ca_cert", None) ssl_verify_cfg = client_kwargs.pop("ssl_verify", None) httpx_verify = resolve_httpx_verify(ca_bundle=ssl_ca_cert, ssl_verify=ssl_verify_cfg) @@ -2745,6 +2813,13 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo ) # ── Re-evaluate prompt caching ── + # Refresh the custom-provider snapshot from the config just loaded above + # so the per-model ``prompt_caching`` capability lookup sees the same + # live list the context-length resolution used — without this, a flag + # added to config.yaml after session start is invisible to a /model + # switch (the policy would read the stale init-time snapshot). + if _sm_custom_providers is not None: + agent._custom_providers = _sm_custom_providers agent._use_prompt_caching, agent._use_native_cache_layout = ( agent._anthropic_prompt_cache_policy( provider=new_provider, @@ -3015,6 +3090,7 @@ def invoke_tool(agent, function_name: str, function_args: dict, effective_task_i around_message_id=next_args.get("around_message_id"), window=next_args.get("window", 5), sort=next_args.get("sort"), + detail=next_args.get("detail", "adaptive"), db=session_db, current_session_id=agent.session_id, ), @@ -3552,8 +3628,10 @@ def sanitize_api_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any] if cid: seen_assistant_call_ids.add(cid) kept_tcs.append(tc) - if len(kept_tcs) != len(msg.get("tool_calls") or []): + if kept_tcs: msg = {**msg, "tool_calls": kept_tcs} + elif len(kept_tcs) != len(msg.get("tool_calls") or []): + msg = {k: v for k, v in msg.items() if k != "tool_calls"} deduped.append(msg) elif role == "tool": cid = (msg.get("tool_call_id") or "").strip() @@ -3842,7 +3920,9 @@ def _iter_pool_sockets(client: Any): traversal defensive because these are private transport internals and vary across httpx/httpcore releases. - Also walks ``httpx`` mount transports — see ``_iter_httpx_pool_objects``. + Also walks ``httpx`` mount transports — see ``_iter_httpx_pool_objects`` + — and in-flight httpcore ``PoolRequest.connection`` objects, which stay + reachable even when ``_connections`` is empty during checkout (#85252). """ try: http_client = getattr(client, "_client", None) @@ -3859,12 +3939,18 @@ def _iter_pool_sockets(client: Any): seen: set[int] = set() for pool in pools: - connections = ( - getattr(pool, "_connections", None) - or getattr(pool, "_pool", None) - or [] - ) - for conn in list(connections): + # Empty-list is falsy: use ``is None`` so an empty ``_connections`` + # still lets us walk in-flight ``_requests`` rather than skipping + # the pool entirely. + raw_conns = getattr(pool, "_connections", None) + if raw_conns is None: + raw_conns = getattr(pool, "_pool", None) + connections = list(raw_conns or []) + for pool_req in list(getattr(pool, "_requests", None) or []): + conn = getattr(pool_req, "connection", None) + if conn is not None: + connections.append(conn) + for conn in connections: for candidate in _connection_candidates(conn): stream = ( getattr(candidate, "_network_stream", None) @@ -4139,6 +4225,16 @@ def force_close_tcp_sockets(client: Any) -> int: try: for sock in _iter_pool_sockets(client): try: + # Clear a blocking timeout first so a hung SSL_read on the + # owner thread notices the shutdown. Some stacks ignore + # SHUT_RDWR alone while recv is blocked with timeout=None + # (#85252). Still no close() — that is the #29507 race. + settimeout = getattr(sock, "settimeout", None) + if callable(settimeout): + try: + settimeout(0) + except OSError: + pass sock.shutdown(_socket.SHUT_RDWR) except OSError: # Already shut down / not connected / FD invalid — all benign. diff --git a/agent/anthropic_adapter.py b/agent/anthropic_adapter.py index 3c36ee3b91..68b8ca228d 100644 --- a/agent/anthropic_adapter.py +++ b/agent/anthropic_adapter.py @@ -2814,7 +2814,26 @@ def convert_messages_to_anthropic( p.get("cache_control") for p in content if isinstance(p, dict) ) if has_cache: - system = [p for p in content if isinstance(p, dict)] + # Copy blocks before coercing so the caller's message + # dicts are never mutated, then replace blank/whitespace + # text with the shared non-whitespace placeholder — + # Anthropic rejects a blank system text block with the + # same HTTP 400 as message blocks ("text content blocks + # must contain non-whitespace text"), and a blank block + # carrying a cache_control breakpoint cannot simply be + # dropped (#70909). + system = [] + for p in content: + if not isinstance(p, dict): + continue + if ( + p.get("type") == "text" + and isinstance(p.get("text"), str) + and not p["text"].strip() + ): + p = dict(p) + p["text"] = _EMPTY_TEXT_PLACEHOLDER + system.append(p) else: system = "\n".join( p["text"] for p in content if p.get("type") == "text" diff --git a/agent/auxiliary_client.py b/agent/auxiliary_client.py index 307d0be442..d49a797ed8 100644 --- a/agent/auxiliary_client.py +++ b/agent/auxiliary_client.py @@ -1271,13 +1271,20 @@ def _to_openai_base_url(base_url: str) -> str: Anthropic-**only** custom gateways (path ends in ``/anthropic`` but has no sibling ``/v1``) must keep their path; rewriting them to ``/v1`` yields 404 on compression/vision/title_generation (#83642). + + ZAI exposes its general API and Coding Plan on separate endpoints. Its + Anthropic-compatible Coding Plan endpoint maps to ``/api/coding/paas/v4`` + on the OpenAI wire, not the general ``/api/paas/v4`` endpoint. Rewriting + to the general endpoint changes the billing pool and can return a false + insufficient-balance error for a valid Coding Plan key. """ url = str(base_url or "").strip().rstrip("/") if url.endswith("/anthropic"): - # ZAI (open.bigmodel.cn) uses /api/anthropic for Anthropic wire - # but /api/paas/v4 for OpenAI wire — the generic /v1 rewrite is wrong. - if "open.bigmodel.cn" in url or "bigmodel" in url: - rewritten = url[: -len("/anthropic")] + "/paas/v4" + # ZAI uses /api/anthropic for the Coding Plan's Anthropic wire. The + # matching OpenAI-wire endpoint is /api/coding/paas/v4; /api/paas/v4 + # is the independently billed general API. + if base_url_host_matches(url, "open.bigmodel.cn") or base_url_host_matches(url, "api.z.ai"): + rewritten = url[: -len("/anthropic")] + "/coding/paas/v4" logger.debug("Auxiliary client: rewrote ZAI base URL %s → %s", url, rewritten) return rewritten if _is_dual_surface_anthropic_host(url): @@ -1290,7 +1297,7 @@ def _to_openai_base_url(base_url: str) -> str: url, ) return url - if "api.kimi.com" in url and url.endswith("/coding"): + if base_url_host_matches(url, "api.kimi.com") and url.endswith("/coding"): # Kimi Code uses /coding/v1/messages for Anthropic SDK (appends /v1/messages) # but /coding/v1/chat/completions for OpenAI SDK (appends /chat/completions) # Without /v1 here, OpenAI SDK hits /coding/chat/completions — a 404. @@ -1493,8 +1500,15 @@ class _CodexCompletionsAdapter: # build_kwargs, so they need the same guard applied independently. _host_for_input = str(getattr(self._client, "base_url", "") or "") _is_github_for_input = base_url_host_matches(_host_for_input, "githubcopilot.com") + # Auxiliary calls never send ``context_management`` (native + # compaction is a main-turn feature), so they must never replay a + # compaction checkpoint from the replayed history nor let one + # restructure this request — the summarizer/aggregator model is + # usually not even the one that minted the blob. input_items = _chat_messages_to_responses_input( - replay_messages, is_github_responses=_is_github_for_input, + replay_messages, + is_github_responses=_is_github_for_input, + native_compaction_eligible=False, ) resp_kwargs: Dict[str, Any] = { @@ -1613,14 +1627,18 @@ class _CodexCompletionsAdapter: or base_url_host_matches(_host_src, "models.github.ai") ) if not _is_xai and not _is_github and "prompt_cache_key" not in resp_kwargs: - # Scope by the owning turn's session so two unrelated sessions - # with the same instructions/tools (e.g. compression, MoA, - # flush_memories firing back-to-back on different sessions) - # don't bucket-share a prompt cache slot (#78941). The main - # transport (agent/transports/codex.py::build_kwargs) does the - # same; this adapter had no session handle before - # set_runtime_main() started threading one through. - _scope = _cache_scope_from_session_id(_runtime_main_value("session_id")) + # Scope by the owning turn's conversation so two unrelated + # sessions with the same instructions/tools (e.g. compression, + # MoA, flush_memories firing back-to-back on different + # sessions) don't bucket-share a prompt cache slot (#78941). + # Prefer the rotation-stable logical scope threaded through + # set_runtime_main() (compression-lineage root, #79017) and + # fall back to the physical session id, mirroring the main + # transport (agent/transports/codex.py::build_kwargs). + _scope = _cache_scope_from_session_id( + _runtime_main_value("cache_scope") + or _runtime_main_value("session_id") + ) _cache_key = _content_cache_key(instructions, resp_kwargs.get("tools"), _scope) if _cache_key: resp_kwargs["prompt_cache_key"] = _cache_key @@ -2195,7 +2213,15 @@ class _BedrockCompletionsAdapter: model=model, messages=messages, tools=kwargs.get("tools"), - max_tokens=int(max_tokens) if max_tokens else 4096, + # Omitted/None caller cap → None: build_converse_kwargs then omits + # inferenceConfig.maxTokens so Bedrock uses the model's maximum + # allowed output, matching the no-cap-by-default policy every + # other aux wire already follows (#10809: vision descriptions + # stayed capped at the shim's old hardcoded 4096 on Bedrock). + # Truthiness (not `is None`) is deliberate — it matches the + # sibling Anthropic shim's reading of max_tokens above, so a + # nonsense explicit 0 is treated as "no cap" on both wires. + max_tokens=int(max_tokens) if max_tokens else None, temperature=kwargs.get("temperature"), top_p=kwargs.get("top_p"), stop_sequences=stop, @@ -3363,11 +3389,17 @@ def set_runtime_main( api_mode: str = "", auth_mode: str = "", session_id: str = "", + cache_scope: str = "", ) -> contextvars.Token: """Record the current context's live main runtime for auxiliary routing. Context-local state prevents concurrent gateway sessions from overwriting one another while retaining compatibility mirrors for legacy readers. + + ``cache_scope`` is the rotation-stable logical cache scope (compression- + lineage root — agent/prompt_cache_scope.py) resolved once per turn by + turn_context; auxiliary Responses calls prefer it over ``session_id`` + for prompt_cache_key derivation (#79017). """ global _RUNTIME_MAIN_PROVIDER, _RUNTIME_MAIN_MODEL global _RUNTIME_MAIN_BASE_URL, _RUNTIME_MAIN_API_KEY, _RUNTIME_MAIN_API_MODE @@ -3385,6 +3417,7 @@ def set_runtime_main( "api_mode": (api_mode or "").strip(), "auth_mode": (auth_mode or "").strip().lower(), "session_id": (session_id or "").strip(), + "cache_scope": (cache_scope or "").strip(), } # Publish authoritative context before updating locked compatibility # mirrors; concurrent sessions never read those mirrors at runtime. @@ -7509,17 +7542,71 @@ def _force_close_async_httpx(client: Any) -> None: pass -def _close_cached_client(client: Any) -> None: - """Apply the canonical best-effort close policy to one cached client.""" +def _schedule_async_close(close_result: Any, client: Any) -> None: + """Finish an async close without leaking an unawaited coroutine.""" + async def _await_close() -> None: + try: + await close_result + except Exception: + pass + finally: + _force_close_async_httpx(client) + + runner = _await_close() + try: + import asyncio as _aio + + try: + loop = _aio.get_running_loop() + except RuntimeError: + _aio.run(runner) + else: + task = loop.create_task(runner) + + def _consume(completed_task) -> None: + try: + completed_task.exception() + except BaseException: + pass + + task.add_done_callback(_consume) + runner = None + except Exception: + if runner is not None: + try: + runner.close() + except Exception: + pass + _force_close_async_httpx(client) + + +def _close_cached_client(client: Any, *, close_async: bool = False) -> None: + """Close one cached client, awaiting async transports only when safe.""" if client is None: return - _force_close_async_httpx(client) + close_fn = getattr(client, "close", None) + if not callable(close_fn): + _force_close_async_httpx(client) + return try: - close_fn = getattr(client, "close", None) - if callable(close_fn) and not inspect.iscoroutinefunction(close_fn): - close_fn() + close_result = close_fn() except Exception: - pass + _force_close_async_httpx(client) + return + if inspect.isawaitable(close_result): + if close_async: + _schedule_async_close(close_result, client) + else: + # Do not await a client owned by another live event loop. + # Closing the coroutine avoids an unawaited-coroutine warning; + # the transport is still neutered for safe eventual GC. + try: + close_result.close() + except Exception: + pass + _force_close_async_httpx(client) + return + _force_close_async_httpx(client) def shutdown_cached_clients() -> None: @@ -7527,14 +7614,34 @@ def shutdown_cached_clients() -> None: Call this during CLI shutdown, *before* the event loop is closed, to avoid ``AsyncHttpxClientWrapper.__del__`` raising on a dead loop. + + Snapshot and clear the cache under the lock, then close transports outside + it. Async transport shutdown may block while an owner loop drains; holding + the global cache lock during that wait stalls unrelated auxiliary callers + and can turn teardown into a process-wide lock convoy. """ with _client_cache_lock: - for key, entry in list(_client_cache.items()): - client = entry[0] - if client is None: - continue - _close_cached_client(client) + clients = [ + (entry[0], entry[2]) + for entry in _client_cache.values() + if entry[0] is not None + ] _client_cache.clear() + try: + import asyncio as _aio + + running_loop = _aio.get_running_loop() + except RuntimeError: + running_loop = None + for client, owner_loop in clients: + # A live foreign loop owns its async transport. Calling its coroutine + # on this thread can bind/close sockets from the wrong loop; neuter it + # and let that owner finish teardown. Closed loops are safe to drain + # locally, and the current loop can await its own client. + close_async = owner_loop is not None and ( + owner_loop.is_closed() or owner_loop is running_loop + ) + _close_cached_client(client, close_async=close_async) def cleanup_stale_async_clients() -> None: @@ -7545,15 +7652,18 @@ def cleanup_stale_async_clients() -> None: This is defense-in-depth — the primary fix is ``neuter_async_httpx_del`` which disables ``__del__`` entirely. """ + stale_clients = [] with _client_cache_lock: stale_keys = [] for key, entry in _client_cache.items(): client, _default, cached_loop = entry if cached_loop is not None and cached_loop.is_closed(): - _force_close_async_httpx(client) stale_keys.append(key) + stale_clients.append(client) for key in stale_keys: del _client_cache[key] + for client in stale_clients: + _close_cached_client(client, close_async=True) def _is_openrouter_client(client: Any) -> bool: @@ -7646,7 +7756,12 @@ def _get_cached_client( effective = _compat_model(cached_client, model, cached_default) return cached_client, effective # Stale — evict and fall through to create a new client. - _force_close_async_httpx(cached_client) + # Only a client whose owner loop is closed may be awaited from + # this thread; a live foreign loop remains force-neutered. + owner_loop_closed = ( + cached_loop is not None and cached_loop.is_closed() + ) + _close_cached_client(cached_client, close_async=owner_loop_closed) del _client_cache[cache_key] else: effective = _compat_model(cached_client, model, cached_default) @@ -7696,7 +7811,7 @@ def _get_cached_client( client, default_model, _ = _client_cache[cache_key] # This concurrently built loser was never exposed to a caller, # so it is safe to close immediately. - _close_cached_client(built_client) + _close_cached_client(built_client, close_async=async_mode) return client, model or default_model diff --git a/agent/bedrock_adapter.py b/agent/bedrock_adapter.py index c399081619..8d63323fd2 100644 --- a/agent/bedrock_adapter.py +++ b/agent/bedrock_adapter.py @@ -1019,7 +1019,7 @@ def build_converse_kwargs( model: str, messages: List[Dict], tools: Optional[List[Dict]] = None, - max_tokens: int = 4096, + max_tokens: Optional[int] = 4096, temperature: Optional[float] = None, top_p: Optional[float] = None, stop_sequences: Optional[List[str]] = None, @@ -1028,16 +1028,24 @@ def build_converse_kwargs( """Build kwargs for ``bedrock-runtime.converse()`` or ``converse_stream()``. Converts OpenAI-format inputs to Converse API parameters. + + ``max_tokens=None`` omits ``inferenceConfig.maxTokens`` entirely, in which + case Bedrock defaults to the model's maximum allowed output — the Converse + field is optional per the AWS API reference. The default stays 4096 so + existing callers are unaffected; callers that want the model's full output + budget (e.g. uncapped auxiliary vision calls) pass ``None`` explicitly. """ system_prompt, converse_messages = convert_messages_to_converse(messages) cache_enabled = _model_supports_prompt_cache(model) + inference_config: Dict[str, Any] = {} + if max_tokens is not None: + inference_config["maxTokens"] = max_tokens + kwargs: Dict[str, Any] = { "modelId": model, "messages": converse_messages, - "inferenceConfig": { - "maxTokens": max_tokens, - }, + "inferenceConfig": inference_config, } if system_prompt: @@ -1086,6 +1094,10 @@ def build_converse_kwargs( if guardrail_config: kwargs["guardrailConfig"] = guardrail_config + if not kwargs["inferenceConfig"]: + # inferenceConfig is optional on the wire; don't send an empty object. + del kwargs["inferenceConfig"] + return kwargs @@ -1094,7 +1106,7 @@ def call_converse( model: str, messages: List[Dict], tools: Optional[List[Dict]] = None, - max_tokens: int = 4096, + max_tokens: Optional[int] = 4096, temperature: Optional[float] = None, top_p: Optional[float] = None, stop_sequences: Optional[List[str]] = None, @@ -1135,7 +1147,7 @@ def call_converse_stream( model: str, messages: List[Dict], tools: Optional[List[Dict]] = None, - max_tokens: int = 4096, + max_tokens: Optional[int] = 4096, temperature: Optional[float] = None, top_p: Optional[float] = None, stop_sequences: Optional[List[str]] = None, diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index 658919a5fd..e998f1a85d 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -29,12 +29,16 @@ from typing import Any, Dict, Optional from hermes_cli.timeouts import get_provider_request_timeout, get_provider_stale_timeout from hermes_constants import PARTIAL_STREAM_STUB_ID, FINISH_REASON_LENGTH -from agent.error_classifier import FailoverReason +from agent.error_classifier import ( + FailoverReason, + PROVIDER_STREAM_NON_JSON_ERROR_CODE, +) from agent.errors import EmptyStreamError from agent.turn_context import substitute_api_content from agent.gemini_native_adapter import is_native_gemini_base_url from agent.model_metadata import is_local_endpoint from agent.message_content import flatten_message_text +from agent.message_metadata import append_message, stamp_message_timestamp from agent.message_sanitization import ( _sanitize_surrogates, _repair_tool_call_arguments, @@ -46,6 +50,9 @@ from utils import base_url_host_matches, base_url_hostname, env_float, env_int logger = logging.getLogger(__name__) _OPENROUTER_PROVIDER_SORT_VALUES = {"throughput", "latency", "price"} +_PROVIDER_STREAM_ERROR_FINISH_REASONS = {"error", "error_finish"} +_PROVIDER_STREAM_SSE_FIELDS = {"event", "data", "id", "retry"} +_PROVIDER_STREAM_ERROR_TEXT_LIMIT = 4096 # When the fallback chain is fully exhausted on a non-rate-limit failure # (e.g. every provider returns a non-retryable client error like HTTP 400), @@ -65,6 +72,35 @@ def _context_thread_target(callback): return lambda: context.run(callback) +def _join_worker_for_relay_teardown(worker, *, label: str) -> None: + """Bounded worker join before raising InterruptedError (#81521). + + Raising immediately lets turn teardown (finish_logical_calls / + end_turn / close_session) race a still-open Relay physical LLM scope + and corrupt the LIFO stack — "scope handle is not at the top of the + stack" → CLI EIO / redraw storm. Only joins when Relay managed + execution is actually live: when no Relay consumers are registered + there is no scope to unwind, and the join would just delay interrupt + detection (tests/run_agent/test_interrupt_propagation.py). + """ + try: + from agent import relay_runtime + + runtime = relay_runtime.get_runtime(create=False) + if runtime is None or not runtime.managed_execution_enabled(): + return + except Exception: + return + worker.join(timeout=2.0) + if worker.is_alive(): + logger.warning( + "%s worker still alive after interrupt abort (2.0s join " + "timeout); Relay teardown will best-effort drain orphaned " + "scopes (#81521).", + label, + ) + + def _ra(): """Lazy ``run_agent`` reference. @@ -76,6 +112,353 @@ def _ra(): return run_agent +class ProviderStreamError(Exception): + """Provider encoded an API error as streaming content instead of an SDK error.""" + + def __init__( + self, + *, + status_code: Optional[int], + body: dict, + raw_text: str, + headers: Any = None, + ): + self.status_code = status_code + self.body = body + self.raw_text = raw_text + self.response = SimpleNamespace(headers=headers or {}) + super().__init__(self._format_message()) + + def _format_message(self) -> str: + error_obj = self.body.get("error", {}) if isinstance(self.body, dict) else {} + code = error_obj.get("code") if isinstance(error_obj, dict) else None + message = error_obj.get("message") if isinstance(error_obj, dict) else None + parts = ["Provider stream returned an error event"] + if self.status_code: + parts.append(f"HTTP {self.status_code}") + if code: + parts.append(str(code)) + text = " - ".join(parts) + if message: + text += f": {message}" + return text + + +def _status_code_from_value(value: Any) -> Optional[int]: + if isinstance(value, int) and 100 <= value < 600: + return value + if not isinstance(value, str): + return None + match = re.search(r"(?:HTTP_STATUS/)?\b([1-5]\d\d)\b", value, re.IGNORECASE) + if not match: + return None + try: + return int(match.group(1)) + except ValueError: + return None + + +def _status_code_from_payload(payload: Any) -> Optional[int]: + if not isinstance(payload, dict): + return None + + candidates = [ + payload.get("status_code"), + payload.get("status"), + payload.get("http_status"), + ] + error_obj = payload.get("error") + if isinstance(error_obj, dict): + candidates.extend([ + error_obj.get("status_code"), + error_obj.get("status"), + error_obj.get("http_status"), + error_obj.get("code"), + ]) + candidates.append(payload.get("code")) + + for candidate in candidates: + status_code = _status_code_from_value(candidate) + if status_code is not None: + return status_code + return None + + +def _json_object_from_text(text: str) -> Optional[dict]: + stripped = (text or "").strip() + if not stripped.startswith("{"): + return None + try: + decoded = json.loads(stripped) + except (json.JSONDecodeError, TypeError): + return None + return decoded if isinstance(decoded, dict) else None + + +def _parse_provider_sse_events(text: str) -> list[dict]: + """Parse provider text that looks like Server-Sent Events.""" + events: list[dict] = [] + current = {"event": None, "data": [], "comments": [], "fields": {}} + + def _has_event_data(event: dict) -> bool: + return bool( + event.get("event") + or event.get("data") + or event.get("comments") + or event.get("fields") + ) + + def _flush_current(): + nonlocal current + if _has_event_data(current): + data_text = "\n".join(current["data"]) + status_candidates = list(current["comments"]) + for key in ("status", "status_code", "http_status"): + if key in current["fields"]: + status_candidates.append(current["fields"][key]) + events.append({ + "event": current["event"], + "data": data_text, + "comments": list(current["comments"]), + "fields": dict(current["fields"]), + "status_code": next( + ( + status + for status in ( + _status_code_from_value(value) + for value in status_candidates + ) + if status is not None + ), + None, + ), + }) + current = {"event": None, "data": [], "comments": [], "fields": {}} + + for raw_line in (text or "").splitlines(): + line = raw_line.rstrip("\r") + if line == "": + _flush_current() + continue + if line.startswith(":"): + current["comments"].append(line[1:].strip()) + continue + + field, sep, value = line.partition(":") + if not sep: + current["fields"][field.strip().lower()] = "" + continue + field = field.strip().lower() + if value.startswith(" "): + value = value[1:] + if field == "event": + current["event"] = value.strip() + elif field == "data": + current["data"].append(value) + else: + current["fields"][field] = value + + _flush_current() + return events + + +def _provider_error_body(payload: dict, status_code: Optional[int]) -> dict: + """Normalize common provider error payloads to OpenAI-style body.error.""" + if isinstance(payload, dict): + error_obj = payload.get("error") + if isinstance(error_obj, dict): + return payload + else: + payload = {} + + code = ( + payload.get("code") + or payload.get("error_code") + or payload.get("type") + or (f"HTTP_{status_code}" if status_code else "provider_stream_error") + ) + message = ( + payload.get("message") + or payload.get("error_description") + or payload.get("error") + or "Provider stream returned an error event." + ) + normalized_error = {"message": str(message)} + if code: + normalized_error["code"] = str(code) + for key in ("request_id", "param", "type"): + if payload.get(key): + normalized_error[key] = payload[key] + return {"error": normalized_error} + + +def _provider_stream_error_from_json_decode_error( + error: json.JSONDecodeError, + *, + response: Any = None, +) -> ProviderStreamError: + """Preserve plain-text SSE data rejected inside the OpenAI SDK. + + OpenAI-compatible providers occasionally send ``event: error`` with a + non-JSON ``data:`` field. The SDK raises from ``sse.json()`` before it can + yield a completion chunk, but ``JSONDecodeError.doc`` still contains the + provider's original message. + """ + from agent.redact import redact_sensitive_text + + raw_text = str(getattr(error, "doc", "") or "").strip() + safe_text = redact_sensitive_text( + _sanitize_surrogates(raw_text), + force=True, + ) + safe_text = safe_text[:_PROVIDER_STREAM_ERROR_TEXT_LIMIT] + message = safe_text or "Provider stream returned non-JSON SSE data." + headers = getattr(response, "headers", None) if response is not None else None + + return ProviderStreamError( + status_code=None, + body=_provider_error_body( + { + "code": PROVIDER_STREAM_NON_JSON_ERROR_CODE, + "message": message, + }, + None, + ), + raw_text=safe_text, + headers=headers, + ) + + +def _iter_provider_stream_chunks(stream, *, response: Any = None): + """Yield SDK chunks while translating SDK-level SSE decode failures.""" + try: + yield from stream + except json.JSONDecodeError as error: + stream_response = response() if callable(response) else response + if stream_response is None: + stream_response = getattr(stream, "response", None) + raise _provider_stream_error_from_json_decode_error( + error, + response=stream_response, + ) from error + + +def _payload_has_error_shape(payload: Any) -> bool: + if not isinstance(payload, dict): + return False + if isinstance(payload.get("error"), (dict, str)): + return True + if payload.get("message") and ( + payload.get("code") + or payload.get("error_code") + or _status_code_from_payload(payload) is not None + ): + return True + return False + + +def _provider_stream_text_may_be_sse(text: str) -> bool: + """Return True while pending text still looks like an SSE control block.""" + stripped = (text or "").lstrip() + if not stripped: + return False + + lines = stripped.splitlines() + trailing_newline = stripped.endswith(("\n", "\r")) + saw_sse_field = False + + for index, raw_line in enumerate(lines): + line = raw_line.rstrip("\r") + if line == "": + continue + if line.startswith(":"): + saw_sse_field = True + continue + + field, sep, _value = line.partition(":") + field_name = field.strip().lower() + if sep and field_name in _PROVIDER_STREAM_SSE_FIELDS: + saw_sse_field = True + continue + + is_last_incomplete = index == len(lines) - 1 and not trailing_newline + if is_last_incomplete and any( + sse_field.startswith(field_name) + for sse_field in _PROVIDER_STREAM_SSE_FIELDS + ): + return True + return False + + return saw_sse_field + + +def _provider_stream_error_from_text( + text: str, + finish_reason: Optional[str], + *, + response: Any = None, +) -> Optional[ProviderStreamError]: + """Convert provider-streamed error text into an exception for retry logic.""" + if not text: + return None + + finish_reason_text = str(finish_reason or "").lower() + has_error_finish = finish_reason_text in _PROVIDER_STREAM_ERROR_FINISH_REASONS + if not has_error_finish: + return None + + for event in _parse_provider_sse_events(text): + event_name = str(event.get("event") or "").strip().lower() + payload = _json_object_from_text(event.get("data") or "") or {} + status_code = event.get("status_code") or _status_code_from_payload(payload) + is_error_event = event_name == "error" + is_http_error = status_code is not None and status_code >= 400 + is_error_payload = _payload_has_error_shape(payload) + is_structured_error_event = is_error_event and ( + has_error_finish or is_http_error or is_error_payload + ) + is_bare_error_finish_payload = ( + not is_error_event and has_error_finish and is_error_payload + ) + + if not ( + is_http_error + or is_structured_error_event + or is_bare_error_finish_payload + ): + continue + + headers = getattr(response, "headers", None) if response is not None else None + return ProviderStreamError( + status_code=status_code, + body=_provider_error_body(payload, status_code), + raw_text=text, + headers=headers, + ) + + payload = _json_object_from_text(text) + if payload is not None: + status_code = _status_code_from_payload(payload) + if has_error_finish or (status_code is not None and status_code >= 400): + headers = getattr(response, "headers", None) if response is not None else None + return ProviderStreamError( + status_code=status_code, + body=_provider_error_body(payload, status_code), + raw_text=text, + headers=headers, + ) + + if has_error_finish and text.strip(): + headers = getattr(response, "headers", None) if response is not None else None + return ProviderStreamError( + status_code=None, + body=_provider_error_body({}, None), + raw_text=text, + headers=headers, + ) + return None + + def estimate_request_context_tokens(api_payload: Any) -> int: """Estimate context/load tokens from an API payload, dict or messages list. @@ -196,6 +579,24 @@ def _provider_preferences_for_agent(agent) -> Dict[str, Any]: return preferences +def _prompt_cache_scope_for_agent(agent) -> "str | None": + """Rotation-stable logical cache scope for *agent*, or None. + + Guarded-import wrapper over the never-raising + ``agent.prompt_cache_scope.resolve_prompt_cache_scope_safe`` — the + transports treat a None/empty value as "fall back to the physical + session_id", so any resolution failure degrades to pre-#79017 behavior + instead of blocking the request build. + """ + try: + from agent.prompt_cache_scope import resolve_prompt_cache_scope_safe + + return resolve_prompt_cache_scope_safe(agent) + except Exception: + logger.debug("prompt-cache scope resolution failed", exc_info=True) + return None + + def _merge_nous_portal_messages_extra_body(agent, anthropic_kwargs: dict) -> dict: """Merge Portal ``tags`` / ``session_id`` onto an Anthropic Messages kwargs dict. @@ -552,6 +953,19 @@ def _dispatch_nonstreaming_api_request(agent, api_kwargs: dict, *, make_client): # MoA is a virtual chat-completions provider backed by the # in-process MoAClient facade. Do not rebuild a request-local # OpenAI client from the virtual runtime metadata. + # + # After a client replacement (credential rotation / + # dead-connection cleanup / fallback+restore), agent.client may + # become a native OpenAI client while agent.provider stays + # "moa". Pop the MoA-internal key so the native SDK does not + # reject it as an unexpected kwarg — but only when the live + # client is NOT the facade: the facade consumes the key, and + # stripping it there forces a wasteful duplicate reference + # fan-out (the facade re-prepares from scratch). Only the MoA + # facade's completions object exposes ``prepare()``. (#78382) + _completions = getattr(getattr(agent.client, "chat", None), "completions", None) + if not callable(getattr(_completions, "prepare", None)): + api_kwargs.pop("_moa_prepared_request", None) return agent.client.chat.completions.create(**api_kwargs) request_client = make_client("chat_completion_request") return request_client.chat.completions.create(**api_kwargs) @@ -634,6 +1048,35 @@ def _resolve_direct_stale_timeout(agent, api_kwargs: dict) -> float: return float(value) +def _inline_nonstream_hard_timeout(stale_timeout: float): + """Socket-level backstop for inline non-streaming calls (#85252). + + The keepalive httpx client uses ``read=None`` so SSE streams can idle + during reasoning. That same client serves cron/subagent non-streaming + calls. Combined with a stranger-thread abort that must not ``close()`` + the FD (#29507), a hung provider then waits until TCP dies — observed + 5–11× past the stale threshold. + + Returns an ``httpx.Timeout`` whose read budget equals the stale + watchdog, a float if httpx is unavailable, or ``None`` when the + watchdog is disarmed (local endpoint / non-finite budget). + """ + if not math.isfinite(stale_timeout) or stale_timeout <= 0: + return None + conn_cap = min(stale_timeout, 60.0) + try: + import httpx as _httpx + + return _httpx.Timeout( + connect=conn_cap, + read=stale_timeout, + write=conn_cap, + pool=conn_cap, + ) + except Exception: + return stale_timeout + + def direct_api_call(agent, api_kwargs: dict): """Run a non-streaming LLM call inline on the conversation thread. @@ -649,14 +1092,16 @@ def direct_api_call(agent, api_kwargs: dict): 450s — surfacing as ``Operation interrupted: waiting for model response``. A stale-call watchdog bounds the request the same way the interrupt - worker's poll loop does (#80759). The httpx read timeout alone is not a - usable bound: it defaults to 1800s and a provider that accepts the request - and then goes silent (connection held open, zero bytes, no error) never - trips it, so a cron run hangs until something external kills it — which - also orphans the execution row. The watchdog aborts the in-flight sockets - through the already-registered abort hook and surfaces a retryable - ``TimeoutError`` so the outer retry loop reconnects with backoff / - credential rotation / provider fallback. + worker's poll loop does (#80759). The keepalive httpx client uses + ``read=None`` (SSE), so the socket itself is not a usable bound: a + provider that accepts the request and then goes silent never trips a + read timeout, and a stranger-thread abort cannot ``close()`` the FD + (#29507). The watchdog aborts in-flight sockets through the already- + registered abort hook; a per-call ``timeout`` matching the stale budget + is the hard backstop when that abort finds nothing to shut down + (#85252). Either path surfaces a retryable ``TimeoutError`` so the + outer retry loop reconnects with backoff / credential rotation / + provider fallback. """ _check_stale_giveup(agent) agent._touch_activity("waiting for non-streaming API response") @@ -769,6 +1214,14 @@ def direct_api_call(agent, api_kwargs: dict): # stalls from the stall monitor. call_start = time.time() stale_timeout = _resolve_direct_stale_timeout(agent, api_kwargs) + # Do not override an explicit per-call timeout (provider config / + # transport already set one). Otherwise pin read=stale_timeout so a + # no-op stranger-thread abort cannot leave the keepalive client's + # read=None socket hanging until TCP dies (#85252). + hard_timeout = _inline_nonstream_hard_timeout(stale_timeout) + if hard_timeout is not None and "timeout" not in api_kwargs: + api_kwargs = dict(api_kwargs) + api_kwargs["timeout"] = hard_timeout activity_hb.start() def _on_stale() -> None: @@ -1316,6 +1769,12 @@ def interruptible_api_call(agent, api_kwargs: dict): _close_request_client_once("interrupt_abort") except Exception: pass + # #81521 (sibling of the streaming-path fix): wait for the worker + # to unwind Relay-managed scopes before surfacing + # InterruptedError, so turn teardown cannot race a still-open + # physical scope and corrupt the LIFO stack. No-op when Relay + # managed execution is not live. + _join_worker_for_relay_teardown(t, label="Non-streaming") raise InterruptedError("Agent interrupted during API call") if result["error"] is not None: raise result["error"] @@ -1375,6 +1834,12 @@ def build_api_kwargs(agent, api_messages: list, tools_for_api: list | None = Non guardrail_config=guardrail, ) + # Rotation-stable logical cache scope, shared by every OpenAI-wire branch + # below (codex + both chat_completions paths). Memoized on the agent — + # cheap after the first call. Resolved after the anthropic/bedrock early + # returns above, which don't use prompt_cache_key. + _cache_scope_id = _prompt_cache_scope_for_agent(agent) + if agent.api_mode == "codex_responses": _ct = agent._get_transport() is_github_responses = ( @@ -1440,6 +1905,7 @@ def build_api_kwargs(agent, api_messages: list, tools_for_api: list | None = Non tools=tools_for_api, reasoning_config=agent.reasoning_config, session_id=getattr(agent, "session_id", None), + cache_scope_id=_cache_scope_id, base_url=agent.base_url, max_tokens=agent.max_tokens, timeout=agent._resolved_api_call_timeout(), @@ -1465,8 +1931,8 @@ def build_api_kwargs(agent, api_messages: list, tools_for_api: list | None = Non base_url_host_matches(agent._base_url_lower, "models.github.ai") or base_url_host_matches(agent._base_url_lower, "githubcopilot.com") ) - _is_nous = "nousresearch" in agent._base_url_lower - _is_nvidia = "integrate.api.nvidia.com" in agent._base_url_lower + _is_nous = base_url_host_matches(agent._base_url_lower, "nousresearch.com") + _is_nvidia = base_url_host_matches(agent._base_url_lower, "integrate.api.nvidia.com") _is_kimi = ( base_url_host_matches(agent.base_url, "api.kimi.com") or base_url_host_matches(agent.base_url, "moonshot.ai") @@ -1549,6 +2015,7 @@ def build_api_kwargs(agent, api_messages: list, tools_for_api: list | None = Non reasoning_config=agent.reasoning_config, request_overrides=agent.request_overrides, session_id=getattr(agent, "session_id", None), + cache_scope_id=_cache_scope_id, provider_profile=_profile, ollama_num_ctx=agent._ollama_num_ctx, # Context forwarded to profile hooks: @@ -1581,6 +2048,7 @@ def build_api_kwargs(agent, api_messages: list, tools_for_api: list | None = Non reasoning_config=agent.reasoning_config, request_overrides=agent.request_overrides, session_id=getattr(agent, "session_id", None), + cache_scope_id=_cache_scope_id, model_lower=(agent.model or "").lower(), is_openrouter=_is_or, is_nous=_is_nous, @@ -1689,12 +2157,12 @@ def build_assistant_message(agent, assistant_message, finish_reason: str) -> dic # a DB-side pad can't survive ``_rows_to_conversation``'s whitespace strip # anyway. Repair belongs at the send boundary, once. - msg = { + msg = stamp_message_timestamp({ "role": "assistant", "content": _san_content, "reasoning": reasoning_text, "finish_reason": finish_reason, - } + }) raw_reasoning_content = getattr(assistant_message, "reasoning_content", None) if raw_reasoning_content is None and hasattr(assistant_message, "model_extra"): @@ -2034,6 +2502,32 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool fb_base_url_hint = (fb.get("base_url") or "").strip() or None fb_api_key_hint = resolve_entry_api_key(fb) + # Determine api_mode from the ORIGINAL base_url (before URL transformation). + # resolve_provider_client() calls _to_openai_base_url() which can rewrite + # a dual-surface /anthropic base to /v1, losing the Anthropic wire signal + # from the client's post-rewrite base_url. Pre-compute here so detection + # sees the URL the user actually configured. (#79787) + # + # An explicit ``api_mode`` on the fallback entry always wins — including + # an explicit "chat_completions" — and suppresses all re-detection below. + fb_api_mode_explicit = bool(str(fb.get("api_mode") or "").strip()) + fb_api_mode = "chat_completions" + if fb_api_mode_explicit: + fb_api_mode = str(fb.get("api_mode")).strip() + elif fb_provider == "anthropic": + # Provider-name check must not be gated on fb_base_url_hint: + # an entry that names provider: anthropic without an explicit + # base_url uses the provider's default endpoint and must still + # resolve to anthropic_messages, not chat_completions. + fb_api_mode = "anthropic_messages" + elif fb_base_url_hint: + _orig_url = fb_base_url_hint.rstrip("/").lower() + if ( + _orig_url.endswith("/anthropic") + or base_url_hostname(fb_base_url_hint) == "api.anthropic.com" + ): + fb_api_mode = "anthropic_messages" + # For Ollama Cloud endpoints, pull OLLAMA_API_KEY from env # when no explicit key is in the fallback config. Host match # (not substring) — see GHSA-76xc-57q6-vm5m. @@ -2044,7 +2538,8 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool fb_client, _resolved_fb_model = resolve_provider_client( fb_provider, model=fb_model, raw_codex=True, explicit_base_url=fb_base_url_hint, - explicit_api_key=fb_api_key_hint) + explicit_api_key=fb_api_key_hint, + api_mode=fb_api_mode) if fb_client is None: logger.warning( "Fallback to %s failed: provider not configured", @@ -2061,51 +2556,53 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool fb_model, fb_provider, _norm_err, ) - # Determine api_mode from provider / base URL / model - fb_api_mode = "chat_completions" + # Re-determine api_mode from provider / resolved base URL / model when + # the pre-computed pass above landed on the default and the user did + # not pin api_mode explicitly. An explicit fb.api_mode (even + # "chat_completions") must never be overridden here. fb_base_url = str(fb_client.base_url) _fb_is_azure = agent._is_azure_openai_url(fb_base_url) - if fb_provider == "openai-codex": - fb_api_mode = "codex_responses" - elif fb_provider in {"nous", "nous-portal", "nousresearch"}: - # Portal is dual-wire: anthropic/* must land on /v1/messages. - # resolve_provider_client still returns an OpenAI client for - # Nous; the anthropic_messages branch below rebuilds the native - # client from that credential + base_url. - from hermes_cli.providers import nous_api_mode - fb_api_mode = nous_api_mode(fb_model) - elif ( - fb_provider == "anthropic" - or fb_base_url.rstrip("/").lower().endswith("/anthropic") - or base_url_hostname(fb_base_url) == "api.anthropic.com" - ): - # Custom providers (e.g. cron-anthropic) point at the native - # api.anthropic.com host with no "/anthropic" path suffix, so the - # name/suffix checks above miss them and they default to - # chat_completions → POST /v1/chat/completions → 404. Match the - # host the same way determine_api_mode() and _detect_api_mode_for_url() - # do on the primary path. (#32243, #49247) - fb_api_mode = "anthropic_messages" - elif _fb_is_azure: - # Azure OpenAI serves gpt-5.x on /chat/completions — does NOT - # support the Responses API. Stay on chat_completions. - fb_api_mode = "chat_completions" - elif agent._is_direct_openai_url(fb_base_url): - fb_api_mode = "codex_responses" - elif agent._provider_model_requires_responses_api( - fb_model, - provider=fb_provider, - ): - # GPT-5.x models usually need Responses API, but keep - # provider-specific exceptions like Copilot gpt-5-mini on - # chat completions. - fb_api_mode = "codex_responses" - elif fb_provider == "bedrock" or ( - base_url_hostname(fb_base_url).startswith("bedrock-runtime.") - and base_url_host_matches(fb_base_url, "amazonaws.com") - ): - fb_api_mode = "bedrock_converse" + if not fb_api_mode_explicit and fb_api_mode == "chat_completions": + if fb_provider == "openai-codex": + fb_api_mode = "codex_responses" + elif fb_provider in {"nous", "nous-portal", "nousresearch"}: + # Portal is dual-wire: anthropic/* must land on /v1/messages. + # resolve_provider_client still returns an OpenAI client for + # Nous; the anthropic_messages branch below rebuilds the native + # client from that credential + base_url. + from hermes_cli.providers import nous_api_mode + + fb_api_mode = nous_api_mode(fb_model) + elif ( + fb_base_url.rstrip("/").lower().endswith("/anthropic") + or base_url_hostname(fb_base_url) == "api.anthropic.com" + ): + # Named custom providers (e.g. cron-anthropic) resolve their + # base_url from config rather than the fallback entry, so the + # pre-resolve hint check above never sees it. Match the host + # the same way determine_api_mode() and _detect_api_mode_for_url() + # do on the primary path. (#32243, #49247) + fb_api_mode = "anthropic_messages" + elif _fb_is_azure: + # Azure OpenAI serves gpt-5.x on /chat/completions — does NOT + # support the Responses API. Stay on chat_completions. + fb_api_mode = "chat_completions" + elif agent._is_direct_openai_url(fb_base_url): + fb_api_mode = "codex_responses" + elif agent._provider_model_requires_responses_api( + fb_model, + provider=fb_provider, + ): + # GPT-5.x models usually need Responses API, but keep + # provider-specific exceptions like Copilot gpt-5-mini on + # chat completions. + fb_api_mode = "codex_responses" + elif fb_provider == "bedrock" or ( + base_url_hostname(fb_base_url).startswith("bedrock-runtime.") + and base_url_host_matches(fb_base_url, "amazonaws.com") + ): + fb_api_mode = "bedrock_converse" old_model = agent.model old_provider = agent.provider @@ -2343,7 +2840,7 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: from agent.context_compressor import MAX_ITERATIONS_SUMMARY_REQUEST summary_request = MAX_ITERATIONS_SUMMARY_REQUEST - messages.append({"role": "user", "content": summary_request}) + append_message(messages, {"role": "user", "content": summary_request}) try: # Build API messages, stripping internal-only fields @@ -2566,7 +3063,10 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: final_response = re.sub(r'.*?\s*', '', final_response, flags=re.DOTALL).strip() if final_response: summary_call_outcome = "success" - messages.append({"role": "assistant", "content": final_response}) + append_message( + messages, + {"role": "assistant", "content": final_response}, + ) else: final_response = "I reached the iteration limit and couldn't generate a summary." else: @@ -2628,7 +3128,10 @@ def handle_max_iterations(agent, messages: list, api_call_count: int) -> str: final_response = re.sub(r'.*?\s*', '', final_response, flags=re.DOTALL).strip() if final_response: summary_call_outcome = "success" - messages.append({"role": "assistant", "content": final_response}) + append_message( + messages, + {"role": "assistant", "content": final_response}, + ) else: final_response = "I reached the iteration limit and couldn't generate a summary." else: @@ -2977,6 +3480,12 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= while t.is_alive(): t.join(timeout=0.3) if agent._interrupt_requested: + # #81521 (sibling of the main streaming-path fix): give + # the Bedrock worker a bounded window to unwind its + # Relay-managed stream scopes before surfacing + # InterruptedError. No-op when Relay managed execution + # is not live. + _join_worker_for_relay_teardown(t, label="Bedrock streaming") raise InterruptedError("Agent interrupted during Bedrock API call") # Liveness watchdog: no Bedrock event for longer than the stale # timeout means the stream has wedged (open socket, keep-alives but @@ -3338,6 +3847,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= request_client_holder["diag"] = _diag _writer_token = {"value": None} attempt_request_client = {"value": None} + attempt_stream_response = {"value": None} def _open_stream(next_api_kwargs: dict[str, Any]): stream_kwargs = { @@ -3366,6 +3876,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= def _stream_created(raw_stream: Any) -> None: response = getattr(raw_stream, "response", None) + attempt_stream_response["value"] = response agent._capture_rate_limits(response) agent._capture_credits(response) agent._stream_diag_capture_response(_diag, response) @@ -3452,7 +3963,31 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= # Hermes interrupts the managed stream; Relay retains sole # ownership of closing the underlying provider stream. _set_request_stream_handle(stream) - for chunk in stream: + pending_text_parts: list[str] = [] + + def _flush_pending_stream_text(): + if not pending_text_parts: + return + pending_parts = list(pending_text_parts) + pending_text_parts.clear() + if not tool_calls_acc: + for text in pending_parts: + _fire_first_delta() + agent._fire_stream_delta(text) + deltas_were_sent["yes"] = True + return + if agent.stream_delta_callback: + for text in pending_parts: + try: + agent.stream_delta_callback(text) + agent._record_streamed_assistant_text(text) + except Exception: + pass + + for chunk in _iter_provider_stream_chunks( + stream, + response=lambda: attempt_stream_response["value"], + ): last_chunk_time["t"] = time.time() agent._touch_activity("receiving stream response") @@ -3508,6 +4043,30 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= # Usage comes in the final chunk with empty choices if hasattr(chunk, "usage") and chunk.usage: usage_obj = chunk.usage + # Some OpenAI-compatible providers (DeepInfra, etc.) + # return validation errors as in-stream error chunks: + # choices=None with error_type/error_message in + # model_extra. Without this check the error is + # silently dropped and the stream ends empty → + # EmptyStreamError → misleading "empty stream" message + # and pointless retries on the same bad request. (#65631) + _err_type = getattr(chunk, "error_type", None) + _err_msg = getattr(chunk, "error_message", None) + if _err_type or _err_msg: + _status = _status_code_from_payload( + {"code": _err_type, "message": _err_msg} + ) or _status_code_from_value(_err_type) + raise ProviderStreamError( + status_code=_status, + body=_provider_error_body( + { + "code": _err_type or "provider_in_stream_error", + "message": str(_err_msg or chunk), + }, + _status, + ), + raw_text=f"{_err_type}: {_err_msg}", + ) continue delta = chunk.choices[0].delta @@ -3533,6 +4092,13 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= if delta and delta.content: content_parts.append(delta.content) if not tool_calls_acc: + if pending_text_parts or _provider_stream_text_may_be_sse(delta.content): + pending_text_parts.append(delta.content) + pending_text = "".join(pending_text_parts) + if _provider_stream_text_may_be_sse(pending_text): + continue + _flush_pending_stream_text() + continue _fire_first_delta() agent._fire_stream_delta(delta.content) deltas_were_sent["yes"] = True @@ -3556,6 +4122,7 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= # Accumulate tool call deltas — notify display on first name if delta and delta.tool_calls: + _flush_pending_stream_text() for tc_delta in delta.tool_calls: raw_idx = tc_delta.index if tc_delta.index is not None else 0 delta_id = tc_delta.id or "" @@ -3807,6 +4374,15 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= if has_truncated_tool_args: effective_finish_reason = "length" + provider_stream_error = _provider_stream_error_from_text( + full_content or "", + effective_finish_reason, + response=getattr(stream, "response", None), + ) + if provider_stream_error is not None: + raise provider_stream_error + _flush_pending_stream_text() + full_reasoning = "".join(reasoning_parts) or None mock_message = SimpleNamespace( role=role, @@ -4566,6 +5142,15 @@ def interruptible_streaming_api_call(agent, api_kwargs: dict, *, on_first_delta= _close_request_client_once("stream_interrupt_abort") except Exception: pass + # Wait for the worker to unwind Relay-managed stream scopes + # (physical LLM + deferred logical) before surfacing + # InterruptedError. Raising immediately lets turn teardown + # (finish_logical_calls / end_turn / close_session) race a + # still-open physical scope and corrupt the LIFO stack — + # "scope handle is not at the top of the stack" → CLI EIO / + # redraw storm (#81521). No-op when Relay managed execution + # is not live. + _join_worker_for_relay_teardown(t, label="Streaming") raise InterruptedError("Agent interrupted during streaming API call") # Worker thread exited before the main thread's poll loop could check # the interrupt flag. If the worker returned early due to an interrupt diff --git a/agent/codex_responses_adapter.py b/agent/codex_responses_adapter.py index 2b431cff72..ac1129d4fc 100644 --- a/agent/codex_responses_adapter.py +++ b/agent/codex_responses_adapter.py @@ -414,6 +414,7 @@ def _chat_messages_to_responses_input( is_github_responses: bool = False, replay_encrypted_reasoning: bool = True, current_issuer_kind: Optional[str] = None, + native_compaction_eligible: bool = False, ) -> List[Dict[str, Any]]: """Convert internal chat-style messages to Responses input items. @@ -458,6 +459,24 @@ def _chat_messages_to_responses_input( ``replay_encrypted_reasoning=False`` is the session-wide kill switch (drops ALL replay); ``current_issuer_kind`` is the per-item filter that runs only when replay is still enabled. + + ``native_compaction_eligible`` mirrors, for THIS request, the decision + made by ``native_compaction.native_compaction_context_management`` — it + is True only when that gate returned a payload, i.e. when the request + actually carries ``context_management``. It controls two things that + must never outlive the gate: replaying ``type: "compaction"`` checkpoint + items, and restructuring the wire around them + (``prune_pre_checkpoint_items``). Checkpoints are persisted in the + ``codex_reasoning_items`` sidecar and survive a mid-session model swap, + a ``compression.enabled: false`` flip, the rejection kill switch and a + resumed session; without this flag a single captured checkpoint would + keep deleting every pre-checkpoint item from every later request, on a + model that cannot decrypt the blob (#85914). Default False = pre-feature + wire, which is also correct for every caller that never sends + ``context_management`` (auxiliary/compression client, ad-hoc + ``convert_messages``). Dropping the checkpoint costs nothing: Hermes' + local history is never truncated by native compaction, so the full + conversation is still on the wire. """ items: List[Dict[str, Any]] = [] seen_item_ids: set = set() @@ -499,6 +518,20 @@ def _chat_messages_to_responses_input( item_id = ri.get("id") if item_id and item_id in seen_item_ids: continue + # Native-compaction gate: a checkpoint is only + # meaningful to the endpoint/model that minted it + # AND only while this request still asks for + # server-side compaction. Once the gate closes + # (model swapped out of the gpt-5.6 family, + # compression disabled, rejection kill switch), + # the persisted checkpoint must not be replayed — + # replaying it is what makes the wire restructure + # below erase pre-checkpoint history forever. + if ( + ri.get("type") == "compaction" + and not native_compaction_eligible + ): + continue # Cross-issuer guard: drop reasoning blocks that # were minted by a different Responses endpoint. # The current endpoint cannot decrypt foreign @@ -697,8 +730,13 @@ def _chat_messages_to_responses_input( # from before the boundary silently vanish from the model's view. Keep # the newest checkpoint first, retain pre-checkpoint USER messages # verbatim within a token budget (Codex CLI parity), and leave the - # post-checkpoint tail untouched. Self-gating: histories without a - # checkpoint (every non-native session) return unchanged. + # post-checkpoint tail untouched. Gated on the CURRENT request's native + # eligibility, not merely on the presence of a checkpoint: a persisted + # checkpoint outlives the gate, and pruning for a request that carries no + # ``context_management`` deletes history the server never compacted. + if not native_compaction_eligible: + return items + from agent.native_compaction import prune_pre_checkpoint_items return prune_pre_checkpoint_items(items) diff --git a/agent/codex_runtime.py b/agent/codex_runtime.py index e098751d98..f01d042b53 100644 --- a/agent/codex_runtime.py +++ b/agent/codex_runtime.py @@ -824,7 +824,10 @@ def run_codex_app_server_turn( # standard {role, content, tool_calls, tool_call_id} entries, which # is exactly what curator.py / sessions DB expect. if turn.projected_messages: - messages.extend(turn.projected_messages) + from agent.message_metadata import append_message + + for projected_message in turn.projected_messages: + append_message(messages, projected_message) # Persist the newly-projected assistant/tool messages ourselves. # This path is an early return that bypasses conversation_loop, whose diff --git a/agent/context_compressor.py b/agent/context_compressor.py index 290016d193..c416462e8a 100644 --- a/agent/context_compressor.py +++ b/agent/context_compressor.py @@ -178,6 +178,7 @@ MAX_ITERATIONS_SUMMARY_REQUEST = ( "Please provide a final response summarizing what you've found and accomplished so far, " "without calling any more tools." ) +_BACKGROUND_PROCESS_NOTIFICATION_PREFIX = "[IMPORTANT: Background process " def _fresh_compaction_message_copy(msg: Dict[str, Any]) -> Dict[str, Any]: @@ -4583,7 +4584,8 @@ This compaction should PRIORITISE preserving all information related to the focu """Recognize internal user-role rows after SessionDB projection. SessionDB preserves role/content but not underscore-prefixed metadata, - so the stable todo and continuation content markers are authoritative. + so stable runtime-notification, todo, and continuation content markers + are authoritative. """ if not isinstance(message, dict) or message.get("role") != "user": return False @@ -4619,6 +4621,8 @@ This compaction should PRIORITISE preserving all information related to the focu _LENGTH_CONTINUATION_NETWORK_STUB, _LENGTH_CONTINUATION_OUTPUT_LIMIT, } or text.startswith( + _BACKGROUND_PROCESS_NOTIFICATION_PREFIX + ) or text.startswith( TODO_INJECTION_HEADER + "\n" ) or text.startswith( _LENGTH_CONTINUATION_DROPPED_TOOLS_PREFIX @@ -6600,17 +6604,15 @@ This compaction should PRIORITISE preserving all information related to the focu _previous_summary_before_scan = self._previous_summary _summary_has_user_turn_before_scan = getattr(self, "_summary_has_user_turn", None) # A persisted handoff summary can sit in the protected head after a - # resume (commonly immediately after the system prompt). Search from - # the first non-system message through the compression window. On the - # first compaction after a restart, extend through the full transcript - # so summaries that landed in the protected tail or drifted past the - # decay probe still rehydrate iterative-summary state instead of being - # copied forward as stacked fossils. + # resume (commonly immediately after the system prompt), or later in + # the live window past a degenerate compress_end (#83248). Always + # search the full transcript for handoff rows: the content-prefix + # check is cheap, Phase 4 already advances tail_start when + # summary_idx >= compress_end, and the #57835 cross-session discard + # must only fire after a full-window miss — never after a narrow + # scan that could hide a same-session handoff beyond the cut. summary_search_start = 1 if messages and messages[0].get("role") == "system" else 0 - summary_search_end = compress_end - if self.compression_count < 1 and not self._previous_summary: - summary_search_end = len(messages) - summary_search_end = min(len(messages), summary_search_end) + summary_search_end = len(messages) summary_indices: set[int] = set() summary_idx = None summary_body = None @@ -6680,11 +6682,12 @@ This compaction should PRIORITISE preserving all information related to the focu if summary_idx >= compress_end: tail_start = summary_idx + 1 elif self._previous_summary: - # No handoff summary found in the current messages, but + # Full-window scan found no handoff in the current messages, but # _previous_summary is non-empty — it was set by a different # (now-ended) session (e.g., a cron job, a prior /new). Discard # it so _generate_summary() does not inject cross-session content # into the summarizer prompt via the iterative-update path. + # Do not clear based on a compress_end-bounded miss (#83248). self._previous_summary = None self._summary_has_user_turn = real_user_present else: diff --git a/agent/conversation_compression.py b/agent/conversation_compression.py index b62b8c8ad4..77d49ec0f8 100644 --- a/agent/conversation_compression.py +++ b/agent/conversation_compression.py @@ -71,7 +71,10 @@ from agent.context_engine import ( automatic_compaction_status_message, sanitize_memory_context, ) -from agent.model_metadata import estimate_request_tokens_rough +from agent.model_metadata import ( + estimate_messages_tokens_rough, + estimate_request_tokens_rough, +) from agent.session_activity import ActivityProvenance, normalize_activity_provenance logger = logging.getLogger(__name__) @@ -1209,27 +1212,39 @@ def _adopt_live_compression_child( session_db: Any, parent_session_id: str, ) -> Optional[List[Dict[str, Any]]]: - """Move a stale compression contender onto the unique durable child. + """Move a stale compression contender onto the live continuation tip. Resolve and load first, then mutate the live agent. This ordering keeps the stale contender fail-closed when lineage is ambiguous or the compacted handoff cannot be read. + + Resolution uses the canonical transitive walk ``get_compression_tip`` so a + lineage with >=2 compression hops (root -> mid -> tip) recovers to the live + tip — the depth-1 ``find_live_compression_child`` lookup this used to call + finds no live *direct* child in that shape and skipped recovery (#82001). + The tip walk returns the input id when no continuation exists, and a + resolved tip is adopted only while its row is still live — both cases fail + closed exactly as before. """ - finder = getattr(type(session_db), "find_live_compression_child", None) + resolver = getattr(type(session_db), "get_compression_tip", None) + row_getter = getattr(type(session_db), "get_session", None) loader = getattr(type(session_db), "get_messages_as_conversation", None) - if not callable(finder) or not callable(loader): + if not callable(resolver) or not callable(row_getter) or not callable(loader): return None - child = finder(session_db, parent_session_id) - if not child or not child.get("id"): + tip = resolver(session_db, parent_session_id) + if not tip or str(tip) == str(parent_session_id): + return None + child_session_id = str(tip) + child = row_getter(session_db, child_session_id) + if not isinstance(child, dict) or child.get("ended_at") is not None: return None - child_session_id = str(child["id"]) recovered = loader(session_db, child_session_id) if not isinstance(recovered, list) or not recovered: return None - # Revalidate after loading: the child may have rotated or a competing + # Revalidate after loading: the tip may have rotated or a competing # continuation may have appeared between the two DB reads. - confirmed = finder(session_db, parent_session_id) - if not confirmed or str(confirmed.get("id") or "") != child_session_id: + confirmed = resolver(session_db, parent_session_id) + if not confirmed or str(confirmed) != child_session_id: return None agent.session_id = child_session_id @@ -2074,10 +2089,15 @@ def _ensure_compressed_has_user_turn(original_messages: list, compressed: list) _fresh_compaction_message_copy(message), ) return - compressed.append({ - "role": "user", - "content": COMPRESSION_CONTINUATION_USER_CONTENT, - }) + from agent.message_metadata import append_message + + append_message( + compressed, + { + "role": "user", + "content": COMPRESSION_CONTINUATION_USER_CONTENT, + }, + ) _PENDING_CONTEXT_ENGINE_NOTIFICATION = ( @@ -3268,6 +3288,51 @@ def compress_context( # away regardless of whether the id rotates). agent.commit_memory_session(messages) + # Anti-growth guard at the COMMIT SITE: never persist a + # compression that makes the transcript larger (observed: + # 379K -> 687K when the generated summary plus retained + # reasoning exceeded what it replaced). Compare like-for-like + # (both rough estimates of the same message shape) so an + # "actual vs estimate" measurement mismatch cannot produce a + # false verdict. The gateway has a rotation-path-only guard + # (#83339), but in-place compaction commits inside this method + # via archive_and_compact — before the gateway can inspect the + # result — so the guard must live here to protect both paths. + # On growth, treat the attempt as a no-op: the original + # transcript stays untouched and durable. + _rough_in = estimate_messages_tokens_rough(messages) + _rough_out = estimate_messages_tokens_rough(compressed) + if _rough_out > _rough_in: + logger.warning( + "Compression refused: compressed transcript would be " + "larger than the original (session=%s, ~%s -> ~%s " + "tokens); keeping the original transcript unchanged", + agent.session_id or "none", + f"{_rough_in:,}", + f"{_rough_out:,}", + ) + try: + agent._emit_warning( + "⚠️ Compression refused: the generated summary " + "would have GROWN the conversation instead of " + "shrinking it. No messages were dropped — " + "conversation continues unchanged." + ) + except Exception: + pass + _existing_sp = getattr(agent, "_cached_system_prompt", None) + if not _existing_sp: + _existing_sp = agent._build_system_prompt(system_message) + _emit_compression_attempt_telemetry( + agent, + started_at=_attempt_started_at, + commit_status="aborted", + split_status="aborted", + failure_class="would_grow", + ) + _release_lock() + return messages, _existing_sp + if in_place: # ── In-place compaction: keep the same session_id ────────── # No end_session, no new row, no parent_session_id, no title @@ -3404,6 +3469,14 @@ def compress_context( migrate_heartbeat_to_session(old_session_id, agent.session_id) except Exception as _hb_err: logger.debug("Could not migrate heartbeat on compression: %s", _hb_err) + # Same boundary hazard for a persistent /loop — carry it + # onto the continuation session so the recurring wakeups + # survive compression. + try: + from hermes_cli.loops import migrate_loop_to_session + migrate_loop_to_session(old_session_id, agent.session_id, reason="compression") + except Exception as _loop_err: + logger.debug("Could not migrate loop on compression: %s", _loop_err) # Carry the title across the compression boundary unchanged. # # This used to renumber ("Fix X" → "Fix X #2") on every diff --git a/agent/conversation_loop.py b/agent/conversation_loop.py index acffc45d95..ba11804a97 100644 --- a/agent/conversation_loop.py +++ b/agent/conversation_loop.py @@ -38,6 +38,7 @@ from agent.conversation_compression import ( from agent.context_engine import automatic_compaction_status_message from agent.display import KawaiiSpinner from agent.error_classifier import FailoverReason, classify_api_error +from agent.message_metadata import append_message from agent.turn_context import ( _compression_warrants_another_preflight_pass, build_turn_context, @@ -132,7 +133,7 @@ def _restore_user_after_reference_handoff( and messages[-1].get("content") == content ): return False - messages.append({"role": "user", "content": content}) + append_message(messages, {"role": "user", "content": content}) return True @@ -168,6 +169,29 @@ _HANDOFF_SKIP_FINAL_RESPONSE = ( # to treat it as cancellation metadata rather than assistant prose. INTERRUPT_WAITING_FOR_MODEL_PREFIX = "Operation interrupted: waiting for model response (" + +def _should_rearm_compression_budget( + compression_attempts: int, + *, + completed_compaction_pending: bool, + prompt_tokens: int, + threshold_tokens: int, +) -> bool: + """Return True after a provider proves a completed compaction worked. + + Rough estimates cannot safely rearm the anti-thrash budget: they can dip + below the threshold while the provider-visible prompt remains too large. + Require the completed-compaction latch plus a positive, normalized prompt + count below the threshold from the next successful provider response. + """ + return bool( + compression_attempts + and completed_compaction_pending + and threshold_tokens > 0 + and 0 < prompt_tokens < threshold_tokens + ) + + # Modules that indicate a deterministic local processing error when they # appear in an exception traceback WITHOUT any API-call module. Used by the # outer-loop error classifier to avoid retrying bugs that will fail @@ -188,6 +212,20 @@ _API_CALL_MODULES = frozenset({ }) +def _moa_client_consumes_prepared_request(client: Any) -> bool: + """True when ``client`` is the in-process MoA facade. + + ``_moa_prepared_request`` is a private handshake with + ``MoAChatCompletions.create``, and only that facade exposes ``prepare()``. + Every other chat-completions object raises TypeError on the unexpected + keyword — including the native OpenAI client that credential rotation, + provider fallback and dead-connection cleanup rebuild from + ``_client_kwargs`` while ``agent.provider`` stays ``"moa"``. + """ + completions = getattr(getattr(client, "chat", None), "completions", None) + return callable(getattr(completions, "prepare", None)) + + def _join_truncated_parts(parts: List[str]) -> str: """Join continuation fragments, adding a newline where two would glue together (#78577).""" joined = "" @@ -276,8 +314,9 @@ def _apply_active_turn_redirect(agent: Any, messages: List[Dict[str, Any]], text if messages and messages[-1].get("role") == "assistant": # Transcript shows the user's own words; the provider replays the # scaffolded form so it still sees the interrupted context. - messages.append( - {"role": "user", "content": text, "api_content": correction} + append_message( + messages, + {"role": "user", "content": text, "api_content": correction}, ) else: # Placeholder preserves role alternation only. Scaffold bytes must @@ -290,9 +329,10 @@ def _apply_active_turn_redirect(agent: Any, messages: List[Dict[str, Any]], text } if not visible: placeholder["display_kind"] = "hidden" - messages.append(placeholder) - messages.append( - {"role": "user", "content": text, "api_content": correction} + append_message(messages, placeholder) + append_message( + messages, + {"role": "user", "content": text, "api_content": correction}, ) agent._current_streamed_assistant_text = "" @@ -489,6 +529,7 @@ def _billing_or_entitlement_message( provider: str, base_url: str, model: str, + unverified: bool = False, ) -> str: if _is_nous_inference_route(provider, base_url): return _nous_entitlement_message(capability) @@ -503,16 +544,45 @@ def _billing_or_entitlement_message( # apply to a subscription — the user waits for the reset or switches to an # API key. if (provider or "").strip().lower() == "anthropic": - lines = [ - ( - f"{provider_label} reported that your Claude subscription usage is " - f"exhausted for {model_label} (included quota + extra-usage credits)." - ), - "Options: wait for the billing cycle to reset, or add extra usage at " - "https://claude.ai/settings/usage", - "You can also switch to an Anthropic API key or another provider with " - "/model --provider .", - ] + # ``unverified`` (ClassifiedError.billing_unverified, #82154): the + # "out of extra usage" 400 is ambiguous — Anthropic returns the same + # body when its server-side content filter rejects part of the request + # on a subscription OAuth token, so the message reliably misdirects + # diagnosis toward buying quota. Hedge the claim and name the other + # cause. A confirmed verdict (e.g. a real 402 or an API-key credit + # depletion) keeps the assertive wording. + if unverified: + lines = [ + ( + f"{provider_label} reported that your Claude subscription usage may be " + f"exhausted for {model_label} (included quota + extra-usage credits) — " + "but this specific error is not proof of a billing problem." + ), + "If https://claude.ai/settings/usage still shows quota remaining, this is " + "probably NOT a billing problem: on a Claude subscription (OAuth) token " + "Anthropic returns this same message when its content filter rejects part " + "of the request — typically a phrase in the system prompt.", + "If usage really is exhausted: wait for the billing cycle to reset, or add " + "extra usage at https://claude.ai/settings/usage", + "You can also switch to an Anthropic API key or another provider with " + "/model --provider .", + # The exhaustion latch replays the stored error without issuing + # a request, so a real fix looks like it didn't work. + "Retry with a fresh credential state: `hermes auth reset anthropic`. Until " + "that cooldown clears, this error can be replayed from cache without " + "contacting the API.", + ] + else: + lines = [ + ( + f"{provider_label} reported that your Claude subscription usage is " + f"exhausted for {model_label} (included quota + extra-usage credits)." + ), + "Options: wait for the billing cycle to reset, or add extra usage at " + "https://claude.ai/settings/usage", + "You can also switch to an Anthropic API key or another provider with " + "/model --provider .", + ] return "\n".join(lines) # Provider-agnostic billing URL derivation (OpenAI, DeepSeek, xAI, Groq, @@ -541,16 +611,84 @@ def _billing_or_entitlement_message( return "\n".join(lines) -def _billing_block_dict(provider, base_url, model, message="") -> Optional[dict]: +def _billing_block_dict( + provider, base_url, model, message="", *, unverified: bool = False +) -> Optional[dict]: """Best-effort structured billing descriptor (None if billing_links is unavailable).""" try: from agent.billing_links import build_billing_block - return build_billing_block( + block = build_billing_block( provider=provider, base_url=str(base_url), model=model, message=message ).to_dict() except Exception: return None + if block is not None and unverified: + # Carry the classifier's ambiguity into the structured descriptor so + # every surface rendering the block can hedge too (#82154). + block["unverified"] = True + return block + + +def _billing_terminal_label(summary: str, unverified: bool) -> str: + """Terminal-failure prefix for a billing-classified error. + + ``unverified`` (#82154): the Anthropic "out of extra usage" 400 can be a + content-filter rejection, so the terminal line must not assert billing + exhaustion as fact. + """ + if unverified: + return ( + "Provider reported usage/credit exhaustion (unverified — the same " + f"error can be a content-filter rejection, not billing): {summary}" + ) + return f"Billing or credits exhausted: {summary}" + + +def _billing_failure_result( + *, + classified, + summary: str, + messages, + api_call_count: int, + provider: str, + base_url, + model: str, + guidance: Optional[str] = None, +) -> dict: + """Structured terminal result for a billing-classified failure. + + Single construction point for the returned terminal response so the + label, guidance, structured block, and ambiguity flag stay consistent + across the non-retryable abort and max-retries paths (#82154). + """ + unverified = bool(getattr(classified, "billing_unverified", False)) + if guidance is None: + guidance = _billing_or_entitlement_message( + capability="model access", + provider=provider, + base_url=str(base_url), + model=model, + unverified=unverified, + ) + final = _billing_terminal_label(summary, unverified) + if guidance: + final += f"\n\n{guidance}" + return { + "final_response": final, + "messages": messages, + "api_calls": api_call_count, + "completed": False, + "failed": True, + "error": summary, + "failure_reason": classified.reason.value, + # The billing verdict may rest on an ambiguous body (#82154) — carry + # that through the structured result, not just the prose. + "billing_unverified": unverified, + "billing_block": _billing_block_dict( + provider, base_url, model, guidance, unverified=unverified + ), + } def _print_billing_or_entitlement_guidance( @@ -560,12 +698,14 @@ def _print_billing_or_entitlement_guidance( provider: str, base_url: str, model: str, + unverified: bool = False, ) -> bool: message = _billing_or_entitlement_message( capability=capability, provider=provider, base_url=base_url, model=model, + unverified=unverified, ) if not message: return False @@ -1615,6 +1755,9 @@ def run_conversation( # Reset alongside the failure flag so a lock-contention diagnosis from a # previous turn can never leak into this turn's user-facing explanation. agent._last_persistence_error_cause = None + # Per-turn diagnostic: a failed compression-tip adoption in a previous + # turn's flush must not be reported against this turn. + agent._compression_adoption_failed = False # Main conversation loop counters (pure locals consumed by the loop below). api_call_count = 0 @@ -1628,7 +1771,10 @@ def run_conversation( compression_attempts = 0 # One resolved per-turn compression attempt cap, shared by every site that # consumes ``compression_attempts``: the pre-API pressure gate, the - # overflow/413 retry handlers, and the post-tool compaction gate. + # overflow/413 retry handlers, and the post-tool compaction gate. The + # counter is a consecutive unverified/ineffective-attempt backstop: a + # completed compaction rearms it only after a successful provider response + # reports a prompt below the threshold. # Config-driven via compression.max_attempts (parsed + validated in # agent_init); default 3 preserves the prior hardcoded behavior for # objects without the attribute (older pickles / minimal stubs). @@ -2230,7 +2376,7 @@ def run_conversation( final_response = _runtime_context_error failed = True _turn_exit_reason = "ollama_runtime_context_too_small" - messages.append({"role": "assistant", "content": final_response}) + append_message(messages, {"role": "assistant", "content": final_response}) agent._emit_status("❌ Ollama runtime context is too small for Hermes tool use") api_call_count -= 1 agent._api_call_count = api_call_count @@ -2684,7 +2830,24 @@ def run_conversation( # only after middleware, hooks, and debug dumps so none of them # attempts to serialize it as part of the provider payload. if _moa_prepared_request is not None and agent.provider == "moa": - api_kwargs["_moa_prepared_request"] = _moa_prepared_request + # Re-read the live client instead of trusting the one that + # prepared the request above. Credential rotation, provider + # fallback and dead-connection cleanup all rebuild + # agent.client from _client_kwargs between attempts, and + # pending_moa_prepared_request carries a prepared request + # across exactly that boundary. The rebuilt client is a + # native OpenAI client while provider stays "moa", so this + # private key would reach the SDK as an unexpected keyword + # — a non-retryable TypeError that kills every remaining + # turn on the session. + if _moa_client_consumes_prepared_request(agent.client): + api_kwargs["_moa_prepared_request"] = _moa_prepared_request + else: + logger.warning( + "MoA client replaced mid-turn (client=%s); sending the " + "prepared prompt without the MoA handshake", + type(agent.client).__name__, + ) # Always prefer the streaming path — even without stream # consumers. Streaming gives us fine-grained health @@ -3434,7 +3597,7 @@ def run_conversation( interim_msg = agent._build_assistant_message(assistant_message, finish_reason) # Marked so the ceiling exit can drop the fragment trail. interim_msg["_length_continuation_fragment"] = True - messages.append(interim_msg) + append_message(messages, interim_msg) if assistant_message.content: truncated_response_parts.append(assistant_message.content) @@ -3474,7 +3637,7 @@ def run_conversation( "content": _continue_content, "_length_continuation_nudge": True, } - messages.append(continue_msg) + append_message(messages, continue_msg) agent._session_messages = messages _retry.restart_with_length_continuation = True break @@ -3505,7 +3668,7 @@ def run_conversation( ) ] if partial_response: - messages.append({ + append_message(messages, { "role": "assistant", "content": partial_response, "finish_reason": "length", @@ -3684,7 +3847,50 @@ def run_conversation( "cache_write_tokens": canonical_usage.cache_write_tokens, "reasoning_tokens": canonical_usage.reasoning_tokens, } + # Capture the boundary latch before update_from_response() + # consumes it. Only a real provider prompt count for the + # request immediately following a completed compaction can + # prove that attempt effective and rearm the shared budget. + _completed_compaction_pending = bool( + getattr( + agent.context_compressor, + "_verify_compaction_cleared_threshold", + False, + ) + ) agent.context_compressor.update_from_response(usage_dict) + _compression_threshold = int( + getattr(agent.context_compressor, "threshold_tokens", 0) + or 0 + ) + if _should_rearm_compression_budget( + compression_attempts, + completed_compaction_pending=_completed_compaction_pending, + prompt_tokens=prompt_tokens, + threshold_tokens=_compression_threshold, + ): + logger.info( + "Compression budget rearmed after provider-confirmed " + "recovery: prompt=%s < threshold=%s (attempts were %s/%s)", + f"{prompt_tokens:,}", + f"{_compression_threshold:,}", + compression_attempts, + max_compression_attempts, + ) + compression_attempts = 0 + # Provider-confirmed recovery also invalidates the + # insufficient-progress preflight state: with the + # prompt proven back below the threshold, a prior + # "insufficient progress" verdict (and the stale + # pressure reading it would be compared against) + # describes a request shape that no longer exists. + # Left armed, _preflight_compression_blocked keeps the + # pre-API gate dark for the rest of the turn even + # though the attempt budget was just rearmed, so a + # later pressure spike would grow unchecked until the + # provider's overflow handler fired. + _preflight_compression_blocked = False + _last_preflight_pressure = None # Stash this response's canonical usage so the post-turn # on_turn_complete() observation hook can forward it (the @@ -3909,7 +4115,7 @@ def run_conversation( getattr(agent, "_current_streamed_assistant_text", "") or "" ).strip() if _partial: - messages.append({"role": "assistant", "content": _partial}) + append_message(messages, {"role": "assistant", "content": _partial}) final_response = _partial else: final_response = f"{INTERRUPT_WAITING_FOR_MODEL_PREFIX}{api_elapsed:.1f}s elapsed)." @@ -4265,6 +4471,7 @@ def run_conversation( has_retried_429=_retry.has_retried_429, classified_reason=classified.reason, error_context=error_context, + billing_unverified=classified.billing_unverified, ) if recovered_with_pool: continue @@ -4909,9 +5116,17 @@ def run_conversation( "switching to fallback model..." ) elif classified.reason == FailoverReason.billing: - agent._buffer_status( - "⚠️ Billing or credits exhausted — switching to fallback provider..." - ) + if classified.billing_unverified: + # Ambiguous body (#82154) — don't assert billing. + agent._buffer_status( + "⚠️ Provider reported usage/credit exhaustion " + "(unverified — may be a content-filter rejection) " + "— switching to fallback provider..." + ) + else: + agent._buffer_status( + "⚠️ Billing or credits exhausted — switching to fallback provider..." + ) elif _is_transport_failure: agent._buffer_status( "⚠️ Provider unreachable — switching to fallback provider..." @@ -5043,7 +5258,7 @@ def run_conversation( if ( status_code == 413 and isinstance(agent.base_url, str) - and "models.inference.ai.azure.com" in agent.base_url + and base_url_host_matches(agent.base_url, "models.inference.ai.azure.com") ): agent._vprint( f"{agent.log_prefix} 💡 GitHub Models free tier (models.inference.ai.azure.com) caps every", @@ -5608,6 +5823,7 @@ def run_conversation( provider=_provider, base_url=str(_base), model=_model, + unverified=classified.billing_unverified, ): pass elif _provider == "nous" and _print_nous_entitlement_guidance( @@ -5734,26 +5950,15 @@ def run_conversation( # the max-retries path so every surface (CLI, TUI, desktop) # renders one consistent billing signal. if classified.reason == FailoverReason.billing: - _ce_guidance = _billing_or_entitlement_message( - capability="model access", + return _billing_failure_result( + classified=classified, + summary=_nonretryable_summary, + messages=messages, + api_call_count=api_call_count, provider=_provider, - base_url=str(_base), + base_url=_base, model=_model, ) - _ce_final = f"Billing or credits exhausted: {_nonretryable_summary}" - if _ce_guidance: - _ce_final += f"\n\n{_ce_guidance}" - _ce_block = _billing_block_dict(_provider, _base, _model, _ce_guidance) - return { - "final_response": _ce_final, - "messages": messages, - "api_calls": api_call_count, - "completed": False, - "failed": True, - "error": _nonretryable_summary, - "failure_reason": classified.reason.value, - "billing_block": _ce_block, - } return { "final_response": _nonretryable_summary, "messages": messages, @@ -5797,12 +6002,20 @@ def run_conversation( _final_summary = agent._summarize_api_error(api_error) _billing_guidance = "" if classified.reason == FailoverReason.billing: - agent._emit_status(f"❌ Billing or credits exhausted — {_final_summary}") + if classified.billing_unverified: + # Ambiguous body (#82154) — hedge the terminal line. + agent._emit_status( + "❌ Provider reported usage/credit exhaustion " + f"(unverified — may be a content-filter rejection) — {_final_summary}" + ) + else: + agent._emit_status(f"❌ Billing or credits exhausted — {_final_summary}") _billing_guidance = _billing_or_entitlement_message( capability="model access", provider=_provider, base_url=str(_base), model=_model, + unverified=classified.billing_unverified, ) _print_billing_or_entitlement_guidance( agent, @@ -5810,6 +6023,7 @@ def run_conversation( provider=_provider, base_url=str(_base), model=_model, + unverified=classified.billing_unverified, ) elif is_rate_limited: agent._emit_status(f"❌ Rate limited after {max_retries} retries — {_final_summary}") @@ -5916,13 +6130,20 @@ def run_conversation( ) agent._persist_session(messages, conversation_history) _billing_block = None + _billing_unverified = False if classified.reason == FailoverReason.billing: - _final_response = f"Billing or credits exhausted: {_final_summary}" + _billing_unverified = classified.billing_unverified + _final_response = _billing_terminal_label( + _final_summary, _billing_unverified + ) if _billing_guidance: _final_response += f"\n\n{_billing_guidance}" # Structured recovery descriptor so every surface renders # the same link + label from one signal (see helper). - _billing_block = _billing_block_dict(_provider, _base, _model, _billing_guidance) + _billing_block = _billing_block_dict( + _provider, _base, _model, _billing_guidance, + unverified=_billing_unverified, + ) else: _final_response = f"API call failed after {max_retries} retries: {_final_summary}" if _is_thinking_timeout: @@ -5962,6 +6183,9 @@ def run_conversation( # different exit code. ``rate_limit`` / ``billing`` here # mean "quota wall, not a task error". "failure_reason": classified.reason.value, + # True when the billing verdict rests on an ambiguous + # body (#82154) — may be a content-filter rejection. + "billing_unverified": _billing_unverified, # Present only for billing walls: structured recovery # descriptor (provider, billing_url, is_nous, message). "billing_block": _billing_block, @@ -6355,7 +6579,7 @@ def run_conversation( else: last_msg[_key] = interim_msg[_key] else: - messages.append(interim_msg) + append_message(messages, interim_msg) agent._emit_interim_assistant_message(interim_msg) if agent._codex_incomplete_retries < 3: @@ -6393,7 +6617,7 @@ def run_conversation( and _last_msg.get("role") == "assistant" ) if not _already_nudged and _last_is_assistant: - messages.append({ + append_message(messages, { "role": "user", "content": _CODEX_INCOMPLETE_NUDGE, }) @@ -6513,7 +6737,7 @@ def run_conversation( } assistant_msg = agent._build_assistant_message(assistant_message, finish_reason) - messages.append(assistant_msg) + append_message(messages, assistant_msg) for tc in assistant_message.tool_calls: _tc_name = tc.function.name if _tc_name not in agent.valid_tool_names: @@ -6524,7 +6748,7 @@ def run_conversation( ) else: content = "Skipped: another tool call in this turn used an invalid name. Please retry this tool call." - messages.append({ + append_message(messages, { "role": "tool", "name": tc.function.name, "tool_call_id": tc.id, @@ -6614,7 +6838,7 @@ def run_conversation( # Append the assistant message with its (broken) tool_calls recovery_assistant = agent._build_assistant_message(assistant_message, finish_reason) - messages.append(recovery_assistant) + append_message(messages, recovery_assistant) # Respond with tool error results for each tool call invalid_names = {name for name, _ in invalid_json_args} @@ -6628,7 +6852,7 @@ def run_conversation( ) else: tool_result = "Skipped: other tool call in this response had invalid JSON." - messages.append({ + append_message(messages, { "role": "tool", "name": tc.function.name, "tool_call_id": tc.id, @@ -6767,7 +6991,7 @@ def run_conversation( and previous_msg.get("finish_reason") == "incomplete" and previous_interim_visible == current_interim_visible ) - messages.append(assistant_msg) + append_message(messages, assistant_msg) # Mixed batch: error-result the invalid calls and strip them # from the execution set. The assistant message above keeps @@ -6776,7 +7000,7 @@ def run_conversation( # provider-side tool_call/result pairing stays intact. if _invalid_batch_calls: for tc in _invalid_batch_calls: - messages.append({ + append_message(messages, { "role": "tool", "name": tc.function.name, "tool_call_id": tc.id, @@ -6861,7 +7085,7 @@ def run_conversation( agent._emit_status( f"⚠️ Tool guardrail halted {decision.tool_name}: {decision.code}" ) - messages.append({"role": "assistant", "content": final_response}) + append_message(messages, {"role": "assistant", "content": final_response}) # Emit the halt message to the client so it's not # indistinguishable from a crash. The stream display # was flushed (callback(None)) before tool execution, @@ -7188,8 +7412,8 @@ def run_conversation( _nudge_msg = agent._build_assistant_message(assistant_message, finish_reason) _nudge_msg["content"] = "(empty)" _nudge_msg["_empty_recovery_synthetic"] = True - messages.append(_nudge_msg) - messages.append({ + append_message(messages, _nudge_msg) + append_message(messages, { "role": "user", "content": _EMPTY_TOOL_RESPONSE_NUDGE, "_empty_recovery_synthetic": True, @@ -7226,7 +7450,7 @@ def run_conversation( assistant_message, "incomplete" ) interim_msg["_thinking_prefill"] = True - messages.append(interim_msg) + append_message(messages, interim_msg) agent._session_messages = messages continue @@ -7349,7 +7573,7 @@ def run_conversation( # were a meaningful model response, which can keep long # tool-heavy sessions stuck in empty-response loops. assistant_msg["_empty_terminal_sentinel"] = True - messages.append(assistant_msg) + append_message(messages, assistant_msg) if reasoning_text: reasoning_preview = reasoning_text[:500] + "..." if len(reasoning_text) > 500 else reasoning_text @@ -7428,14 +7652,14 @@ def run_conversation( ): codex_ack_continuations += 1 interim_msg = agent._build_assistant_message(assistant_message, "incomplete") - messages.append(interim_msg) + append_message(messages, interim_msg) agent._emit_interim_assistant_message(interim_msg) continue_msg = { "role": "user", "content": _CODEX_ACK_CONTINUATION_NUDGE, } - messages.append(continue_msg) + append_message(messages, continue_msg) agent._session_messages = messages # An acknowledgment is explicitly non-final. Do not let its # text suppress iteration-limit summarization if this @@ -7498,8 +7722,8 @@ def run_conversation( # buried mid-list in live memory but is skipped by the # flush regardless of position. final_msg["_dropped_toolcall_nudge"] = True - messages.append(final_msg) - messages.append({ + append_message(messages, final_msg) + append_message(messages, { "role": "user", "content": _DROPPED_TOOLCALL_NUDGE_CONTENT, "_dropped_toolcall_nudge": True, @@ -7559,12 +7783,12 @@ def run_conversation( # Only the nudge is flagged synthetic so it gets stripped # from the durable transcript (#65919 §7). agent._emit_interim_assistant_message(final_msg) - messages.append(final_msg) + append_message(messages, final_msg) try: agent._flush_messages_to_session_db(messages, conversation_history) except Exception: logger.debug("verify-on-stop interim flush failed", exc_info=True) - messages.append({ + append_message(messages, { "role": "user", "content": _verify_nudge, "_verification_stop_synthetic": True, @@ -7631,12 +7855,12 @@ def run_conversation( # Only the nudge is flagged synthetic so it gets stripped # from the durable transcript (#65919 §7). agent._emit_interim_assistant_message(final_msg) - messages.append(final_msg) + append_message(messages, final_msg) try: agent._flush_messages_to_session_db(messages, conversation_history) except Exception: logger.debug("pre_verify interim flush failed", exc_info=True) - messages.append({ + append_message(messages, { "role": "user", "content": _verify_nudge2, "_pre_verify_synthetic": True, @@ -7674,8 +7898,8 @@ def run_conversation( ) final_msg["finish_reason"] = "kanban_terminal_required" final_msg["_kanban_stop_synthetic"] = True - messages.append(final_msg) - messages.append({ + append_message(messages, final_msg) + append_message(messages, { "role": "user", "content": _kanban_nudge, "_kanban_stop_synthetic": True, @@ -7701,7 +7925,7 @@ def run_conversation( final_response = None continue - messages.append(final_msg) + append_message(messages, final_msg) # Make the completed answer durable before leaving the loop — # a session torn down before finalize_turn's _persist_session # otherwise loses a reply the user already saw (#81641). Same @@ -7793,7 +8017,7 @@ def run_conversation( "tool_call_id": tc["id"], "content": f"Error executing tool: {error_msg}", } - messages.append(err_msg) + append_message(messages, err_msg) break # Non-tool errors don't need a synthetic message injected. @@ -7817,7 +8041,7 @@ def run_conversation( final_response = f"I apologize, but I encountered repeated errors: {error_msg}" # Append as assistant so the history stays valid for # session resume (avoids consecutive user messages). - messages.append({"role": "assistant", "content": final_response}) + append_message(messages, {"role": "assistant", "content": final_response}) break # Post-loop turn finalization extracted to agent/turn_finalizer.finalize_turn diff --git a/agent/credential_pool.py b/agent/credential_pool.py index 072140134e..84c5b6834b 100644 --- a/agent/credential_pool.py +++ b/agent/credential_pool.py @@ -136,6 +136,14 @@ EXHAUSTED_TTL_SOLE_CREDENTIAL_SECONDS = 60 # 1 minute # the classifier), so the value is duplicated here rather than referenced. FAILURE_REASON_BILLING = "billing" +# Billing verdict that rests on an ambiguous body (#82154): Anthropic's +# "out of extra usage" 400 is returned both for genuine overage depletion and +# for a server-side content-filter rejection of the request. The latter leaves +# the credential perfectly healthy, so an unverified billing exhaustion gets +# the short transient cooldown instead of the one-hour billing bench — a +# genuine depletion simply re-latches on the next attempt. +FAILURE_REASON_BILLING_UNVERIFIED = "billing_unverified" + # Throttle window for the "no available entries" INFO line. Credential # selection runs on a hot path (every model call, plus auxiliary tasks like # compression/moa/titles), so when a pool is empty or fully exhausted the @@ -332,6 +340,15 @@ def _exhausted_ttl( if error_code == 401: return EXHAUSTED_TTL_401_SECONDS base = EXHAUSTED_TTL_429_SECONDS if error_code == 429 else EXHAUSTED_TTL_DEFAULT_SECONDS + # Unverified billing (#82154): the same 400 body can be a content-filter + # rejection of the request itself, in which case the credential is healthy + # and an hour-long bench just blocks it (and, for a sole credential, + # replays the stored error for the full hour — making a real fix look like + # it did not work). Short cooldown regardless of pool size; a genuine + # depletion re-latches on the next attempt. A true 402 stays a full bench + # even if something mislabeled it unverified. + if failure_reason == FAILURE_REASON_BILLING_UNVERIFIED and error_code != 402: + return min(base, EXHAUSTED_TTL_SOLE_CREDENTIAL_SECONDS) # Sole credential: shorten only TRANSIENT throttles (429 rate-limit, 403 # edge-throttle, 5xx server, or unknown). Billing exhaustion — whether # classified as such or self-evident from a 402 — is a genuine depletion diff --git a/agent/deadline.py b/agent/deadline.py new file mode 100644 index 0000000000..5aa58e6c06 --- /dev/null +++ b/agent/deadline.py @@ -0,0 +1,544 @@ +"""Unified deadline layer — one bounded-execution primitive, one timeout resolver. + +Phase 1 of the architectural fix for the timeout/hang backlog +(https://github.com/NousResearch/hermes-agent/issues/85125). + +The tree currently carries at least six site-local deadline mechanisms, each +built for one incident, none shared (tool_executor batch deadline, telegram +``_await_with_thread_deadline``, gateway turn lease, reasoning stale floors, +``human_wait_ceiling``, per-MCP-handler timeouts). Every new stall report +grows that list by one. This module is the shared foundation the call sites +migrate onto in later phases: + +* :func:`resolve_timeout` — one config-first resolution path for timeout + values (``timeouts:`` section in config.yaml > legacy env var > default), + so new surfaces stop inventing ``HERMES_*_TIMEOUT`` env vars (".env is for + secrets only") and hardcoded literals stop ignoring user config + (#63302, #53161, #43272 class). + +* :func:`clamp_timeout` — platform-safe clamping. Large user-supplied + timeouts overflow ``time_t`` inside ``threading.Lock.acquire(timeout=...)`` + / ``Thread.join(timeout=...)`` on macOS and kill whole tool batches + (#83220). Clamping at the shared boundary fixes that class once, for + every consumer. + +* :func:`run_bounded_async` — a wall-clock deadline for awaitables that does + NOT depend on event-loop timers. ``asyncio.wait_for`` schedules its expiry + on the loop; when the loop thread itself is blocked in a synchronous call + (family A of the #84047 stall triage), every asyncio-based timeout in the + process is silently disabled. This helper drives the deadline from a + daemon ``threading.Timer`` (generalizing the proven telegram-adapter + primitive) and abandons cancellation-shielded tasks instead of waiting for + cancellation to complete. The telegram adapter's private copy + (``plugins/platforms/telegram/adapter.py:_await_with_thread_deadline``) + migrates onto this in Phase 2 of #85125 — do not let the two drift in the + meantime; fix bugs here first. + +* :func:`run_bounded_sync` — the same contract for synchronous callables + bounded from a synchronous context (daemon worker thread, abandoned on + expiry). + +* :func:`kill_process_tree` — portable whole-tree termination so + kill-on-timeout stops orphaning descendants (#71148, #59549, #84967, + #68139 class). Existing site-local tree-kills that migrate onto this in + Phase 4 of #85125: ``gateway/status.py`` (taskkill wrapper + psutil + snapshot/reap pair) and ``tools/code_execution_tool.py`` (psutil + recursive children kill). + +Design invariants: + +* Exceptions raised by the bounded operation propagate unchanged — callers + keep their existing error handling. Only the *timeout* outcome is + reified (as :class:`BoundedResult`), because that is the outcome the + call sites keep getting wrong. +* A timeout produced by this layer is OUR deadline, not the provider's. + Callers that feed errors into ``agent/error_classifier.py`` should + classify :class:`DeadlineExpired` distinctly from transport timeouts + (the #59549 / #80323 misattribution class). +* ``None`` timeout means unbounded, and non-positive resolved values are + normalized to ``None`` (matching the existing + ``HERMES_CONCURRENT_TOOL_TIMEOUT_S`` convention). +""" + +from __future__ import annotations + +import asyncio +import faulthandler +import logging +import os +import subprocess +import sys +import threading +import time +from dataclasses import dataclass +from typing import Any, Awaitable, Callable, Optional + +logger = logging.getLogger(__name__) + +__all__ = [ + "MAX_SAFE_TIMEOUT_S", + "BoundedResult", + "DeadlineExpired", + "clamp_timeout", + "resolve_timeout", + "run_bounded_async", + "run_bounded_sync", + "kill_process_tree", +] + +# Upper bound for any timeout handed to platform wait primitives. +# +# CPython converts ``threading.Lock.acquire(timeout=...)`` / +# ``Thread.join(timeout=...)`` deadlines to an absolute timestamp; very large +# relative timeouts overflow ``time_t`` on macOS and raise +# ``OverflowError: timestamp out of range for platform time_t`` (#83220). +# One year is semantically "unbounded" for every wait in this codebase while +# staying far below any platform conversion limit. +MAX_SAFE_TIMEOUT_S = 31_536_000.0 # 365 days + +# Grace period after a deadline fires before concluding the event loop thread +# is blocked in a synchronous call and dumping stacks (family A diagnostics). +_LOOP_BLOCKED_DUMP_GRACE_S = 5.0 + + +class DeadlineExpired(TimeoutError): + """A deadline enforced by this layer expired. + + Distinct from transport/provider timeout types on purpose: when this is + raised (or a :class:`BoundedResult` reports ``timed_out``), the timeout + was Hermes's own bound — error classification must not attribute it to + the provider (#59549 / #80323 misattribution class). + """ + + def __init__(self, label: str, timeout_s: float): + super().__init__(f"deadline expired after {timeout_s:.1f}s: {label}") + self.label = label + self.timeout_s = timeout_s + + +@dataclass(frozen=True, kw_only=True) +class BoundedResult: + """Outcome of a bounded operation. + + ``timed_out`` is the reified outcome; on completion ``value`` holds the + operation's return value. Operation exceptions are never captured here — + they propagate to the caller unchanged. + """ + + timed_out: bool + value: Any + elapsed_s: float + timeout_s: Optional[float] + label: str + + def raise_if_timed_out(self) -> Any: + """Return ``value``, raising :class:`DeadlineExpired` on timeout.""" + if self.timed_out: + raise DeadlineExpired(self.label, float(self.timeout_s or 0.0)) + return self.value + + +def clamp_timeout(timeout: Optional[float]) -> Optional[float]: + """Normalize a timeout value for platform wait primitives. + + * ``None`` stays ``None`` (unbounded). + * Non-positive values become ``None`` (unbounded) — matching the existing + ``HERMES_CONCURRENT_TOOL_TIMEOUT_S`` "0 disables the bound" convention. + * Values above :data:`MAX_SAFE_TIMEOUT_S` are capped so they can never + overflow ``time_t`` inside ``Lock.acquire`` / ``Thread.join`` on macOS + (#83220). + * Non-numeric values are treated as unset (``None``) with a warning + rather than crashing the call path they were meant to protect. + """ + if timeout is None: + return None + try: + value = float(timeout) + except (TypeError, ValueError): + logger.warning("clamp_timeout: non-numeric timeout %r; treating as unbounded", timeout) + return None + if value != value: # NaN + logger.warning("clamp_timeout: NaN timeout; treating as unbounded") + return None + if value <= 0: + return None + return min(value, MAX_SAFE_TIMEOUT_S) + + +# --------------------------------------------------------------------------- +# Timeout resolution: config.yaml ``timeouts:`` section > legacy env var > +# registered default. +# --------------------------------------------------------------------------- + +def _timeouts_section() -> dict: + """Read the ``timeouts:`` root section from config.yaml (read-only). + + Isolated for testability and so a broken config read can never take down + the call path the timeout was protecting. + """ + try: + from hermes_cli.config import load_config_readonly + + section = load_config_readonly().get("timeouts") + return section if isinstance(section, dict) else {} + except Exception: + logger.debug("timeouts: config read failed; using defaults", exc_info=True) + return {} + + +def _lookup_dotted(section: dict, key: str) -> Any: + """Walk ``a.b.c`` through nested dicts; return None when absent.""" + node: Any = section + for part in key.split("."): + if not isinstance(node, dict) or part not in node: + return None + node = node[part] + return node + + +def resolve_timeout( + key: str, + *, + default: Optional[float], + env_var: Optional[str] = None, +) -> Optional[float]: + """Resolve a timeout in seconds for a dotted config key. + + Precedence (established by the ``providers.*.request_timeout_seconds`` + pattern — config wins over the legacy env var): + + 1. ``timeouts.`` in config.yaml (dotted key walks nested maps, e.g. + ``tools.concurrent_batch`` reads ``timeouts: {tools: {concurrent_batch: ...}}``) + 2. ``env_var`` when set and non-empty (legacy bridge — internal mechanism + and back-compat only; new surfaces must not grow new user-facing + ``HERMES_*`` timeout env vars) + 3. ``default`` + + The winning value is passed through :func:`clamp_timeout`, so ``0`` or a + negative value means "unbounded" and oversized values are made + platform-safe. Invalid (non-numeric) config/env values fall through to + the next source with a warning instead of breaking the protected path. + """ + raw = _lookup_dotted(_timeouts_section(), key) + if raw is not None: + # Explicit float() (clamp_timeout would also convert) so that invalid + # config values FALL THROUGH to the env var / default instead of + # resolving as unbounded — do not "simplify" this away. bool is + # rejected because YAML `true` would silently become a 1-second + # deadline; NaN is rejected for the same fall-through reason. + if not isinstance(raw, bool): + try: + value = float(raw) + if value == value: # not NaN + return clamp_timeout(value) + except (TypeError, ValueError): + pass + logger.warning("timeouts.%s: invalid value %r in config.yaml; ignoring", key, raw) + + if env_var: + env_raw = os.getenv(env_var, "").strip() + if env_raw: + try: + return clamp_timeout(float(env_raw)) + except ValueError: + logger.warning("invalid %s=%r; ignoring", env_var, env_raw) + + return clamp_timeout(default) + + +# --------------------------------------------------------------------------- +# Bounded execution — async flavor. +# +# Generalizes plugins/platforms/telegram/adapter.py:_await_with_thread_deadline +# (the #63309 fix): the deadline is driven by a daemon threading.Timer so a +# blocked event loop cannot disable it, and a second timer dumps all thread +# stacks when the loop provably failed to process the expiry — the one piece +# of information loop-blocked hangs otherwise never surface. +# --------------------------------------------------------------------------- + +def _consume_abandoned(task: "asyncio.Future[Any]") -> None: + """Observe an abandoned task's outcome so it never logs 'never retrieved'.""" + try: + if not task.cancelled(): + task.exception() + except Exception: + pass + + +async def _run_abandon_cleanup(on_abandon: Callable[[], Awaitable[Any]]) -> None: + """Run abandonment cleanup fully fire-and-forget (its failures swallowed).""" + try: + await on_abandon() + except Exception: + logger.debug("deadline abandon-cleanup failed", exc_info=True) + + +def _dump_blocked_loop_diagnostics(label: str, timeout_s: float) -> None: + logger.warning( + "[deadline] %r deadline (%.0fs) expired but the event loop has not " + "processed the expiry after a further %.0fs — the loop thread appears " + "BLOCKED in a synchronous call, which is why no asyncio timeout can " + "fire. Dumping all thread stacks to stderr to identify the blocking " + "frame.", + label, + timeout_s, + _LOOP_BLOCKED_DUMP_GRACE_S, + ) + try: + faulthandler.dump_traceback(all_threads=True) + except Exception: + logger.debug("faulthandler traceback dump failed", exc_info=True) + + +async def run_bounded_async( + awaitable: Awaitable[Any], + timeout: Optional[float], + *, + label: str = "operation", + on_abandon: Optional[Callable[[], Awaitable[Any]]] = None, + dump_on_blocked_loop: bool = True, +) -> BoundedResult: + """Await ``awaitable`` under a wall-clock deadline independent of loop timers. + + On completion returns ``BoundedResult(timed_out=False, value=...)``; + exceptions from the operation (including ``asyncio.CancelledError`` from a + caller cancelling *us*) propagate unchanged. + + On timeout the underlying task is cancelled and **abandoned** — we do not + await cancellation completion, because cancellation-shielded scopes (anyio, + httpcore init, MCP SDK teardown) are exactly the paths that wedge forever. + ``on_abandon`` (zero-arg callable returning an awaitable) is scheduled as + detached best-effort cleanup for the half-built state the abandoned task + may leave behind. Returns ``BoundedResult(timed_out=True, value=None)``. + + ``timeout=None`` (or a non-positive resolved value) awaits unbounded. + """ + timeout_s = clamp_timeout(timeout) + start = time.monotonic() + if timeout_s is None: + value = await awaitable + return BoundedResult(timed_out=False, value=value, elapsed_s=time.monotonic() - start, timeout_s=None, label=label) + + task = asyncio.ensure_future(awaitable) + loop = asyncio.get_running_loop() + deadline: "asyncio.Future[None]" = loop.create_future() + loop_processed_expiry = threading.Event() + + def _mark_expired() -> None: + loop_processed_expiry.set() + if not deadline.done(): + deadline.set_result(None) + + def _expire_from_thread() -> None: + loop.call_soon_threadsafe(_mark_expired) + + def _watchdog_check() -> None: + if not loop_processed_expiry.is_set(): + _dump_blocked_loop_diagnostics(label, timeout_s) + + timer = threading.Timer(timeout_s, _expire_from_thread) + timer.daemon = True + timer.start() + watchdog: Optional[threading.Timer] = None + if dump_on_blocked_loop: + watchdog = threading.Timer( + timeout_s + _LOOP_BLOCKED_DUMP_GRACE_S, _watchdog_check + ) + watchdog.daemon = True + watchdog.start() + try: + try: + done, _ = await asyncio.wait( + {task, deadline}, return_when=asyncio.FIRST_COMPLETED + ) + except asyncio.CancelledError: + # The CALLER cancelled us. Without this, `task` would keep running + # unobserved (and later log "exception was never retrieved") — + # a leak the telegram original also had. Cancel + abandon it, then + # let the cancellation propagate. + task.cancel() + task.add_done_callback(_consume_abandoned) + raise + if task in done: + if not deadline.done(): + deadline.cancel() + value = await task + return BoundedResult(timed_out=False, value=value, elapsed_s=time.monotonic() - start, timeout_s=timeout_s, label=label) + + task.cancel() + task.add_done_callback(_consume_abandoned) + if on_abandon is not None: + cleanup = asyncio.ensure_future(_run_abandon_cleanup(on_abandon)) + cleanup.add_done_callback(_consume_abandoned) + logger.warning("[deadline] %r timed out after %.1fs; task abandoned", label, timeout_s) + return BoundedResult(timed_out=True, value=None, elapsed_s=time.monotonic() - start, timeout_s=timeout_s, label=label) + finally: + timer.cancel() + if watchdog is not None: + watchdog.cancel() + # cancel() cannot stop a Timer whose callback is already running; + # setting the event closes that race so a completed await can never + # be misreported as a blocked loop. + loop_processed_expiry.set() + + +# --------------------------------------------------------------------------- +# Bounded execution — sync flavor. +# --------------------------------------------------------------------------- + +def run_bounded_sync( + fn: Callable[[], Any], + timeout: Optional[float], + *, + label: str = "operation", + on_timeout: Optional[Callable[[], None]] = None, +) -> BoundedResult: + """Run ``fn`` in a daemon worker thread under a wall-clock deadline. + + On completion returns its value (exceptions re-raised in the caller). + On expiry the worker thread is **abandoned** (daemon, so it cannot block + interpreter exit), ``on_timeout`` (if given) runs best-effort in the + caller's thread — e.g. to mark a backend suspect or kill a subprocess — + and ``BoundedResult(timed_out=True)`` is returned. + + Intended for infrequent, seconds-scale blocking backend calls. Do NOT + use per-item in hot loops: each call spawns a thread, and every timeout + permanently leaks an abandoned daemon thread — a wedged backend called + in a retry loop would accumulate them. + + ``timeout=None`` (or non-positive) blocks until ``fn`` returns. + """ + timeout_s = clamp_timeout(timeout) + start = time.monotonic() + if timeout_s is None: + return BoundedResult(timed_out=False, value=fn(), elapsed_s=time.monotonic() - start, timeout_s=None, label=label) + + box: dict[str, Any] = {} + done = threading.Event() + + def _worker() -> None: + try: + box["value"] = fn() + except BaseException as exc: # re-raised in caller; must not vanish + box["exc"] = exc + finally: + done.set() + + thread = threading.Thread( + target=_worker, name=f"deadline-{label}", daemon=True + ) + thread.start() + if not done.wait(timeout_s): + logger.warning("[deadline] %r timed out after %.1fs; worker abandoned", label, timeout_s) + if on_timeout is not None: + try: + on_timeout() + except Exception: + logger.debug("deadline on_timeout callback failed", exc_info=True) + return BoundedResult(timed_out=True, value=None, elapsed_s=time.monotonic() - start, timeout_s=timeout_s, label=label) + + if "exc" in box: + raise box["exc"] + return BoundedResult(timed_out=False, value=box.get("value"), elapsed_s=time.monotonic() - start, timeout_s=timeout_s, label=label) + + +# --------------------------------------------------------------------------- +# Whole-tree process termination. +# --------------------------------------------------------------------------- + +def kill_process_tree(pid: int, *, sig: Optional[int] = None) -> bool: + """Terminate ``pid`` and all its descendants, portably. + + Kill-on-timeout that signals only the direct child orphans process trees + (cron scripts, in-container shells, browser daemons — #71148 class). + + * Windows: ``taskkill /F /T`` terminates the tree (``sig`` ignored; + Windows has no equivalent). Console-window flash is suppressed via + ``windows_hide_flags`` and the exit code is checked, so a dead or + inaccessible PID reports ``False`` like the POSIX path. + * POSIX: the descendant set is snapshotted via psutil (a hard + dependency) BEFORE any signal — once the parent dies its children are + reparented and can no longer be found by a parent walk. Then the + process group is signalled when ``pid`` leads one (covers + grandchildren in the same session in one syscall), and every + snapshotted descendant is signalled individually — which also reaches + descendants that created their OWN sessions (a child that called + ``setsid``, exactly what user shell commands do; see + tools/environments/base.py). ``sig`` defaults to ``SIGKILL``. + psutil's identity-aware ``Process`` (PID + create time) means a + recycled PID is never signalled. + + Returns True when the target (or any of its tree) was signalled, False + when the process was already gone or every termination call failed. + """ + if sys.platform == "win32": + try: + from hermes_cli._subprocess_compat import windows_hide_flags + + creationflags = windows_hide_flags() + except Exception: + creationflags = 0 + try: + proc = subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(pid)], + capture_output=True, + timeout=15, + check=False, + creationflags=creationflags, + ) + # taskkill exits non-zero for not-found / access-denied; keep the + # cross-platform contract (False = nothing was terminated). + return proc.returncode == 0 + except Exception: + logger.debug("kill_process_tree: taskkill failed for pid %s", pid, exc_info=True) + return False + + import signal as _signal + + if sig is None: + sig = _signal.SIGKILL + + # Snapshot descendants while the parent is still alive — after it dies + # they reparent to init/subreaper and a parent walk finds nothing. + descendants: list = [] + try: + import psutil + + descendants = psutil.Process(int(pid)).children(recursive=True) + except Exception: + # Already gone, or psutil unavailable in a stripped env — the + # group-signal below still covers same-session descendants. + descendants = [] + + signalled = False + try: + # NOTE: getpgid→killpg has an inherent TOCTOU (pid could be reaped and + # recycled between the calls). All existing killpg sites share it; the + # psutil sweep below is identity-aware and does not. + pgid = os.getpgid(pid) + except (ProcessLookupError, PermissionError, OSError): + pgid = None + try: + if pgid is not None and pgid == pid: + # pid leads its own group: one syscall covers the whole group. + # (The == check guards against signalling the caller's own group + # when pid is not a leader.) + os.killpg(pgid, sig) # windows-footgun: ok — POSIX-only branch (win32 returns above) + else: + os.kill(pid, sig) + signalled = True + except ProcessLookupError: + pass + except (PermissionError, OSError): + logger.debug("kill_process_tree: signal failed for pid %s", pid, exc_info=True) + + # Sweep the snapshot: reaches descendants outside the parent's group + # (their own setsid sessions) and the non-group-leader case. + for child in descendants: + try: + if child.is_running(): # identity-aware: recycled PIDs skipped + child.send_signal(sig) + signalled = True + except Exception: + continue + return signalled diff --git a/agent/error_classifier.py b/agent/error_classifier.py index d28ddb0602..8198888813 100644 --- a/agent/error_classifier.py +++ b/agent/error_classifier.py @@ -18,6 +18,12 @@ from typing import Any, Dict, Optional logger = logging.getLogger(__name__) +# Synthetic error code used when the OpenAI SDK rejects a provider's SSE +# ``data:`` field before Hermes receives a completion chunk. Keeping this +# distinct from generic JSON parse failures lets the classifier make narrow, +# provider-stream-specific recovery decisions without inventing an HTTP status. +PROVIDER_STREAM_NON_JSON_ERROR_CODE = "provider_stream_non_json_data" + # ── Error taxonomy ────────────────────────────────────────────────────── @@ -96,6 +102,15 @@ class ClassifiedError: def is_auth(self) -> bool: return self.reason in {FailoverReason.auth, FailoverReason.auth_permanent} + @property + def billing_unverified(self) -> bool: + """True when a ``billing`` verdict rests on an ambiguous body. + + Anthropic's "out of extra usage" 400 can also be a content-filter + rejection (#82154); surfaces must hedge rather than assert exhaustion. + """ + return bool(self.error_context.get("billing_unverified")) + # ── Provider-specific patterns ────────────────────────────────────────── @@ -108,6 +123,8 @@ _BILLING_PATTERNS = [ "credit balance", "credits exhausted", "credits have been exhausted", + "requires available credits", + "account balance is too low", "no usable credits", "top up your credits", "payment required", @@ -123,6 +140,25 @@ _BILLING_PATTERNS = [ "not available on the free tier", ] +# Billing-pattern matches that are NOT proof of billing exhaustion. Anthropic +# returns the identical "out of extra usage" body on a subscription OAuth +# token both when the overage bucket is genuinely depleted AND when its +# server-side content filter rejects part of the request (#82154) — the two +# are indistinguishable from the response. Classification stays ``billing`` +# (rotation + fallback remain the right recovery either way), but the +# ambiguity is carried in ``error_context`` so downstream surfaces hedge +# instead of asserting exhaustion as fact, and the credential pool applies a +# short cooldown instead of the one-hour billing bench (a content-filter +# rejection leaves the credential perfectly healthy). +_UNVERIFIED_BILLING_PATTERNS = ("out of extra usage",) + + +def _billing_ambiguity_context(error_msg: str) -> Dict[str, Any]: + """error_context marking a billing verdict as unverified (see above).""" + if any(p in error_msg for p in _UNVERIFIED_BILLING_PATTERNS): + return {"billing_unverified": True, "possible_content_filter": True} + return {} + # xAI's explicit Grok credit-exhaustion code. Keep the HTTP 403 special case # provider-scoped: other providers' generic billing codes historically remain # auth failures when they arrive as 403. @@ -1503,6 +1539,10 @@ def _classify_400( retryable=False, should_rotate_credential=True, should_fallback=True, + # "out of extra usage" on a 400 is ambiguous — it can also be a + # content-filter rejection (#82154). Mark the verdict unverified + # so downstream hedges and the pool skips the 1-hour bench. + error_context=_billing_ambiguity_context(error_msg), ) # Generic 400 + large session → probable context overflow @@ -1558,6 +1598,20 @@ def _classify_by_error_code( """Classify by structured error codes from the response body.""" code_lower = error_code.lower() + if ( + code_lower == PROVIDER_STREAM_NON_JSON_ERROR_CODE + and "request validation failed:" in error_msg + ): + # Some OpenAI-compatible endpoints encode deterministic request + # validation failures as plain-text ``event: error`` SSE data behind + # HTTP 200. Retrying the unchanged request cannot succeed, but a + # configured provider fallback still may. + return result_fn( + FailoverReason.format_error, + retryable=False, + should_fallback=True, + ) + if code_lower in {"resource_exhausted", "throttled", "rate_limit_exceeded"}: return result_fn( FailoverReason.rate_limit, @@ -1670,6 +1724,10 @@ def _classify_by_message( retryable=False, should_rotate_credential=True, should_fallback=True, + # Status-less path: adapters can strip the HTTP status from the + # Anthropic "out of extra usage" 400, so the same ambiguity + # marking applies here (#82154). + error_context=_billing_ambiguity_context(error_msg), ) # Rate limit patterns diff --git a/agent/image_routing.py b/agent/image_routing.py index a8337c9ac6..a3762fccd4 100644 --- a/agent/image_routing.py +++ b/agent/image_routing.py @@ -432,7 +432,14 @@ def _lookup_supports_vision( caps = None try: from agent.models_dev import get_model_capabilities - caps = get_model_capabilities(provider, model) + # allow_network=True on purpose: vision-capability lookup runs when + # an image actually needs routing (not per turn), and the #31179 + # text-only-main guard depends on catalog data — a cold cache + # returning "unknown" would fall back to attempting the call and + # reintroduce the bug. This preserves the historical + # network-on-cold-cache behavior for this one path; the fetch is + # cached (4h TTL) and backoff-limited after failures. + caps = get_model_capabilities(provider, model, allow_network=True) except Exception as exc: # pragma: no cover - defensive logger.debug("image_routing: caps lookup failed for %s:%s — %s", provider, model, exc) if caps is not None: diff --git a/agent/insights.py b/agent/insights.py index 34e78a6ff9..c1dec9e073 100644 --- a/agent/insights.py +++ b/agent/insights.py @@ -21,16 +21,28 @@ import sqlite3 import time from collections import Counter, defaultdict from datetime import datetime +from decimal import Decimal from typing import Any, Dict, List, Optional from agent.usage_pricing import ( CanonicalUsage, estimate_usage_cost, + format_cost_label, format_duration_compact, has_known_pricing, ) +def _fmt_est_cost(est_cost: float) -> str: + """Format an aggregate estimated cost via the shared cost-label helper. + + Routes through ``format_cost_label`` so sub-cent aggregates render at + 4dp instead of collapsing to "~$0.00" (#79220 bug class — the same + dishonesty this module's cost buckets exist to fix, #77223). + """ + return format_cost_label(Decimal(str(est_cost))) + + def _estimate_cost( @@ -1000,6 +1012,29 @@ class InsightsEngine: lines.append(f" Avg msgs/session: {o['avg_messages_per_session']:.1f}") lines.append("") + # Cost breakdown — surface the three buckets so subscription-included + # and unknown-cost sessions are visible instead of silently collapsing + # to $0. See #77223. + est_cost = o.get("estimated_cost", 0.0) + included_sessions = o.get("included_cost_sessions", 0) + unknown_sessions = o.get("unknown_cost_sessions", 0) + if est_cost > 0 or included_sessions > 0 or unknown_sessions > 0: + lines.append(" 💰 Cost") + lines.append(" " + "─" * 56) + if est_cost > 0: + lines.append(f" Estimated: {_fmt_est_cost(est_cost)}") + if included_sessions > 0: + lines.append( + f" Included: {included_sessions} session(s) " + f"(subscription — no provider invoice)" + ) + if unknown_sessions > 0: + lines.append( + f" Unknown: {unknown_sessions} session(s) " + f"(no pricing data)" + ) + lines.append("") + # Model breakdown if report["models"]: lines.append(" 🤖 Models Used") @@ -1114,6 +1149,21 @@ class InsightsEngine: lines.append(f"**Active time:** ~{format_duration_compact(o['total_hours'] * 3600)} | **Avg session:** ~{format_duration_compact(o['avg_session_duration'])}") lines.append("") + # Cost breakdown — surface buckets so included/unknown are visible + est_cost = o.get("estimated_cost", 0.0) + included = o.get("included_cost_sessions", 0) + unknown = o.get("unknown_cost_sessions", 0) + cost_parts: list[str] = [] + if est_cost > 0: + cost_parts.append(f"{_fmt_est_cost(est_cost)} estimated") + if included > 0: + cost_parts.append(f"{included} included (subscription)") + if unknown > 0: + cost_parts.append(f"{unknown} unknown") + if cost_parts: + lines.append(f"**Cost:** {' | '.join(cost_parts)}") + lines.append("") + # Models (top 5) if report["models"]: lines.append("**🤖 Models:**") diff --git a/agent/message_metadata.py b/agent/message_metadata.py new file mode 100644 index 0000000000..284edc523d --- /dev/null +++ b/agent/message_metadata.py @@ -0,0 +1,41 @@ +"""Internal metadata attached to durable conversation messages.""" + +from __future__ import annotations + +from time import time as wall_time +from typing import Any, MutableMapping, Optional, TypeVar + + +# These fields describe Hermes' durable record, not provider-visible message +# content. They must not influence context-pressure decisions. +PERSISTENCE_ONLY_MESSAGE_FIELDS = frozenset({"timestamp"}) + +_Message = TypeVar("_Message", bound=MutableMapping[str, Any]) + + +def stamp_message_timestamp( + message: _Message, + *, + timestamp: Optional[float] = None, +) -> _Message: + """Attach a creation timestamp without replacing source-provided time. + + Gateway adapters can supply the platform event time. All other callers use + the local wall clock at the point the message enters the live transcript. + Returning the same mapping keeps the helper convenient at append sites. + """ + if message.get("timestamp") is None: + message["timestamp"] = wall_time() if timestamp is None else timestamp + return message + + +def append_message( + messages: list[Any], + message: _Message, + *, + timestamp: Optional[float] = None, +) -> _Message: + """Stamp and append one live transcript message.""" + stamp_message_timestamp(message, timestamp=timestamp) + messages.append(message) + return message diff --git a/agent/message_sanitization.py b/agent/message_sanitization.py index 0fa7d95fff..e68715cf39 100644 --- a/agent/message_sanitization.py +++ b/agent/message_sanitization.py @@ -318,7 +318,9 @@ def close_interrupted_tool_sequence(messages: list, final_response: Any = None) if not isinstance(last, dict) or last.get("role") != "tool": return False text = final_response if isinstance(final_response, str) else "" - messages.append({ + from agent.message_metadata import append_message + + append_message(messages, { "role": "assistant", "content": text.strip() or "Operation interrupted.", }) diff --git a/agent/model_metadata.py b/agent/model_metadata.py index 6006c28b13..c3a0fc4b41 100644 --- a/agent/model_metadata.py +++ b/agent/model_metadata.py @@ -24,6 +24,7 @@ if TYPE_CHECKING: # pragma: no cover — runtime import is lazy (see below) from utils import atomic_json_write, atomic_yaml_write, base_url_host_matches, base_url_hostname from hermes_constants import OPENROUTER_MODELS_URL +from agent.message_metadata import PERSISTENCE_ONLY_MESSAGE_FIELDS logger = logging.getLogger(__name__) @@ -48,18 +49,37 @@ def __getattr__(name: str): raise AttributeError(f"module {__name__!r} has no attribute {name!r}") -def _resolve_requests_verify() -> bool | str: - """Resolve SSL verify setting for `requests` calls from env vars. +def _resolve_requests_verify(base_url: str = "") -> bool | str: + """Resolve SSL verify setting for `requests` calls. - The `requests` library only honours REQUESTS_CA_BUNDLE / CURL_CA_BUNDLE - by default. Hermes also honours HERMES_CA_BUNDLE (its own convention) - and SSL_CERT_FILE (used by the stdlib `ssl` module and by httpx), so - that a single env var can cover both `requests` and `httpx` callsites - inside the same process. + Priority (mirrors ``agent.ssl_verify.resolve_httpx_verify`` so the + ``requests``-based ``/models`` probes agree with the httpx chat client): - Returns either a filesystem path to a CA bundle, or True to defer to - the requests default (certifi). + 1. Per-provider ``ssl_verify: false`` for ``base_url`` — disable verification. + 2. Per-provider ``ssl_ca_cert`` for ``base_url`` — an explicit CA bundle. + Without this, a custom endpoint whose chain only verifies against the + provider's configured bundle (not the process ``SSL_CERT_FILE``) logs a + spurious CERTIFICATE_VERIFY_FAILED on every probe even though the chat + path succeeds (per-provider ``ssl_ca_cert`` was reaching only httpx). + 3. Env vars ``HERMES_CA_BUNDLE`` / ``REQUESTS_CA_BUNDLE`` / ``SSL_CERT_FILE`` + (a single var covers both ``requests`` and ``httpx`` in-process). + 4. ``True`` — defer to the requests default (certifi). + + ``base_url`` is optional so existing callers (OpenRouter, etc.) keep the + env-only behavior unchanged; only probes that pass a base_url pick up the + per-provider override. """ + if base_url: + try: + from hermes_cli.config import get_custom_provider_tls_settings + tls = get_custom_provider_tls_settings(base_url) + if tls.get("ssl_verify") is False: + return False + ca = tls.get("ssl_ca_cert") + if isinstance(ca, str) and ca and os.path.isfile(ca): + return ca + except Exception: + pass # fall through to env vars — never break a probe on config lookup for env_var in ("HERMES_CA_BUNDLE", "REQUESTS_CA_BUNDLE", "SSL_CERT_FILE"): val = os.getenv(env_var) if val and os.path.isfile(val): @@ -1261,7 +1281,7 @@ def fetch_endpoint_model_metadata( server_url.rstrip("/") + "/api/v1/models", headers=headers, timeout=(5, 10), - verify=_resolve_requests_verify(), + verify=_resolve_requests_verify(normalized), ) response.raise_for_status() payload = response.json() @@ -1324,7 +1344,7 @@ def fetch_endpoint_model_metadata( url, headers=headers, timeout=(5, 10), - verify=_resolve_requests_verify(), + verify=_resolve_requests_verify(normalized), stream=True, ) if response.status_code in (401, 403): @@ -1364,7 +1384,7 @@ def fetch_endpoint_model_metadata( try: # Try /v1/props first (current llama.cpp); fall back to /props for older builds base = request_candidate.rstrip("/").replace("/v1", "") - _verify = _resolve_requests_verify() + _verify = _resolve_requests_verify(normalized) props_resp = requests.get(base + "/v1/props", headers=headers, timeout=5, verify=_verify) if not props_resp.ok: props_resp = requests.get(base + "/props", headers=headers, timeout=5, verify=_verify) @@ -2284,7 +2304,7 @@ def _query_anthropic_context_length(model: str, base_url: str, api_key: str) -> "anthropic-version": "2023-06-01", } _ensure_requests() - resp = requests.get(url, headers=headers, timeout=(5, 10), verify=_resolve_requests_verify()) + resp = requests.get(url, headers=headers, timeout=(5, 10), verify=_resolve_requests_verify(base_url)) if resp.status_code != 200: return None data = resp.json() @@ -2577,6 +2597,7 @@ def get_model_context_length( Resolution order: 0. Explicit config override (model.context_length or custom_providers per-model) + 0b. model_overrides config (per-provider+model context_window override) 0c. Endpoint-scoped metadata for models validated on one multiplexed endpoint 1. Persistent cache (previously discovered via probing). Nous URLs, LM Studio, and Codex OAuth bypass the cache here so their provider @@ -2638,7 +2659,23 @@ def get_model_context_length( logger.debug("MoA aggregator context-length resolution failed", exc_info=True) # Fall through to the generic default if aggregator resolution failed. - # 0b. custom_providers per-model override — check before any probe. + # 0b. model_overrides config — EXPLICIT per-provider+model context_window + # override only (fill-gap _default entries are applied later, inside + # lookup_models_dev_context at step 5f, once the catalog has actually + # missed — so a _default can never preempt custom_providers or live + # probes). This is the supported self-unblock path for models with + # wrong context in models.dev (#84482) and for custom/local models + # (#8731). Config-read only; never blocks on the network. + if provider and model: + try: + from agent.models_dev import _override_context_window + mo_ctx = _override_context_window(provider, model) + if mo_ctx is not None and mo_ctx > 0: + return mo_ctx + except Exception: + pass # fall through to other resolution paths + + # 0c. custom_providers per-model override — check before any probe. # This closes the gap where /model switch and display paths used to fall # back to 128K despite the user having a per-model context_length set. # See #15779. @@ -3314,7 +3351,7 @@ def _wire_message_shadow(msg: Dict[str, Any]) -> Dict[str, Any]: ) shadow: Dict[str, Any] = {} for k, v in msg.items(): - if k in ("_anthropic_content_blocks", "reasoning_details"): + if k in ("_anthropic_content_blocks", "reasoning_details") or k in PERSISTENCE_ONLY_MESSAGE_FIELDS: continue if k == "api_content": # Always popped before the request is built; only counted when it diff --git a/agent/models_dev.py b/agent/models_dev.py index 52c4100741..fca9201eb6 100644 --- a/agent/models_dev.py +++ b/agent/models_dev.py @@ -15,8 +15,25 @@ Data resolution order: served rather than blocking callers on the network) 3. Network fetch (https://models.dev/api.json) — only when no cache exists at all; failed refreshes back off for 5 minutes process-wide -Latency-sensitive callers (gateway route-identity checks) pass -``allow_network=False`` and never touch the network. + +Network hardening: + +- **ETag conditional GET**: network refreshes send ``If-None-Match`` + with the last-known ETag whenever a servable registry is held (memory, + hydrated from disk on cold force-refresh). A 304 Not Modified response + is a no-op — the existing cache is re-confirmed fresh without + re-downloading the full registry (≈2 MB). The ETag is persisted + atomically alongside the cache file. +- **No-network-on-hot-paths invariant**: resolution, picker, and resume + paths NEVER perform network I/O. ``allow_network=False`` is threaded + through every query function, and hot-path callers (vision routing, + image routing, cost guard, context-length lookup) pass it explicitly. +- **Corrupt-cache rejection**: a disk cache that fails to parse, is not a + dict, or is empty is ignored with a warning rather than served as + ``{}`` and silently breaking provider/model resolution. +- **Mirror URL override**: ``models_dev.url`` in config.yaml lets + deployments point at a mirror (e.g. a self-hosted copy) without code + changes. Other modules should import the dataclasses and query functions from here rather than parsing the raw JSON themselves. @@ -37,7 +54,7 @@ import requests logger = logging.getLogger(__name__) MODELS_DEV_URL = "https://models.dev/api.json" -_MODELS_DEV_CACHE_TTL = 3600 # 1 hour in-memory +_MODELS_DEV_CACHE_TTL = 4 * 3600 # 4 hours — ETag conditional GET makes refresh cheap _MODELS_DEV_RETRY_DELAY = 300 # 5 minutes after a failed refresh # In-memory cache @@ -197,8 +214,20 @@ PROVIDER_TO_MODELS_DEV: Dict[str, str] = { "ollama-cloud": "ollama-cloud", } -# Reverse mapping: models.dev → Hermes (built lazily) -_MODELS_DEV_TO_PROVIDER: Optional[Dict[str, str]] = None +# Reverse mapping: models.dev id → Hermes ids (built lazily; many-to-one, +# e.g. both "meta" and "meta-ai" may map to the same models.dev id). +_MODELS_DEV_TO_PROVIDER: Optional[Dict[str, List[str]]] = None + + +def _models_dev_to_hermes_ids(mdev_id: str) -> List[str]: + """Return the Hermes provider ids that map to *mdev_id* (may be []).""" + global _MODELS_DEV_TO_PROVIDER + if _MODELS_DEV_TO_PROVIDER is None: + reverse: Dict[str, List[str]] = {} + for hermes_id, mapped in PROVIDER_TO_MODELS_DEV.items(): + reverse.setdefault(mapped, []).append(hermes_id) + _MODELS_DEV_TO_PROVIDER = reverse + return _MODELS_DEV_TO_PROVIDER.get(mdev_id, []) @@ -208,18 +237,121 @@ def _get_cache_path() -> Path: return get_hermes_home() / "models_dev_cache.json" +def _get_etag_path() -> Path: + """Return path to the ETag sidecar file for conditional GET.""" + from hermes_constants import get_hermes_home + return get_hermes_home() / "models_dev_cache.etag" + + +def _load_etag() -> str: + """Load the last-known ETag from disk, or empty string if missing.""" + try: + etag_path = _get_etag_path() + if etag_path.exists(): + return etag_path.read_text(encoding="utf-8").strip() + except Exception as e: + logger.debug("Failed to load models.dev ETag: %s", e) + return "" + + +def _save_etag(etag: str) -> None: + """Persist an ETag to the sidecar file atomically.""" + try: + from utils import atomic_write_text + + etag_path = _get_etag_path() + etag_path.parent.mkdir(parents=True, exist_ok=True) + atomic_write_text(etag_path, etag) + except Exception as e: + logger.debug("Failed to save models.dev ETag: %s", e) + + +def _clear_etag() -> None: + """Delete the ETag sidecar so the next fetch is unconditional. + + Called when the cached registry the ETag vouches for is gone or + unusable — sending If-None-Match without a servable cache invites a + 304 that would leave the process with no data at all. + """ + try: + _get_etag_path().unlink(missing_ok=True) + except Exception as e: + logger.debug("Failed to clear models.dev ETag: %s", e) + + +def _get_models_dev_url() -> str: + """Resolve the models.dev API URL, honoring a config.yaml override. + + The ``models_dev.url`` config key lets deployments point at a mirror + (e.g. a self-hosted copy behind a corporate proxy) without code changes. + Falls back to the default public URL when unset or empty. + """ + try: + from hermes_cli.config import cfg_get, load_config_readonly + cfg = load_config_readonly() + url = cfg_get(cfg, "models_dev", "url", default="") + if isinstance(url, str) and url.strip(): + return url.strip() + except Exception: + pass + # Fall back to the module global (not the constant) so existing + # code/tests that patch MODELS_DEV_URL keep working. + return MODELS_DEV_URL + + +def _validate_registry(data: Any) -> bool: + """Return True if *data* is a non-empty dict suitable for serving.""" + return isinstance(data, dict) and len(data) > 0 + + def _load_disk_cache() -> Dict[str, Any]: - """Load models.dev data from disk cache.""" + """Load models.dev data from disk cache. + + A corrupt cache (invalid JSON, not a dict, or empty) is rejected with + a warning so it doesn't silently masquerade as ``{}`` and break + provider/model resolution for every caller. + """ try: cache_path = _get_cache_path() if cache_path.exists(): with open(cache_path, encoding="utf-8") as f: - return json.load(f) + data = json.load(f) + if not _validate_registry(data): + logger.warning( + "models.dev disk cache is corrupt or empty; " + "quarantining (will refetch from network)" + ) + _quarantine_corrupt_cache(cache_path) + return {} + return data except Exception as e: - logger.debug("Failed to load models.dev disk cache: %s", e) + logger.warning( + "Failed to load models.dev disk cache; quarantining: %s", e + ) + try: + _quarantine_corrupt_cache(_get_cache_path()) + except Exception: + pass return {} +def _quarantine_corrupt_cache(cache_path: Path) -> None: + """Move a rejected cache aside and drop its ETag sidecar. + + Renaming (rather than leaving the file in place) makes the rejection + a one-time event: without it, every hot-path call that finds the + in-memory cache empty re-reads and re-parses the corrupt file and + re-emits the warning until a network fetch succeeds. The sidecar is + cleared because it vouches for a registry we no longer hold — a 304 + against a missing cache would leave the process with no data at all. + """ + try: + cache_path.rename(cache_path.with_suffix(".json.corrupt")) + except Exception as e: + logger.debug("Could not quarantine corrupt models.dev cache: %s", e) + _clear_etag() + + def _disk_cache_age_seconds() -> Optional[float]: """Return age (in seconds) of the disk cache file, or None if missing. @@ -246,31 +378,68 @@ def _disk_cache_age_seconds() -> Optional[float]: return None -def _save_disk_cache(data: Dict[str, Any]) -> None: - """Save models.dev data to disk cache atomically.""" +def _save_disk_cache(data: Dict[str, Any], etag: str = "") -> None: + """Save models.dev data to disk cache atomically. + + Also persists the ETag sidecar when *etag* is non-empty so the next + refresh can issue a conditional GET. + """ try: cache_path = _get_cache_path() atomic_json_write(cache_path, data, indent=None, separators=(",", ":")) except Exception as e: logger.debug("Failed to save models.dev disk cache: %s", e) + if etag: + _save_etag(etag) -def _fetch_models_dev_from_network() -> Dict[str, Any]: - """Fetch the live models.dev registry without touching local caches. +class _NotModified(Exception): + """Server returned 304 Not Modified — existing cache is still valid.""" - Raises on network errors and on an empty/invalid registry payload. + +def _fetch_models_dev_from_network( + *, conditional: bool = False +) -> Tuple[Dict[str, Any], str]: + """Fetch the live models.dev registry. + + ``conditional`` enables ETag conditional GET (``If-None-Match`` with + the sidecar's ETag). Callers must pass True ONLY while holding + ``_models_dev_fetch_lock`` AND holding a servable registry the 304 + can re-confirm — a conditional request without one invites a 304 + that leaves the process with no data at all (previously a permanent + empty-registry loop when the sidecar outlived a corrupt cache file). + A 304 raises ``_NotModified`` so the caller can re-confirm the + existing cache's freshness without re-downloading the full payload. + + Returns ``(registry, etag)``; the etag is empty when the server sent + none. The caller persists it together with the cache body + (``_commit_registry``) so the sidecar can never get ahead of the data + it vouches for. Raises on network errors and on an empty/invalid + registry payload. """ + url = _get_models_dev_url() + headers: Dict[str, str] = {} + if conditional: + etag = _load_etag() + if etag: + headers["If-None-Match"] = etag + # Tuple (connect, read): a flat timeout=15 let a blackholed connect # stall the first-turn critical path for the full 15 s. 5 s connect # fails fast on unreachable hosts; 10 s read still tolerates a slow # registry response (matches the OpenRouter fetch convention in # agent/model_metadata.py). - response = requests.get(MODELS_DEV_URL, timeout=(5, 10)) + response = requests.get(url, headers=headers, timeout=(5, 10)) + + if response.status_code == 304: + raise _NotModified() + response.raise_for_status() data = response.json() - if not isinstance(data, dict) or not data: + if not _validate_registry(data): raise ValueError("models.dev returned an empty or invalid registry") - return data + + return data, response.headers.get("ETag", "") def _mark_stale_cache_grace() -> None: @@ -286,7 +455,7 @@ def _mark_stale_cache_grace() -> None: _models_dev_cache_time = grace_time -def _commit_registry(data: Dict[str, Any], *, where: str) -> None: +def _commit_registry(data: Dict[str, Any], *, etag: str = "", where: str) -> None: """Persist a freshly fetched registry: disk + in-mem + clear backoff. Callers must hold ``_models_dev_fetch_lock`` so a failing refresh on one @@ -295,7 +464,7 @@ def _commit_registry(data: Dict[str, Any], *, where: str) -> None: immediately after a successful ``force_refresh``). """ global _models_dev_cache, _models_dev_cache_time, _models_dev_retry_after - _save_disk_cache(data) + _save_disk_cache(data, etag) _models_dev_cache = data _models_dev_cache_time = time.time() _models_dev_retry_after = 0 @@ -307,6 +476,39 @@ def _commit_registry(data: Dict[str, Any], *, where: str) -> None: ) +def _confirm_cache_not_modified(*, where: str) -> None: + """Re-confirm the existing cache as fresh after a 304 Not Modified. + + Callers must hold ``_models_dev_fetch_lock``. Clears the backoff and + resets the in-memory cache timestamp so the next caller hits the fast + path. The disk cache itself is not rewritten — its contents are + unchanged, only its freshness marker is advanced. + """ + global _models_dev_cache_time, _models_dev_retry_after + if not _models_dev_cache: + # Pathological: a 304 arrived but we hold no registry. Should be + # unreachable now that conditional GETs require a servable cache + # (see _fetch_models_dev_from_network); kept as defense in depth + # because this state previously caused a permanent empty-registry + # loop. Drop the sidecar so the next attempt is unconditional and + # arm the normal failure backoff instead of marking {} "fresh". + _clear_etag() + _models_dev_retry_after = time.time() + _MODELS_DEV_RETRY_DELAY + logger.warning( + "models.dev returned 304 but no cached registry is held (%s); " + "cleared ETag sidecar, will refetch unconditionally", + where, + ) + return + _models_dev_cache_time = time.time() + _models_dev_retry_after = 0 + logger.debug( + "models.dev registry unchanged (304 Not Modified, %s); " + "cache re-confirmed fresh", + where, + ) + + def _note_refresh_failure(exc: Exception, *, where: str) -> None: """Record a failed refresh: arm the process-wide 5-minute backoff. @@ -326,9 +528,19 @@ def _background_refresh_models_dev() -> None: """Best-effort refresh after serving stale cache data.""" global _models_dev_refresh_in_flight try: - data = _fetch_models_dev_from_network() + # Fetch INSIDE the lock: symmetric with the foreground path, so + # conditional-GET inputs (memory cache + etag sidecar) can't be + # mutated mid-fetch by a concurrent force_refresh, and the two + # paths can't double-download concurrently. Hot-path callers are + # unaffected — they return stale data without touching this lock. with _models_dev_fetch_lock: - _commit_registry(data, where="background") + data, etag = _fetch_models_dev_from_network( + conditional=bool(_models_dev_cache) + ) + _commit_registry(data, etag=etag, where="background") + except _NotModified: + with _models_dev_fetch_lock: + _confirm_cache_not_modified(where="background") except Exception as e: with _models_dev_fetch_lock: _note_refresh_failure(e, where="background") @@ -372,6 +584,12 @@ def fetch_models_dev( Returns the full registry dict keyed by provider ID, or empty dict on failure. + Network requests use ETag conditional GET when a cached ETag exists + AND a servable registry is held (on a cold ``force_refresh`` the + memory cache is hydrated from disk first). A 304 Not Modified + response re-confirms the existing cache's freshness without + re-downloading the full (~2 MB) registry. + Cache hierarchy (when ``force_refresh=False``): 1. Fresh in-memory cache → return immediately. 2. Stale in-memory cache → return immediately and refresh in a single @@ -380,6 +598,7 @@ def fetch_models_dev( new models, so stale data is preferable to a foreground timeout. 3. Disk cache file (any age) → load, populate in-mem, return immediately. Stale disk caches trigger the same background refresh. + A corrupt or empty disk cache is rejected with a warning. 4. No cache at all → singleflight foreground network fetch. On success, save to disk + in-mem and return. 5. Any failed refresh (foreground or background) suppresses further @@ -390,8 +609,9 @@ def fetch_models_dev( backoff are bypassed; the function hits the network and only falls back to cached data if the call fails. When ``allow_network=False``, any memory or disk cache is returned regardless of age and no request is - made — used by latency-sensitive paths (gateway route-identity checks) - that must never wait on the network. + made — used by latency-sensitive paths (gateway route-identity checks, + vision routing, context-length lookup) that must never wait on the + network. """ global _models_dev_cache, _models_dev_cache_time, _models_dev_retry_after @@ -472,10 +692,28 @@ def fetch_models_dev( if now < _models_dev_retry_after: return _models_dev_cache + # Cold force_refresh (fresh CLI process): stages 1-3 were skipped, + # so the memory cache may be empty even though a servable disk + # cache + ETag sidecar exist. Hydrate first so the conditional GET + # fires (a 304 then re-confirms the disk data instead of + # re-downloading the full ~2 MB registry). + if force_refresh and not _models_dev_cache: + disk = _load_disk_cache() + if disk: + _models_dev_cache = disk + _models_dev_cache_time = 0 # servable but not fresh + try: - data = _fetch_models_dev_from_network() - _commit_registry(data, where="foreground") + data, etag = _fetch_models_dev_from_network( + conditional=bool(_models_dev_cache) + ) + _commit_registry(data, etag=etag, where="foreground") return data + except _NotModified: + # Server confirmed our cache is still valid. Re-confirm freshness + # without re-downloading the full registry. + _confirm_cache_not_modified(where="foreground") + return _models_dev_cache except Exception as e: _note_refresh_failure(e, where="foreground") @@ -494,24 +732,47 @@ def fetch_models_dev( return _models_dev_cache -def lookup_models_dev_context(provider: str, model: str) -> Optional[int]: +def lookup_models_dev_context( + provider: str, model: str, *, allow_network: bool = False +) -> Optional[int]: """Look up context_length for a provider+model combo in models.dev. Returns the context window in tokens, or None if not found. Handles case-insensitive matching and filters out context=0 entries. + + An EXPLICIT ``model_overrides`` config entry for this provider+model + wins over the catalog value; ``_default`` entries fill the gap only + when the catalog has no answer — the supported self-unblock path for + models with wrong or missing context in models.dev (#84482). + + ``allow_network`` defaults to False — context-length lookup is a + hot path (called during every conversation turn) and must never block + on the network. Pass True only from explicit refresh flows. """ + # Explicit config override — checked before catalog so it always wins. + override_ctx = _override_context_window(provider, model) + if override_ctx is not None: + return override_ctx + mdev_provider_id = PROVIDER_TO_MODELS_DEV.get(provider) if not mdev_provider_id: - return None + return _default_override_context(provider) - data = fetch_models_dev() + # NOTE: keep the zero-argument call on the allow_network path. Dozens + # of test sites monkeypatch fetch_models_dev with zero-arg lambdas; + # passing the kwarg unconditionally breaks them all (TypeError). + data = ( + fetch_models_dev() + if allow_network + else fetch_models_dev(allow_network=False) + ) provider_data = data.get(mdev_provider_id) if not isinstance(provider_data, dict): - return None + return _default_override_context(provider) models = provider_data.get("models", {}) if not isinstance(models, dict): - return None + return _default_override_context(provider) # Exact match entry = models.get(model) @@ -550,7 +811,16 @@ def lookup_models_dev_context(provider: str, model: str) -> Optional[int]: if ctx: return ctx - return None + # Catalog miss — a _default override may fill the gap (#84482). + return _default_override_context(provider) + + +def _default_override_context(provider: str) -> Optional[int]: + """Fill-gap context from a ``_default`` override, for catalog misses.""" + default = _default_model_override(provider) + if default is None: + return None + return _override_int(default, "context_window") def _extract_context(entry: Dict[str, Any]) -> Optional[int]: @@ -586,16 +856,266 @@ class ModelCapabilities: model_family: str = "" -def _get_provider_models(provider: str) -> Optional[Dict[str, Any]]: +# --------------------------------------------------------------------------- # +# Per-model metadata overrides (config.yaml → model_overrides) # +# --------------------------------------------------------------------------- # +# +# Canonical override schema (the ONLY key space consumers accept): +# context_window, max_output_tokens, supports_tools, supports_vision, +# supports_reasoning, model_family +# +# Resolution semantics: +# 1. ``model_overrides..`` — explicit override. Always +# wins over the catalog for the fields it sets (partial patch). +# 2. ``model_overrides.._default`` / ``model_overrides._default`` +# — FILL-GAP defaults. They apply ONLY to models the catalog does not +# know (the #8731/#84482 self-unblock path for custom/local/new +# models) and never displace catalog data for known models. A +# ``_default: {context_window: 128000}`` therefore cannot clamp every +# catalog-known model of a provider. +# +# Provider keys accept the Hermes provider id (as used elsewhere in +# config.yaml) or the models.dev provider id. Model ids match exactly, +# then case-insensitively (mirroring catalog lookup). + +_OVERRIDE_WARNED_KEYS: set = set() + + +def _load_model_overrides() -> Dict[str, Any]: + """Load the ``model_overrides`` config section. + + No local memoization on purpose: ``load_config_readonly()`` is already + (mtime, size)-cached upstream (a hit is ~one stat, no deepcopy, no + parse), and an ``id(cfg)``-keyed layer here can serve stale overrides + after a config reload when CPython reuses the freed dict's address. + Returns empty dict on any failure. + """ + try: + from hermes_cli.config import cfg_get, load_config_readonly + raw = cfg_get(load_config_readonly(), "model_overrides", default={}) + return raw if isinstance(raw, dict) else {} + except Exception: + return {} + + +def _provider_override_section(provider: str) -> Optional[Dict[str, Any]]: + """Return the override section for *provider*, or None. + + Accepts either the Hermes provider id or the models.dev provider id as + the config key, so ``copilot`` and ``github-copilot`` both work + regardless of which id space a caller passes in. + """ + overrides = _load_model_overrides() + if not overrides: + return None + provider_key = (provider or "").strip() + if not provider_key: + return None + + candidates = [provider_key] + mapped = PROVIDER_TO_MODELS_DEV.get(provider_key) + if mapped and mapped != provider_key: + candidates.append(mapped) + # Reverse: caller passed a models.dev id, config keyed by Hermes id. + for hermes_id in _models_dev_to_hermes_ids(provider_key): + if hermes_id != provider_key: + candidates.append(hermes_id) + + for key in candidates: + section = overrides.get(key) + if isinstance(section, dict): + return section + return None + + +def _explicit_model_override(provider: str, model: str) -> Optional[Dict[str, Any]]: + """Return the explicit per-provider+model override dict, or None. + + Model ids match exactly first, then case-insensitively (skipping the + ``_default`` sentinel), mirroring catalog lookup behavior. + """ + model_key = (model or "").strip() + if not model_key: + return None + section = _provider_override_section(provider) + if section is None: + return None + + entry = section.get(model_key) + if isinstance(entry, dict): + return entry + + model_lower = model_key.lower() + for mid, mdata in section.items(): + if mid == "_default": + continue + if mid.lower() == model_lower and isinstance(mdata, dict): + return mdata + return None + + +def _default_model_override(provider: str) -> Optional[Dict[str, Any]]: + """Return the fill-gap ``_default`` override for *provider*, or None. + + Checks the per-provider ``_default`` first, then the global one. Only + consulted for models the catalog does not know — see the block comment. + """ + section = _provider_override_section(provider) + if section is not None: + default = section.get("_default") + if isinstance(default, dict): + return default + overrides = _load_model_overrides() + global_default = overrides.get("_default") + if isinstance(global_default, dict): + return global_default + return None + + +def _override_for( + provider: str, model: str, *, catalog_hit: bool +) -> Optional[Dict[str, Any]]: + """Select the override dict for a lookup, honoring fill-gap semantics. + + Explicit per-provider+model overrides always apply. ``_default`` + entries apply only when the catalog has no entry for the model. + """ + explicit = _explicit_model_override(provider, model) + if explicit is not None: + return explicit + if catalog_hit: + return None + return _default_model_override(provider) + + +def _override_int(override: Dict[str, Any], key: str) -> Optional[int]: + """Coerce an override field to a positive int, warning once on garbage.""" + raw = override.get(key) + if raw is None: + return None + try: + value = int(raw) + if value > 0: + return value + except (TypeError, ValueError): + pass + warn_key = (key, repr(raw)) + if warn_key not in _OVERRIDE_WARNED_KEYS: + _OVERRIDE_WARNED_KEYS.add(warn_key) + logger.warning( + "model_overrides: ignoring invalid %s value %r " + "(expected a positive integer)", key, raw, + ) + return None + + +def _override_context_window(provider: str, model: str) -> Optional[int]: + """Return the EXPLICITLY overridden context_window, or None. + + Explicit-only on purpose: this runs early in the resolution chain + (agent/model_metadata.py step 0b, before custom_providers and live + probes), where a ``_default`` must not preempt more specific sources. + Fill-gap defaults are applied later by ``lookup_models_dev_context`` + once the catalog has actually missed. + """ + ov = _explicit_model_override(provider, model) + if ov is None: + return None + return _override_int(ov, "context_window") + + +def _override_to_catalog_shape( + override: Dict[str, Any], +) -> Tuple[Dict[str, Any], Optional[bool]]: + """Translate canonical override keys into a models.dev-shaped patch. + + ``get_model_info``/``_parse_model_info`` consume the raw catalog shape + (``limit.context``, ``tool_call``, ...). All override consumers accept + ONE canonical schema (the documented ``context_window``/``supports_*`` + keys), so this boundary translates rather than forcing users to know + the internal catalog shape. + + Returns ``(patch, vision)`` — vision is returned out-of-band (not as + a key in the patch) because it maps onto the catalog's + ``modalities.input`` list rather than a scalar field. + """ + patch: Dict[str, Any] = {} + limit: Dict[str, Any] = {} + ctx = _override_int(override, "context_window") + if ctx is not None: + limit["context"] = ctx + out = _override_int(override, "max_output_tokens") + if out is not None: + limit["output"] = out + if limit: + patch["limit"] = limit + if "supports_tools" in override: + patch["tool_call"] = bool(override["supports_tools"]) + if "supports_reasoning" in override: + patch["reasoning"] = bool(override["supports_reasoning"]) + vision: Optional[bool] = None + if "supports_vision" in override: + vision = bool(override["supports_vision"]) + patch["attachment"] = vision + if "model_family" in override: + patch["family"] = str(override["model_family"] or "") + return patch, vision + + +def _merge_catalog_entry_with_override( + raw: Dict[str, Any], override: Dict[str, Any] +) -> Dict[str, Any]: + """Patch a catalog entry with a canonical-schema override. + + Sub-dicts (``limit``, ``modalities``) are merged, not clobbered — an + override setting only ``context_window`` must not wipe the catalog's + ``limit.output``. + """ + shaped, vision_override = _override_to_catalog_shape(override) + merged = dict(raw) + limit_patch = shaped.pop("limit", None) + if limit_patch: + base_limit = raw.get("limit") + base_limit = dict(base_limit) if isinstance(base_limit, dict) else {} + base_limit.update(limit_patch) + merged["limit"] = base_limit + if vision_override is not None: + base_mods = raw.get("modalities") + base_mods = dict(base_mods) if isinstance(base_mods, dict) else {} + input_mods = base_mods.get("input") + input_mods = list(input_mods) if isinstance(input_mods, list) else [] + if vision_override and "image" not in input_mods: + input_mods.append("image") + elif not vision_override and "image" in input_mods: + input_mods.remove("image") + base_mods["input"] = input_mods + merged["modalities"] = base_mods + merged.update(shaped) + return merged + + +def _get_provider_models( + provider: str, *, allow_network: bool = False +) -> Optional[Dict[str, Any]]: """Resolve a Hermes provider ID to its models dict from models.dev. Returns the models dict or None if the provider is unknown or has no data. + + ``allow_network`` defaults to False — this is called from hot paths + (vision routing, image routing, capability checks) and must never block. """ mdev_provider_id = PROVIDER_TO_MODELS_DEV.get(provider) if not mdev_provider_id: return None - data = fetch_models_dev() + # NOTE: keep the zero-argument call on the allow_network path. Dozens + # of test sites monkeypatch fetch_models_dev with zero-arg lambdas; + # passing the kwarg unconditionally breaks them all (TypeError). + data = ( + fetch_models_dev() + if allow_network + else fetch_models_dev(allow_network=False) + ) provider_data = data.get(mdev_provider_id) if not isinstance(provider_data, dict): return None @@ -608,7 +1128,15 @@ def _get_provider_models(provider: str) -> Optional[Dict[str, Any]]: def _find_model_entry(models: Dict[str, Any], model: str) -> Optional[Dict[str, Any]]: - """Find a model entry by exact match, then case-insensitive fallback.""" + """Find a model entry: exact, case-insensitive, then suffix fallback. + + The ``:cloud``/``-cloud`` suffix fallback mirrors + ``lookup_models_dev_context`` so "is this model in the catalog" means + the same thing to every consumer — important for ``model_overrides`` + fill-gap ``_default`` semantics, where a suffix-keyed catalog model + (e.g. ``kimi-k2.6:cloud``) must count as KNOWN and keep its catalog + metadata rather than being displaced by a ``_default``. + """ # Exact match entry = models.get(model) if isinstance(entry, dict): @@ -620,15 +1148,39 @@ def _find_model_entry(models: Dict[str, Any], model: str) -> Optional[Dict[str, if mid.lower() == model_lower and isinstance(mdata, dict): return mdata + # Suffix-aware fallback (e.g. ollama-cloud stores kimi-k2.6:cloud + # while the live API returns the bare name). + for suffix in (":cloud", "-cloud"): + entry = models.get(model + suffix) + if isinstance(entry, dict): + return entry + suffixed_lower = model_lower + suffix + for mid, mdata in models.items(): + if mid.lower() == suffixed_lower and isinstance(mdata, dict): + return mdata + return None -def get_model_capabilities(provider: str, model: str) -> Optional[ModelCapabilities]: +def get_model_capabilities( + provider: str, model: str, *, allow_network: bool = False +) -> Optional[ModelCapabilities]: """Look up full capability metadata from models.dev cache. Uses the existing fetch_models_dev() and PROVIDER_TO_MODELS_DEV mapping. Returns None if model not found. + EXPLICIT ``model_overrides`` entries (per-provider+model) win over + catalog values for the fields they set. ``_default`` entries fill the + gap only for models the catalog does not know — the supported + self-unblock path for custom/local models (#8731) and for models with + wrong metadata in models.dev (#84482). An override may set any subset + of fields; unspecified fields fall through to the catalog value (or + sensible defaults when the model is absent from the catalog). + + ``allow_network`` defaults to False — capability lookup is a hot path + (vision routing, image routing) and must never block on the network. + Extracts from model entry fields: - reasoning (bool) → supports_reasoning - tool_call (bool) → supports_tools @@ -637,42 +1189,72 @@ def get_model_capabilities(provider: str, model: str) -> Optional[ModelCapabilit - limit.output (int) → max_output_tokens - family (str) → model_family """ - models = _get_provider_models(provider) - if models is None: + models = _get_provider_models(provider, allow_network=allow_network) + entry = _find_model_entry(models, model) if models is not None else None + + # Select the override AFTER the catalog lookup: explicit overrides + # always apply; _default entries only fill gaps for catalog misses. + override = _override_for(provider, model, catalog_hit=entry is not None) + + # If no catalog entry and no override, we can't resolve capabilities. + if entry is None and override is None: return None - entry = _find_model_entry(models, model) - if entry is None: - return None + # Start from catalog entry (if found), else use defaults. + if entry is not None: + supports_tools = bool(entry.get("tool_call", False)) + # Vision: prefer explicit `modalities.input` when models.dev provides it. + # The older `attachment` flag can be stale or too broad for image routing; + # fall back to it only when the input modalities are absent/invalid. + input_mods = entry.get("modalities", {}) + if isinstance(input_mods, dict): + input_mods = input_mods.get("input") + else: + input_mods = None + if isinstance(input_mods, list): + supports_vision = "image" in input_mods + else: + supports_vision = bool(entry.get("attachment", False)) + supports_reasoning = bool(entry.get("reasoning", False)) - # Extract capability flags (default to False if missing) - supports_tools = bool(entry.get("tool_call", False)) - # Vision: prefer explicit `modalities.input` when models.dev provides it. - # The older `attachment` flag can be stale or too broad for image routing; - # fall back to it only when the input modalities are absent/invalid. - input_mods = entry.get("modalities", {}) - if isinstance(input_mods, dict): - input_mods = input_mods.get("input") + limit = entry.get("limit", {}) + if not isinstance(limit, dict): + limit = {} + + ctx = limit.get("context") + context_window = int(ctx) if isinstance(ctx, (int, float)) and ctx > 0 else 200000 + + out = limit.get("output") + max_output_tokens = int(out) if isinstance(out, (int, float)) and out > 0 else 8192 + + model_family = entry.get("family", "") or "" else: - input_mods = None - if isinstance(input_mods, list): - supports_vision = "image" in input_mods - else: - supports_vision = bool(entry.get("attachment", False)) - supports_reasoning = bool(entry.get("reasoning", False)) + # Unknown model — derive sensible defaults. The override will + # patch whichever fields it specifies; the rest stay at defaults + # that are safe for agentic use (tools on, vision/reasoning off). + supports_tools = True + supports_vision = False + supports_reasoning = False + context_window = 200000 + max_output_tokens = 8192 + model_family = "" - # Extract limits - limit = entry.get("limit", {}) - if not isinstance(limit, dict): - limit = {} - - ctx = limit.get("context") - context_window = int(ctx) if isinstance(ctx, (int, float)) and ctx > 0 else 200000 - - out = limit.get("output") - max_output_tokens = int(out) if isinstance(out, (int, float)) and out > 0 else 8192 - - model_family = entry.get("family", "") or "" + # Apply override patches (each field is optional in the override dict). + if override is not None: + if "supports_tools" in override: + supports_tools = bool(override["supports_tools"]) + if "supports_vision" in override: + supports_vision = bool(override["supports_vision"]) + if "supports_reasoning" in override: + supports_reasoning = bool(override["supports_reasoning"]) + ctx_ov = _override_int(override, "context_window") + if ctx_ov is not None: + context_window = ctx_ov + out_ov = _override_int(override, "max_output_tokens") + if out_ov is not None: + max_output_tokens = out_ov + if "model_family" in override: + model_family = str(override["model_family"] or "") return ModelCapabilities( supports_tools=supports_tools, @@ -684,15 +1266,21 @@ def get_model_capabilities(provider: str, model: str) -> Optional[ModelCapabilit ) -def list_provider_models(provider: str) -> List[str]: +def list_provider_models( + provider: str, *, allow_network: bool = True +) -> List[str]: """Return all model IDs for a provider from models.dev. Returns an empty list if the provider is unknown or has no data. + + ``allow_network`` defaults to True — this is called from the model + picker (``hermes model``), which is an interactive user-facing flow + where a fresh catalog is worth a short network wait. """ from hermes_cli.models import normalize_provider provider = normalize_provider(provider) or provider - models = _get_provider_models(provider) + models = _get_provider_models(provider, allow_network=allow_network) if models is None: return [] return [ @@ -748,14 +1336,19 @@ def _should_hide_from_provider_catalog(provider: str, model_id: str) -> bool: return False -def list_agentic_models(provider: str) -> List[str]: +def list_agentic_models( + provider: str, *, allow_network: bool = True +) -> List[str]: """Return model IDs suitable for agentic use from models.dev. Filters for tool_call=True and excludes noise (TTS, embedding, dated preview snapshots, live/streaming, image-only models). Returns an empty list on any failure. + + ``allow_network`` defaults to True — like ``list_provider_models``, + this is called from interactive model selection flows. """ - models = _get_provider_models(provider) + models = _get_provider_models(provider, allow_network=allow_network) if models is None: return [] @@ -854,6 +1447,11 @@ def get_provider_info( Accepts either a Hermes provider ID (e.g. "kilocode") or a models.dev ID (e.g. "kilo"). Returns None if the provider is not in the catalog. + + ``allow_network`` defaults to True — the primary caller is + ``resolve_provider_full`` during interactive setup, where a fresh + catalog is worth a short network wait. Hot-path callers should pass + ``allow_network=False``. """ # Resolve Hermes ID → models.dev ID mdev_id = PROVIDER_TO_MODELS_DEV.get(provider_id, provider_id) @@ -878,33 +1476,75 @@ def get_provider_info( # --------------------------------------------------------------------------- def get_model_info( - provider_id: str, model_id: str + provider_id: str, model_id: str, *, allow_network: bool = False ) -> Optional[ModelInfo]: """Get full model metadata from models.dev. Accepts Hermes or models.dev provider ID. Tries exact match then case-insensitive fallback. Returns None if not found. + + ``model_overrides`` entries use the SAME canonical schema as every + other consumer (``context_window``, ``max_output_tokens``, + ``supports_*``, ``model_family``) — they are translated into the + catalog shape at this boundary, and sub-dicts (``limit``, + ``modalities``) are merged rather than clobbered. EXPLICIT entries + patch known catalog models; ``_default`` entries fill the gap only + for models the catalog does not know (#8731, #84482). + + ``allow_network`` defaults to False — model info lookup is a hot path + (cost guard, inventory) and must never block on the network. """ mdev_id = PROVIDER_TO_MODELS_DEV.get(provider_id, provider_id) - data = fetch_models_dev() + def _from_override_alone() -> Optional[ModelInfo]: + override = _override_for(provider_id, model_id, catalog_hit=False) + if override is None: + return None + # Seed the same safe defaults get_model_capabilities uses for + # unknown models (200K context, tools on) so the two + # unknown-model paths agree; the override patches its fields on + # top. + base = { + "limit": {"context": 200000, "output": 8192}, + "tool_call": True, + } + shaped = _merge_catalog_entry_with_override(base, override) + return _parse_model_info(model_id, shaped, mdev_id) + + # NOTE: keep the zero-argument call on the allow_network path. Dozens + # of test sites monkeypatch fetch_models_dev with zero-arg lambdas; + # passing the kwarg unconditionally breaks them all (TypeError). + data = ( + fetch_models_dev() + if allow_network + else fetch_models_dev(allow_network=False) + ) pdata = data.get(mdev_id) if not isinstance(pdata, dict): - return None + return _from_override_alone() models = pdata.get("models", {}) if not isinstance(models, dict): - return None + return _from_override_alone() + + def _with_override(mid: str, raw: Dict[str, Any]) -> ModelInfo: + override = _override_for(provider_id, model_id, catalog_hit=True) + if override is not None: + merged = _merge_catalog_entry_with_override(raw, override) + return _parse_model_info(mid, merged, mdev_id) + return _parse_model_info(mid, raw, mdev_id) # Exact match raw = models.get(model_id) if isinstance(raw, dict): - return _parse_model_info(model_id, raw, mdev_id) + return _with_override(model_id, raw) # Case-insensitive fallback model_lower = model_id.lower() for mid, mdata in models.items(): if mid.lower() == model_lower and isinstance(mdata, dict): - return _parse_model_info(mid, mdata, mdev_id) + return _with_override(mid, mdata) - return None + # Model not in catalog — an override (explicit or _default) may still + # provide the metadata. + return _from_override_alone() diff --git a/agent/pet/generate/orchestrate.py b/agent/pet/generate/orchestrate.py index 54a1adf5b0..1b267a4de2 100644 --- a/agent/pet/generate/orchestrate.py +++ b/agent/pet/generate/orchestrate.py @@ -71,8 +71,24 @@ def _harden_transparency(path: Path) -> Path: # Zero the RGB of any leftover semi-transparent edge pixels so a keyed # draft has no colored halo when composited on the dark UI. keyed = atlas._clear_transparent_rgb(keyed) - out = path.with_suffix(".png") + # PNG inputs are hardened in place, including mixed-case suffixes like + # .PNG. with_suffix(".png") would name a different Path string that still + # resolves to the same file on case-insensitive filesystems (macOS APFS, + # Windows), and unlinking path after save would delete the hardened output. + if path.suffix.lower() == ".png": + out = path + else: + out = path.with_suffix(".png") keyed.save(out, format="PNG") + if out != path: + # The hardened PNG stands in for the draft. When the provider handed + # back a non-PNG file (webp, jpg, gif), out is a different path, so + # remove the original instead of leaving it behind in cache/images + # (nothing prunes that directory outside the gateway loop). + try: + path.unlink(missing_ok=True) + except OSError: + pass return out except Exception as exc: # noqa: BLE001 - cosmetic; fall back to the raw image logger.debug("base draft transparency hardening failed for %s: %s", path, exc) @@ -246,6 +262,7 @@ def hatch_pet( if cancelled(): return state, None strict = attempt < _ROW_GEN_ATTEMPTS - 1 + strips: list[Path] = [] try: strips = imagegen.generate( prompts.build_row_prompt(state, count, label, style=style), @@ -274,6 +291,18 @@ def hatch_pet( "pet hatch %r: row %r attempt %d/%d failed: %s", slug, state, attempt + 1, _ROW_GEN_ATTEMPTS, exc, ) + finally: + # The strip is an intermediate. extract_strip_frames has already + # decoded its frames into memory, so drop the row image after + # every attempt (success or failure). Nothing prunes + # cache/images outside the gateway housekeeping loop, so a CLI + # or desktop hatch would otherwise leave each strip behind for + # good and grow the cache without bound. + for strip in strips: + try: + Path(strip).unlink(missing_ok=True) + except OSError: + pass logger.warning( "pet hatch %r: row %r gave up after %.1fs: %s", slug, state, time.monotonic() - t0, last_exc, diff --git a/agent/prompt_builder.py b/agent/prompt_builder.py index 96f3eb4786..ccfa217f4e 100644 --- a/agent/prompt_builder.py +++ b/agent/prompt_builder.py @@ -13,7 +13,13 @@ import contextvars from collections import OrderedDict from pathlib import Path -from hermes_constants import get_hermes_home, get_skills_dir, is_wsl +from hermes_constants import ( + get_hermes_home, + get_skills_dir, + is_wsl, + reset_hermes_home_override, + set_hermes_home_override, +) from typing import List, Optional from agent.runtime_cwd import resolve_agent_cwd @@ -191,10 +197,21 @@ SESSION_SEARCH_GUIDANCE = ( "asking them to repeat themselves." ) +# NOTE (#82154): the opening sentence is worded deliberately. Anthropic's +# server-side content filter rejects the previous phrasing ("After completing a +# complex task (5+ tool calls), fixing a tricky error, or discovering a +# non-trivial workflow, save the approach as a skill with skill_manage so you +# can reuse it next time.") on subscription OAuth credentials, and surfaces that +# rejection as a billing-shaped HTTP 400 ("You're out of extra usage"), which +# sends users to buy quota they do not need. Bisected against the live API: that +# sentence alone reproduces the 400 and removing it alone clears it; size and +# the system[0] identity gate were both ruled out. The reword is empirically +# validated, not understood — if you rewrite this sentence, re-verify against a +# subscription OAuth token, not an sk-ant-api… key, which does not hit the +# filter. SKILLS_GUIDANCE = ( - "After completing a complex task (5+ tool calls), fixing a tricky error, " - "or discovering a non-trivial workflow, save the approach as a " - "skill with skill_manage so you can reuse it next time.\n" + "When you work out a non-trivial workflow, record it with skill_manage " + "for future reuse.\n" "When using a skill and finding it outdated, incomplete, or wrong, " "patch it immediately with skill_manage(action='patch') — don't wait to be asked. " "Skills that aren't maintained become liabilities.\n" @@ -1477,7 +1494,12 @@ def drain_truncation_warnings() -> list: # Skills prompt cache # ========================================================================= -_SKILLS_PROMPT_CACHE_MAX = 8 +# Sized for multi-profile processes: since #86313 the cache key carries a +# per-profile skills_dir (one entry per profile × platform), so the old cap +# of 8 could thrash on a gateway multiplexing default + several bots (each +# miss = full os.walk manifest rebuild). ~32 costs low single-digit MB worst +# case. +_SKILLS_PROMPT_CACHE_MAX = 32 _SKILLS_PROMPT_CACHE: OrderedDict[tuple, str] = OrderedDict() _SKILLS_PROMPT_CACHE_LOCK = threading.Lock() # v2: entries gained org provenance fields (org_id/org_author/rel_dir) for M2 @@ -1718,6 +1740,7 @@ def build_skills_system_prompt( available_tools: "set[str] | None" = None, available_toolsets: "set[str] | None" = None, compact_categories: "frozenset[str] | None" = None, + skills_dir_override: "Path | None" = None, ) -> str: """Build a compact skill index for the system prompt. @@ -1739,13 +1762,44 @@ def build_skills_system_prompt( visible and loadable via ``skill_view`` / ``skills_list``; only the descriptions are dropped, and a footer note explains the demotion. """ - skills_dir = get_skills_dir() - external_dirs = get_all_skills_dirs()[1:] # skip local (index 0) + # Home resolution is EXPLICIT when a caller passes skills_dir_override + # (the agent knows its own profile home from its session_db path). This + # avoids the ContextVar-on-a-thread trap: build threads that didn't bind + # HERMES_HOME would otherwise fall back to the launch (default) home and + # leak the default profile's skills into a bot's prompt (confirmed: a + # no-override thread builds default's full index). Snapshot + external + # dirs are scoped to the same home so nothing reads ambient state. + if skills_dir_override is not None: + skills_dir = Path(skills_dir_override) + _home_token = set_hermes_home_override(str(skills_dir.parent)) + else: + skills_dir = get_skills_dir() + _home_token = None + try: + external_dirs = get_all_skills_dirs()[1:] # skip local (index 0) - if not skills_dir.exists() and not external_dirs: - return "" + if not skills_dir.exists() and not external_dirs: + return "" - # ── Layer 1: in-process LRU cache ───────────────────────────────── + return _build_skills_system_prompt_inner( + skills_dir, + external_dirs, + available_tools, + available_toolsets, + compact_categories, + ) + finally: + if _home_token is not None: + reset_hermes_home_override(_home_token) + + +def _build_skills_system_prompt_inner( + skills_dir: "Path", + external_dirs: "list[Path]", + available_tools: "set[str] | None", + available_toolsets: "set[str] | None", + compact_categories: "frozenset[str] | None", +) -> str: # Include the resolved platform so per-platform disabled-skill lists # produce distinct cache entries (gateway serves multiple platforms). _platform_hint = _current_session_platform_hint() @@ -2116,12 +2170,21 @@ def _truncate_content( return head + marker + tail -def load_soul_md(context_length: Optional[int] = None) -> Optional[str]: +def load_soul_md( + context_length: Optional[int] = None, + home_override: "Path | None" = None, +) -> Optional[str]: """Load SOUL.md from HERMES_HOME and return its content, or None. Used as the agent identity (slot #1 in the system prompt). When this returns content, ``build_context_files_prompt`` should be called with ``skip_soul=True`` so SOUL.md isn't injected twice. + + ``home_override`` scopes the read to an explicit profile home (the agent + knows its own home from its session_db path). Without it, resolution is + ambient — which on a thread that lost the HERMES_HOME ContextVar falls + back to the launch home and reads the wrong profile's SOUL.md (#50233, + same class as the skills-index leak fixed in #86313). """ try: from hermes_cli.config import ensure_hermes_home @@ -2129,7 +2192,8 @@ def load_soul_md(context_length: Optional[int] = None) -> Optional[str]: except Exception as e: logger.debug("Could not ensure HERMES_HOME before loading SOUL.md: %s", e) - soul_path = get_hermes_home() / "SOUL.md" + _home = Path(home_override) if home_override is not None else get_hermes_home() + soul_path = _home / "SOUL.md" if not soul_path.exists(): return None try: @@ -2312,6 +2376,7 @@ def build_context_files_prompt( skip_soul: bool = False, context_length: Optional[int] = None, allow_install_tree_fallback: bool = False, + home_override: "Path | None" = None, ) -> str: """Discover and load context files for the system prompt. @@ -2375,7 +2440,7 @@ def build_context_files_prompt( # SOUL.md from HERMES_HOME only — skip when already loaded as identity if not skip_soul: - soul_content = load_soul_md(context_length) + soul_content = load_soul_md(context_length, home_override=home_override) if soul_content: sections.append(soul_content) diff --git a/agent/prompt_cache_scope.py b/agent/prompt_cache_scope.py new file mode 100644 index 0000000000..b39fb29672 --- /dev/null +++ b/agent/prompt_cache_scope.py @@ -0,0 +1,125 @@ +"""Rotation-stable logical cache scope for prompt_cache_key derivation. + +Context-compression rotation (legacy ``compression.in_place: false`` mode) +mints a new physical ``session_id`` mid-conversation to segment the +transcript. The prompt-cache scope introduced by #79161 was derived from that +physical id, so every rotation moved the conversation into a fresh cache +bucket even though it is logically the same conversation continuing +(issue #79017). + +``resolve_prompt_cache_scope()`` maps the physical session id to the ROOT of +its *compression lineage* — the pre-rotation session id — using +``SessionDB.get_compression_lineage()``, whose fork-aware semantics +(hardened in #79193) give exactly the scope boundaries the cache key needs. +NOT ``SessionDB.get_conversation_root`` / ``run_agent._conversation_root_id`` +(the Portal-attribution walk): that one follows ``parent_session_id`` blindly, +collapsing /branch children and whole delegate trees into one id, which would +violate the #79161 isolation this scope must preserve. The two resolvers are +intentionally different — do not "deduplicate" them. + +- compression-rotation children walk back to the original segment + (rotation-stable scope — the fix); +- ``/new`` starts a lineage-less session (fresh scope); +- ``/branch`` children (``_branched_from``), delegate subagents + (``_delegate_from``), and tool-tagged children (``source="tool"``) are + explicit fork children and keep their own isolated scope, preserving the + sibling/subagent isolation #79161 established; +- cron fires keep their physical ``cron__`` id here — the per-fire + timestamp is stripped later by ``_cache_scope_from_session_id`` exactly as + before. + +The resolution is memoized per (agent, session_id): the lineage walk runs +once per transcript segment — NOT per API call — and re-runs only when +rotation actually changes ``agent.session_id`` (per the no-DB-on-the-hot-path +constraint recorded on #79017). Default installs compact in place and never +rotate, so they hit the memo forever and behave byte-identically to before. +""" + +import logging +from typing import Any, Optional + +logger = logging.getLogger(__name__) + +_MEMO_ATTR = "_prompt_cache_scope_memo" + + +def _lineage_root(session_id: str, session_db: Any) -> Optional[str]: + """Return the compression-lineage root of *session_id*, or None. + + Defensive about the DB handle: test doubles and partially constructed + agents can hand back non-list results — anything that is not a non-empty + list/tuple whose first element is a non-empty string is ignored. + """ + if session_db is None: + return None + try: + lineage = session_db.get_compression_lineage(session_id) + except Exception: + logger.debug("prompt-cache scope lineage walk failed", exc_info=True) + return None + if isinstance(lineage, (list, tuple)) and lineage: + root = lineage[0] + if isinstance(root, str) and root: + return root + return None + + +def resolve_prompt_cache_scope(agent: Any) -> str: + """Resolve the rotation-stable cache-scope id for *agent*'s conversation. + + Returns the compression-lineage ROOT of ``agent.session_id`` (the + physical id itself when the session has no compression ancestry, no DB + is attached, or the walk fails). The result is memoized on the agent + keyed by the current session id, so the DB walk happens once per + transcript segment rather than once per API call. + """ + sid = str(getattr(agent, "session_id", None) or "") + if not sid: + return "" + db = getattr(agent, "_session_db", None) + # Memo key includes DB presence: an agent that starts DB-less and gains a + # handle later (run_agent._get_session_db_for_recall lazily attaches one) + # must re-resolve instead of staying pinned to the physical id. + key = (sid, db is not None) + memo = getattr(agent, _MEMO_ATTR, None) + if isinstance(memo, tuple) and len(memo) == 2 and memo[0] == key: + return memo[1] + root = _lineage_root(sid, db) if db is not None else None + scope = root or sid + # Memoize on a successful walk, or when there is no DB to consult at all, + # or when the agent will never persist a row (background-review forks set + # _persist_disabled but still hold a DB handle — without this, every API + # call would re-run the lineage query forever). + # A failed/empty walk on a persisting agent is NOT memoized: falling back + # to the physical id is the correct degraded answer right now (row not + # persisted yet, transient DB error), but pinning it for the whole segment + # would keep the scope wrong after the session row lands. + if ( + root is not None + or db is None + or getattr(agent, "_persist_disabled", False) + ): + try: + setattr(agent, _MEMO_ATTR, (key, scope)) + except Exception: + # Frozen/slotted test doubles — resolution still works, just + # unmemoized. + pass + return scope + + +def resolve_prompt_cache_scope_safe(agent: Any) -> Optional[str]: + """Never-raising variant of :func:`resolve_prompt_cache_scope`. + + Returns None on any failure (or when there is no scope). Consumers treat + None/empty as "fall back to the physical session_id", so a resolution + failure degrades to pre-#79017 behavior instead of blocking the caller — + important at turn_context's call site, where an exception raised inside + the ``set_runtime_main(...)`` argument list would otherwise skip the whole + runtime binding, not just the cache scope. + """ + try: + return resolve_prompt_cache_scope(agent) or None + except Exception: + logger.debug("prompt-cache scope resolution failed", exc_info=True) + return None diff --git a/agent/relay_llm.py b/agent/relay_llm.py index 58a5c1bd5e..7758a2bdcf 100644 --- a/agent/relay_llm.py +++ b/agent/relay_llm.py @@ -897,7 +897,8 @@ def _complete_logical( output["response_model"] = response_model_name lease.host.run_in_session( lease.session, - lease.host.relay.scope.pop, + relay_runtime.pop_relay_scope, + lease.host.relay, handle, output=output, metadata={ diff --git a/agent/relay_runtime.py b/agent/relay_runtime.py index a1af9385ae..003213d87d 100644 --- a/agent/relay_runtime.py +++ b/agent/relay_runtime.py @@ -94,6 +94,40 @@ def _run_bounded_on_exit_thread(fn: Callable[[], Any], timeout: float) -> Any: return result[0] if result else None +def pop_relay_scope( + relay: Any, + handle: Any, + *, + output: Any = None, + metadata: Any = None, + timestamp: Any = None, +) -> Any: + """Pop a Relay scope without passing kwargs the binding rejects. + + NeMo Relay ``scope.pop`` gained ``metadata`` in 0.4+. Older wheels (e.g. + 0.3.x) raise ``TypeError: pop() got an unexpected keyword argument + 'metadata'`` when Hermes finalization forwards runtime metadata. Filter to + parameters the live binding accepts so turn/session close can complete. + """ + pop = relay.scope.pop + kwargs: dict[str, Any] = {} + if output is not None: + kwargs["output"] = output + if metadata is not None: + kwargs["metadata"] = metadata + if timestamp is not None: + kwargs["timestamp"] = timestamp + try: + params = inspect.signature(pop).parameters + except (TypeError, ValueError): + params = {} + if params and not any( + param.kind == inspect.Parameter.VAR_KEYWORD for param in params.values() + ): + kwargs = {key: value for key, value in kwargs.items() if key in params} + return pop(handle, **kwargs) + + @dataclass class RelaySession: """One isolated Relay scope stack owned by a Hermes session.""" @@ -543,6 +577,148 @@ class RelayRuntime: ) return result if isinstance(result, dict) else args + def _close_scope_handle( + self, + session: RelaySession, + handle: Any, + *, + output: dict[str, Any] | None = None, + allow_closing: bool = False, + failure_label: str = "scope close failed", + drain_limit: int = 32, + ) -> str | None: + """Pop ``handle``, draining orphaned children in the same session context. + + Relay scopes are strict LIFO. Empty-stream retries + interrupt can + abandon a physical LLM scope above TURN/SESSION (#81521). Drain and + close must run inside one ``run_in_session`` callback so ContextVar + stack views stay consistent across pops. + """ + if handle is None: + return None + metadata = { + RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, + RUNTIME_INSTANCE_KEY: self.runtime_id, + } + close_output = output or {} + session_root = session.handle + drained_holder = {"count": 0} + error_holder: dict[str, BaseException] = {} + + def close_with_drain() -> None: + def current_top() -> Any: + # Version-correct accessor first: the pinned nemo-relay + # binding exposes ``scope.get_handle()`` returning the + # current top-of-stack ScopeHandle. Its + # ``get_scope_stack()`` returns a native ScopeStack object + # that ``scope.pop`` rejects with TypeError, so it must + # never be treated as a handle (#81601 review). + get_handle = getattr( + getattr(self.relay, "scope", None), "get_handle", None + ) + if callable(get_handle): + try: + return get_handle() + except Exception: + pass + top = self.relay.get_scope_stack() + # Some Relay builds return the live stack (list). Others + # return the top handle directly — including tuple handles + # like ("scope", name, serial) from the test fake. Only + # unwrap real list stacks; never index a handle tuple. + if isinstance(top, list): + return top[-1] if top else None + return top + + def same_handle(a: Any, b: Any) -> bool: + # Native ScopeHandle instances do not implement __eq__ by + # value — two handles for the same scope compare unequal — + # so compare by uuid when both sides expose one. + if a is None or b is None: + return a is b + if a is b or a == b: + return True + a_uuid = getattr(a, "uuid", None) + b_uuid = getattr(b, "uuid", None) + return a_uuid is not None and a_uuid == b_uuid + + try: + pop_relay_scope( + self.relay, + handle, + output=close_output, + metadata=metadata, + ) + return + except Exception as first_exc: + error_holder["first"] = first_exc + + for _ in range(drain_limit): + top = current_top() + if top is None or same_handle(top, handle): + break + # Never pop the session root while draining for a nested handle. + if ( + session_root is not None + and same_handle(top, session_root) + and handle is not session_root + ): + break + try: + pop_relay_scope( + self.relay, + top, + output={ + "outcome": "cancelled", + "hermes.orphan_drain": True, + }, + metadata=metadata, + ) + drained_holder["count"] += 1 + except Exception as drain_exc: + error_holder["drain"] = drain_exc + logger.warning( + "Hermes Relay orphaned scope drain failed", + exc_info=True, + ) + break + + if drained_holder["count"]: + logger.warning( + "Hermes Relay drained %d orphaned scope(s) before closing %s", + drained_holder["count"], + handle, + ) + try: + pop_relay_scope( + self.relay, + handle, + output=close_output, + metadata=metadata, + ) + error_holder.pop("first", None) + error_holder.pop("drain", None) + except Exception as retry_exc: + error_holder["retry"] = retry_exc + + try: + self.run_in_session( + session, + close_with_drain, + allow_closing=allow_closing, + # Bound the whole drain+close like the direct pops it + # replaced: a wedged native pipeline must cost at most one + # span, never block turn/session completion (see + # tests/agent/test_relay_runtime_bounded_scope_ops.py). + timeout=_SCOPE_OP_TIMEOUT, + ) + except Exception as exc: + return f"{failure_label}: {exc}" + retry_exc = error_holder.get("retry") or error_holder.get("first") + if retry_exc is not None: + return f"{failure_label}: {retry_exc}" + return None + def close_session(self, event: dict[str, Any]) -> None: """Close one session scope and remove it from the core registry.""" session_id = _session_id(event) @@ -559,21 +735,15 @@ class RelayRuntime: return session.closing = True if session.handle is not None: - try: - self.run_in_session( - session, - self.relay.scope.pop, - session.handle, - output={}, - metadata={ - RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, - RUNTIME_INSTANCE_KEY: self.runtime_id, - }, - allow_closing=True, - timeout=_SCOPE_OP_TIMEOUT, - ) - except Exception as exc: - failures.append(f"session scope close failed: {exc}") + failure = self._close_scope_handle( + session, + session.handle, + output={}, + allow_closing=True, + failure_label="session scope close failed", + ) + if failure: + failures.append(failure) try: try: _scope_op_executor().submit( @@ -964,21 +1134,16 @@ class RelaySessionCoordinator: if isinstance(lease.host, RelayRuntime) and lease.session is not None: self._finish_logical_calls(turn, outcome=outcome) if turn.handle is not None: - try: - lease.host.run_in_session( - lease.session, - lease.host.relay.scope.pop, - turn.handle, - output={"outcome": outcome}, - metadata={ - RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, - RUNTIME_INSTANCE_KEY: lease.host.runtime_id, - }, - timeout=_SCOPE_OP_TIMEOUT, - ) - except Exception: + failure = lease.host._close_scope_handle( + lease.session, + turn.handle, + output={"outcome": outcome}, + failure_label="turn scope close failed", + ) + if failure: logger.warning( - "Hermes Relay turn finalization failed", exc_info=True + "Hermes Relay turn finalization failed: %s", + failure, ) finally: try: @@ -1153,35 +1318,28 @@ class RelaySessionCoordinator: turn.logical_llm_calls.clear() for index in range(len(logical_calls) - 1, -1, -1): request_id, logical_handle = logical_calls[index] - try: - lease.host.run_in_session( - lease.session, - lease.host.relay.scope.pop, - logical_handle, - output={"outcome": outcome}, - metadata={ - RUNTIME_SCHEMA_KEY: RUNTIME_SCHEMA_VERSION, - RUNTIME_INSTANCE_KEY: lease.host.runtime_id, - }, - timeout=_SCOPE_OP_TIMEOUT, - ) - except Exception: - with turn.logical_llm_lock: - # Relay scopes are stack-owned. If the newest remaining - # handle cannot close, older handles cannot close safely - # either, so retain the unclosed prefix for diagnostics. - for pending_request_id, pending_handle in logical_calls[ - : index + 1 - ]: - turn.logical_llm_calls.setdefault( - pending_request_id, - pending_handle, - ) - logger.warning( - "Hermes Relay logical LLM finalization failed", - exc_info=True, - ) - break + failure = lease.host._close_scope_handle( + lease.session, + logical_handle, + output={"outcome": outcome}, + failure_label="logical LLM scope close failed", + ) + if failure is None: + continue + with turn.logical_llm_lock: + # Relay scopes are stack-owned. If the newest remaining + # handle cannot close even after orphan drain, older + # handles cannot close safely either — retain the + # unclosed prefix for diagnostics (#81521). + for pending_request_id, pending_handle in logical_calls[ + : index + 1 + ]: + turn.logical_llm_calls.setdefault( + pending_request_id, + pending_handle, + ) + logger.warning("Hermes Relay logical LLM finalization failed: %s", failure) + break @staticmethod def _reset_turn_context(turn: RelayTurnContext) -> None: diff --git a/agent/system_prompt.py b/agent/system_prompt.py index c15989e16e..a61ff7668c 100644 --- a/agent/system_prompt.py +++ b/agent/system_prompt.py @@ -50,7 +50,8 @@ from agent.prompt_builder import ( drain_truncation_warnings, ) from agent.runtime_cwd import resolve_context_cwd -from hermes_constants import get_hermes_home +from hermes_constants import get_default_hermes_root, get_hermes_home +from pathlib import Path from utils import is_truthy_value logger = logging.getLogger(__name__) @@ -162,9 +163,16 @@ def _plugin_session_info(agent: Any) -> Dict[str, str]: except Exception: cwd = "" try: - from hermes_cli.profiles import get_active_profile_name + # Prefer the agent's own home (override-aware, session_db fallback) — + # ambient get_active_profile_name() misreports on threads that lost + # the HERMES_HOME ContextVar (#86313 class; plugin half per @helix4u). + _home = _agent_home(agent) + if _home is not None: + profile_name = _profile_name_for_home(_home) + else: + from hermes_cli.profiles import get_active_profile_name - profile_name = str(get_active_profile_name() or "default") + profile_name = str(get_active_profile_name() or "default") except Exception: profile_name = "default" return { @@ -262,6 +270,71 @@ def _plugin_section_blocks(sections: tuple, position: str) -> List[str]: return [block] if block else [] +def _agent_home(agent: Any) -> Optional[Path]: + """The agent's OWN profile home. + + Resolution order: + + 1. A bound HERMES_HOME ContextVar override wins. Surfaces that multiplex + several profiles over ONE shared session DB (the messaging gateway: + ``gateway/run.py`` hands every agent the launch-home ``state.db`` and + binds the profile home per turn via ``_profile_runtime_scope`` + + ``copy_context``) would otherwise have the db-derived launch home + STOMP the correctly-bound profile — inverting the leak this helper + exists to fix (found by @kshitijk4poor's post-merge probe on #86313). + 2. Fallback: the home containing the agent's ``_session_db.db_path`` + (``/state.db``) — ground truth on threads that lost the + ContextVar (ContextVars don't propagate into ``threading.Thread``), + where the unbound build previously fell back to the launch home and + leaked the default profile's skills/identity into a bot prompt. + + Returns None when neither resolves so callers fall back to ambient. + """ + try: + from hermes_constants import get_hermes_home_override + + override = get_hermes_home_override() + if override: + return Path(override) + except Exception: + pass + try: + db = getattr(agent, "_session_db", None) + db_path = getattr(db, "db_path", None) + if db_path: + return Path(db_path).parent + except Exception: + pass + return None + + +def _agent_skills_dir(agent: Any) -> Optional[Path]: + """The agent's own ``/skills`` dir, or None to use ambient home.""" + home = _agent_home(agent) + return (home / "skills") if home is not None else None + + +def _profile_name_for_home(home: Path) -> str: + """Derive the profile name for an explicit agent home. + + ``/profiles/X`` -> ``"X"``; anything else -> ``"default"``. + + Uses :func:`get_default_hermes_root` (NOT ``get_hermes_home()``): on a + correctly bound profile session the ambient home IS the profile dir, so + ``get_hermes_home()/profiles`` would never contain ``home`` and every + profile would misreport as "default". + """ + try: + from hermes_constants import get_default_hermes_root + + root = get_default_hermes_root() + rel = home.resolve().relative_to((root / "profiles").resolve()) + return rel.parts[0] if rel.parts else "default" + except (ValueError, OSError): + # Home IS the root (default profile) or unrelatable -> default. + return "default" + + def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) -> Dict[str, str]: """Assemble the system prompt as three ordered cache tiers. @@ -304,7 +377,10 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) # cwd project instructions disabled. _soul_loaded = False if agent.load_soul_identity or not agent.skip_context_files: - _soul_content = _r.load_soul_md(_ctx_len) + # Scope the SOUL.md read to the agent's OWN home (see _agent_home) — + # ambient resolution on a thread that lost the HERMES_HOME ContextVar + # reads the launch profile's SOUL.md instead (#50233). + _soul_content = _r.load_soul_md(_ctx_len, home_override=_agent_home(agent)) if _soul_content: stable_parts.append(_soul_content) _soul_loaded = True @@ -435,6 +511,7 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) available_tools=agent.valid_tool_names, available_toolsets=avail_toolsets, compact_categories=_compact_cats or None, + skills_dir_override=_agent_skills_dir(agent), ) else: skills_prompt = "" @@ -514,26 +591,57 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) # mid-session, so this doesn't break the prompt cache. # See file_safety._resolve_active_profile_name + classify_cross_profile_target # for the matching tool-side guard. + # + # Resolve from the agent's OWN home first (its session_db path), not the + # ambient HERMES_HOME: on a build thread that lost the ContextVar this + # line would otherwise print "default" for a bot profile — the same + # thread-fallback bug that leaked default's skills index. + _agent_home_path = _agent_home(agent) + active_profile = "default" try: - from agent.file_safety import _resolve_active_profile_name - active_profile = _resolve_active_profile_name() + if _agent_home_path is not None: + active_profile = _profile_name_for_home(_agent_home_path) + else: + from agent.file_safety import _resolve_active_profile_name + active_profile = _resolve_active_profile_name() except Exception: active_profile = "default" + # Home string for the message text: prefer the agent's own home so the + # paths named match the profile just resolved. When we have an explicit + # agent home, the root (where the default profile's data lives) comes + # from get_default_hermes_root(): get_hermes_home() on a bound profile + # session is the PROFILE dir, which would misname the default profile's + # paths. Without an agent home, keep the ambient resolution byte-identical + # to the legacy behavior (and patchable via this module's get_hermes_home). + if _agent_home_path is not None: + _home_str = str(_agent_home_path) + _root_str = str(get_default_hermes_root()) + else: + _home_str = _root_str = str(get_hermes_home()) if active_profile == "default": post_workspace_parts.append( "Active Hermes profile: default. Other profiles (if any) live " - "under " + str(get_hermes_home()) + "/profiles//. Each profile has its own " + "under " + _root_str + "/profiles//. Each profile has its own " "skills/, plugins/, cron/, and memories/ that affect a different " "session than this one. Do not modify another profile's " "skills/plugins/cron/memories unless the user explicitly directs " "you to." ) else: + # A non-default name is only ever returned when the resolved home is + # ALREADY /profiles/ — that is exactly how both + # _profile_name_for_home() and _resolve_active_profile_name() derive + # it. So the profile home is the session home itself; appending + # /profiles/ again doubled it (#72894). The default profile's + # data sits at the ROOT (get_default_hermes_root()), which in ambient + # profile mode is NOT get_hermes_home(). + profile_home = _home_str + default_root = get_default_hermes_root() post_workspace_parts.append( f"Active Hermes profile: {active_profile}. This session reads " - f"and writes {get_hermes_home()}/profiles/{active_profile}/. The default " - f"profile's data lives at {get_hermes_home()}/skills/, {get_hermes_home()}/plugins/, " - f"{get_hermes_home()}/cron/, {get_hermes_home()}/memories/ — those belong to a " + f"and writes {profile_home}/. The default " + f"profile's data lives at {default_root}/skills/, {default_root}/plugins/, " + f"{default_root}/cron/, {default_root}/memories/ — those belong to a " f"different session run from a different shell. Do NOT modify " f"another profile's skills/plugins/cron/memories unless the user " f"explicitly directs you to. The cross-profile write guard will " @@ -612,7 +720,8 @@ def build_system_prompt_parts(agent: Any, system_message: Optional[str] = None) context_files_prompt = _r.build_context_files_prompt( cwd=resolve_context_cwd(), skip_soul=_soul_loaded, context_length=_ctx_len, - allow_install_tree_fallback=agent.platform in ("cli", "tui")) + allow_install_tree_fallback=agent.platform in ("cli", "tui"), + home_override=_agent_home(agent)) if context_files_prompt: context_parts.append(context_files_prompt) diff --git a/agent/thread_scoped_output.py b/agent/thread_scoped_output.py index e46608e492..3c4a7be891 100644 --- a/agent/thread_scoped_output.py +++ b/agent/thread_scoped_output.py @@ -30,6 +30,20 @@ _install_lock = threading.Lock() # Maps the proxy we installed for a given attribute ("stdout"/"stderr") so we # never double-wrap and so we can recover the original stream. _installed: dict[str, "_ThreadRoutingStream"] = {} +# One process-lifetime sink per stream. Temporary process-global redirects can +# displace and later restore a routing proxy; they must not allocate another +# permanent /dev/null descriptor every time that happens. +_sinks: dict[str, TextIO] = {} +_routing_states: dict[str, "_RoutingState"] = {} + + +class _RoutingState: + """Silencing registry shared by every proxy generation for one stream.""" + + def __init__(self, sink: TextIO) -> None: + self.sink = sink + self.silenced: dict[int, int] = {} + self.lock = threading.Lock() class _ThreadRoutingStream: @@ -42,32 +56,27 @@ class _ThreadRoutingStream: ``.fileno()`` behave like the underlying stream for the calling thread. """ - def __init__(self, passthrough: TextIO, sink: TextIO) -> None: + def __init__(self, passthrough: TextIO, state: _RoutingState) -> None: self._passthrough = passthrough - self._sink = sink - # ident -> nesting depth. A thread is silenced while depth > 0, so - # nested ``thread_scoped_silence()`` on the same thread composes - # correctly (the inner exit decrements rather than fully clearing). - self._silenced: dict[int, int] = {} - self._lock = threading.Lock() + self._state = state def _target(self) -> TextIO: - if self._silenced.get(threading.get_ident(), 0) > 0: - return self._sink + if self._state.silenced.get(threading.get_ident(), 0) > 0: + return self._state.sink return self._passthrough # --- registration ----------------------------------------------------- def silence(self, ident: int) -> None: - with self._lock: - self._silenced[ident] = self._silenced.get(ident, 0) + 1 + with self._state.lock: + self._state.silenced[ident] = self._state.silenced.get(ident, 0) + 1 def unsilence(self, ident: int) -> None: - with self._lock: - depth = self._silenced.get(ident, 0) - 1 + with self._state.lock: + depth = self._state.silenced.get(ident, 0) - 1 if depth > 0: - self._silenced[ident] = depth + self._state.silenced[ident] = depth else: - self._silenced.pop(ident, None) + self._state.silenced.pop(ident, None) # --- file-like surface ------------------------------------------------ def write(self, data): # type: ignore[no-untyped-def] @@ -109,14 +118,28 @@ def _ensure_installed(attr: str, passthrough: TextIO) -> "_ThreadRoutingStream": with _install_lock: proxy = _installed.get(attr) current = getattr(sys, attr, None) + if isinstance(current, _ThreadRoutingStream): + # A redirect context can restore an older routing proxy after a + # temporary replacement. Adopt it instead of wrapping it and + # growing an unbounded proxy chain. + _installed[attr] = current + _routing_states[attr] = current._state + return current if proxy is not None and current is proxy: return proxy # Capture whatever is currently bound as the passthrough. If a prior # global redirect_stdout is active, route non-silenced threads to that # stream to preserve the old behavior. passthrough = current if current is not None else passthrough - sink = open(os.devnull, "w", encoding="utf-8") - proxy = _ThreadRoutingStream(passthrough, sink) + sink = _sinks.get(attr) + if sink is None or sink.closed: + sink = open(os.devnull, "w", encoding="utf-8") + _sinks[attr] = sink + state = _routing_states.get(attr) + if state is None or state.sink is not sink: + state = _RoutingState(sink) + _routing_states[attr] = state + proxy = _ThreadRoutingStream(passthrough, state) setattr(sys, attr, proxy) _installed[attr] = proxy return proxy diff --git a/agent/tool_dispatch_helpers.py b/agent/tool_dispatch_helpers.py index 6e76e52081..af0970accb 100644 --- a/agent/tool_dispatch_helpers.py +++ b/agent/tool_dispatch_helpers.py @@ -32,6 +32,7 @@ import re from pathlib import Path from typing import Any, Dict, List, Optional +from agent.message_metadata import stamp_message_timestamp from agent.tool_result_classification import ( FILE_MUTATING_TOOL_NAMES as _FILE_MUTATING_TOOLS, ) @@ -557,13 +558,13 @@ def make_tool_result_message( callers should compare by value, not by ``is``. """ wrapped = _maybe_wrap_untrusted(name, content) - message = { + message = stamp_message_timestamp({ "role": "tool", "name": name, "tool_name": name, "content": wrapped, "tool_call_id": tool_call_id, - } + }) try: risk_metadata = _tool_output_risk_metadata(name, content) except Exception as exc: diff --git a/agent/tool_executor.py b/agent/tool_executor.py index c1697a6c31..bdf4efc235 100644 --- a/agent/tool_executor.py +++ b/agent/tool_executor.py @@ -33,6 +33,7 @@ from agent.display import ( _detect_tool_failure, ) from agent.tool_dispatch_helpers import ( + _NEVER_PARALLEL_TOOLS, _is_destructive_command, _is_multimodal_tool_result, _multimodal_text_summary, @@ -158,21 +159,19 @@ def _parse_tool_arguments(raw_arguments: Any) -> tuple[dict, Optional[str]]: def _resolve_concurrent_tool_timeout() -> float | None: - raw = os.getenv("HERMES_CONCURRENT_TOOL_TIMEOUT_S", "").strip() - if not raw: - return _DEFAULT_CONCURRENT_TOOL_TIMEOUT_S - try: - value = float(raw) - except ValueError: - logger.warning( - "invalid HERMES_CONCURRENT_TOOL_TIMEOUT_S=%r; using %.0fs", - raw, - _DEFAULT_CONCURRENT_TOOL_TIMEOUT_S, - ) - return _DEFAULT_CONCURRENT_TOOL_TIMEOUT_S - if value <= 0: - return None - return value + """Resolve the per-batch concurrent tool deadline. + + Delegates to the unified resolver (#85125): ``timeouts.tools.concurrent_batch`` + in config.yaml wins, the legacy ``HERMES_CONCURRENT_TOOL_TIMEOUT_S`` env var + remains the back-compat bridge, and ``0``/negative still disables the bound. + """ + from agent.deadline import resolve_timeout + + return resolve_timeout( + "tools.concurrent_batch", + default=_DEFAULT_CONCURRENT_TOOL_TIMEOUT_S, + env_var="HERMES_CONCURRENT_TOOL_TIMEOUT_S", + ) def _flush_session_db_after_tool_progress( @@ -389,6 +388,10 @@ class _ManagedToolResult: dispatched: bool +class _ToolTimeoutResult(str): + """Marker for a synthesized sequential-tool timeout result.""" + + class _ConcurrentToolAuthorizationGate: """Serialize policy prompts and exclude human approval waits from batch deadlines. @@ -480,6 +483,50 @@ def _managed_values( ) +# Cadence for the in-flight tool activity heartbeat. Must stay far below the +# gateway turn-inactivity timeout (default 1800s) so a silent-but-healthy +# tool call never looks idle to the watchdog. +_TOOL_ACTIVITY_HEARTBEAT_INTERVAL_S = 30.0 + + +def _run_tool_activity_heartbeat( + agent, + stop_event: threading.Event, + label: str, + interval: float = _TOOL_ACTIVITY_HEARTBEAT_INTERVAL_S, +) -> None: + """Refresh the agent's activity clock while a tool call is in flight. + + The gateway's turn-inactivity watchdog + (``gateway/run.py::_watch_gateway_turn_inactivity``) abandons a turn + once ``seconds_since_activity`` exceeds the inactivity timeout + (default 30 min). Activity is stamped when a tool *starts* and when it + *completes*, but a tool call that runs silently for 30+ minutes + (quiet builds, long pytest suites, large downloads, network waits that + emit no output) previously froze the clock at "executing tool: " + and the watchdog hard-abandoned a turn that was still making progress, + reaping the tool's processes mid-execution. + + This daemon thread touches ``agent._touch_activity`` every ``interval`` + seconds until ``stop_event`` is set (the tool call returned), so the + gateway keeps seeing a live turn for the whole duration of the call. + + A tool that truly hangs is still bounded by the tool layer's own + timeouts (terminal ``timeout`` default 180s, the concurrent batch + deadline ~420s), so the heartbeat only extends the turn's life for as + long as the tool call is legitimately executing — it does not unbind + wedged tools. The 30-min gateway backstop remains for turns whose + agent loop itself stalls (no API call, no tool call in flight). + """ + + try: + while not stop_event.wait(interval): + agent._touch_activity(label) + except Exception: + # A heartbeat must never break the agent loop. + pass + + def _run_agent_tool_execution_middleware( agent, *, @@ -608,7 +655,27 @@ def _run_agent_tool_execution_middleware( agent._iters_since_skill = 0 _advance_start_order(_begin) - return execute(final_args) + + # Keep the gateway turn-inactivity watchdog from abandoning a turn + # whose tool call runs silently for longer than the inactivity + # timeout (#84491): stamp activity periodically while the tool is + # in flight, not just at start/completion. Both the sequential and + # the concurrent paths funnel through here, so a single heartbeat + # covers every tool. + _hb_stop = threading.Event() + _hb_thread = threading.Thread( + target=_run_tool_activity_heartbeat, + args=(agent, _hb_stop, f"tool running: {function_name}"), + kwargs={"interval": _TOOL_ACTIVITY_HEARTBEAT_INTERVAL_S}, + daemon=True, + name=f"tool-activity-hb-{function_name[:24]}", + ) + _hb_thread.start() + try: + return execute(final_args) + finally: + _hb_stop.set() + _hb_thread.join(timeout=2.0) def _hermes_pipeline(relay_args: dict[str, Any]) -> Any: request_result = apply_tool_request_middleware( @@ -663,6 +730,146 @@ def _run_agent_tool_execution_middleware( ) +def _resolve_sequential_tool_timeout() -> float | None: + """Deadline for one sequential tool call (#85125 Phase 2a). + + ``timeouts.tools.sequential_call`` in config.yaml wins; when unset, the + sequential path inherits the concurrent batch deadline (same value, same + ``HERMES_CONCURRENT_TOOL_TIMEOUT_S`` legacy bridge) so the two executor + paths cannot drift apart by default. ``0``/negative disables the bound. + + NOTE: this path deliberately does NOT use ``agent.deadline.run_bounded_sync``. + The sequential/concurrent executors extend their deadline dynamically while + a human approval prompt is open (``_ConcurrentToolAuthorizationGate`` + excluded seconds — a MUST-preserve invariant) and touch agent activity + mid-wait; the shared primitive is fixed-deadline by design. Simpler call + sites migrate onto the primitive; these two stay symmetric with each other. + """ + from agent.deadline import resolve_timeout + + return resolve_timeout( + "tools.sequential_call", + default=_resolve_concurrent_tool_timeout(), + ) + + +def _run_sequential_tool_execution_middleware( + agent, + *, + function_name: str, + function_args: dict, + effective_task_id: str, + tool_call_id: str, + execute, + scope_block: str | None = None, + display_index: int | None = None, + middleware_trace: list[dict[str, Any]] | None = None, +) -> _ManagedToolResult: + """Run one sequential call with the concurrent executor's deadline. + + Interactive input tools such as ``clarify`` wait on a human. Their own + timeout (``agent.clarify_timeout``: default 3600s, or unlimited when + ``<= 0``) owns that wait. Applying the generic tool deadline here would + return ``tool_timeout`` while the prompt and worker stay active. + """ + timeout_s = _resolve_sequential_tool_timeout() + kwargs = { + "function_name": function_name, + "function_args": function_args, + "effective_task_id": effective_task_id, + "tool_call_id": tool_call_id, + "execute": execute, + "scope_block": scope_block, + "display_index": display_index, + "middleware_trace": middleware_trace, + } + if timeout_s is None or function_name in _NEVER_PARALLEL_TOOLS: + return _run_agent_tool_execution_middleware(agent, **kwargs) + + from tools.daemon_pool import DaemonThreadPoolExecutor + + authorization_gate = _ConcurrentToolAuthorizationGate() + worker_tid: list[int] = [] + + def _run() -> _ManagedToolResult: + tid = threading.current_thread().ident + worker_tid.append(tid) + with agent._tool_worker_threads_lock: + agent._tool_worker_threads.add(tid) + try: + return _run_agent_tool_execution_middleware( + agent, authorization_gate=authorization_gate, **kwargs + ) + finally: + with agent._tool_worker_threads_lock: + agent._tool_worker_threads.discard(tid) + try: + _ra()._set_interrupt(False, tid) + except Exception: + pass + + executor = DaemonThreadPoolExecutor(max_workers=1) + future = executor.submit(propagate_context_to_thread(_run)) + deadline = time.monotonic() + timeout_s + started = time.monotonic() + timed_out = False + try: + while True: + remaining = ( + deadline + authorization_gate.excluded_seconds() - time.monotonic() + ) + if remaining <= 0: + timed_out = True + break + try: + return future.result(timeout=min(5.0, remaining)) + except concurrent.futures.TimeoutError: + elapsed = int(time.monotonic() - started) + if elapsed > 0 and elapsed % 30 < 5: + agent._touch_activity( + f"sequential tool running ({elapsed}s): {function_name}" + ) + + message = ( + f"Error executing tool '{function_name}': " + f"timed out after {timeout_s:.1f}s" + ) + logger.warning( + "sequential tool %s timed out after %.1fs", function_name, timeout_s + ) + future.cancel() + for tid in worker_tid: + try: + _ra()._set_interrupt(True, tid) + except Exception: + pass + trace = middleware_trace if middleware_trace is not None else [] + _emit_terminal_post_tool_call( + agent, + function_name=function_name, + function_args=function_args, + result=message, + effective_task_id=effective_task_id, + tool_call_id=tool_call_id, + duration_ms=int(timeout_s * 1000), + status="timeout", + error_type="tool_timeout", + error_message=message, + middleware_trace=list(trace), + ) + return _ManagedToolResult( + result=_ToolTimeoutResult(message), + args=function_args, + middleware_trace=trace, + blocked=False, + dispatched=True, + ) + finally: + # Never join a wedged worker. DaemonThreadPoolExecutor also keeps it out + # of the stdlib atexit join, matching the concurrent timeout path. + executor.shutdown(wait=not timed_out, cancel_futures=timed_out) + + def _begin_tool_execution( agent, *, @@ -1610,6 +1817,12 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe """ # Resolve the context-scaled tool-output budget once per turn. _tool_budget = _budget_for_agent(agent) + + # Keep every runtime-tool branch on one bounded execution funnel without + # duplicating timeout policy across the branch-specific callbacks below. + def _run_agent_tool_execution_middleware(agent, **kwargs): + return _run_sequential_tool_execution_middleware(agent, **kwargs) + for i, tool_call in enumerate(assistant_message.tool_calls, 1): if getattr(agent, "_incremental_persistence_failed", False): return @@ -1762,6 +1975,7 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe around_message_id=next_args.get("around_message_id"), window=next_args.get("window", 5), sort=next_args.get("sort"), + detail=next_args.get("detail", "adaptive"), db=session_db, current_session_id=agent.session_id, ) @@ -2048,27 +2262,30 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe _spinner_result = None try: def _execute(next_args: dict) -> Any: - return _ra().handle_function_call( - function_name, - next_args, - effective_task_id, - tool_call_id=tool_call.id, - session_id=agent.session_id or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") - or "", - enabled_tools=( - list(agent.valid_tool_names) - if agent.valid_tool_names - else None - ), - skip_pre_tool_call_hook=True, - skip_tool_request_middleware=True, - skip_tool_execution_middleware=True, - tool_request_middleware_trace=list(middleware_trace), - enabled_toolsets=getattr(agent, "enabled_toolsets", None), - disabled_toolsets=getattr(agent, "disabled_toolsets", None), - ) + from model_tools import suppress_post_tool_call_hook + + with suppress_post_tool_call_hook(): + return _ra().handle_function_call( + function_name, + next_args, + effective_task_id, + tool_call_id=tool_call.id, + session_id=agent.session_id or "", + turn_id=getattr(agent, "_current_turn_id", "") or "", + api_request_id=getattr(agent, "_current_api_request_id", "") + or "", + enabled_tools=( + list(agent.valid_tool_names) + if agent.valid_tool_names + else None + ), + skip_pre_tool_call_hook=True, + skip_tool_request_middleware=True, + skip_tool_execution_middleware=True, + tool_request_middleware_trace=list(middleware_trace), + enabled_toolsets=getattr(agent, "enabled_toolsets", None), + disabled_toolsets=getattr(agent, "disabled_toolsets", None), + ) ( function_result, @@ -2127,27 +2344,30 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe else: try: def _execute(next_args: dict) -> Any: - return _ra().handle_function_call( - function_name, - next_args, - effective_task_id, - tool_call_id=tool_call.id, - session_id=agent.session_id or "", - turn_id=getattr(agent, "_current_turn_id", "") or "", - api_request_id=getattr(agent, "_current_api_request_id", "") - or "", - enabled_tools=( - list(agent.valid_tool_names) - if agent.valid_tool_names - else None - ), - skip_pre_tool_call_hook=True, - skip_tool_request_middleware=True, - skip_tool_execution_middleware=True, - tool_request_middleware_trace=list(middleware_trace), - enabled_toolsets=getattr(agent, "enabled_toolsets", None), - disabled_toolsets=getattr(agent, "disabled_toolsets", None), - ) + from model_tools import suppress_post_tool_call_hook + + with suppress_post_tool_call_hook(): + return _ra().handle_function_call( + function_name, + next_args, + effective_task_id, + tool_call_id=tool_call.id, + session_id=agent.session_id or "", + turn_id=getattr(agent, "_current_turn_id", "") or "", + api_request_id=getattr(agent, "_current_api_request_id", "") + or "", + enabled_tools=( + list(agent.valid_tool_names) + if agent.valid_tool_names + else None + ), + skip_pre_tool_call_hook=True, + skip_tool_request_middleware=True, + skip_tool_execution_middleware=True, + tool_request_middleware_trace=list(middleware_trace), + enabled_toolsets=getattr(agent, "enabled_toolsets", None), + disabled_toolsets=getattr(agent, "disabled_toolsets", None), + ) ( function_result, @@ -2195,6 +2415,7 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe logger.error("handle_function_call raised for %s: %s", function_name, tool_error, exc_info=True) tool_duration = time.time() - tool_start_time + _execution_timed_out = isinstance(function_result, _ToolTimeoutResult) if isinstance(function_result, str): result_preview = function_result if agent.verbose_logging else ( function_result[:200] if len(function_result) > 200 else function_result @@ -2212,15 +2433,12 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe # context-engine, memory-manager, clarify, delegate_task) are # dispatched inline — they never reach handle_function_call, so the # executor is the one that has to fire post_tool_call. For - # registry-dispatched tools the else-branch above invoked - # handle_function_call, which already fires the hook. - from agent.agent_runtime_helpers import agent_runtime_owns_post_tool_hook + # Every dispatch suppresses the inner handle_function_call observer so + # the executor owns one terminal event for this tool_call_id. This also + # prevents an abandoned timeout worker from reporting late success. _executor_must_emit_post_hook = ( not _execution_blocked - and ( - not _execution_dispatched - or agent_runtime_owns_post_tool_hook(agent, function_name) - ) + and not _execution_timed_out ) if _executor_must_emit_post_hook: _emit_terminal_post_tool_call( @@ -2289,7 +2507,12 @@ def execute_tool_calls_sequential(agent, assistant_message, messages: list, effe # Unwrap _multimodal dicts to an OpenAI-style content list # (see parallel path for rationale). String results pass through. _tool_content = agent._tool_result_content_for_active_model(function_name, function_result) - tool_message = make_tool_result_message(function_name, _tool_content, tool_call.id) + tool_message = make_tool_result_message( + function_name, + _tool_content, + tool_call.id, + effect_disposition="unknown" if _execution_timed_out else None, + ) messages.append(tool_message) risk_metadata = tool_message.get("_tool_output_risk") if not _flush_session_db_after_tool_progress( diff --git a/agent/trace_upload.py b/agent/trace_upload.py index 404d9be70b..d2c97bdc7f 100644 --- a/agent/trace_upload.py +++ b/agent/trace_upload.py @@ -336,10 +336,16 @@ def load_session_messages( """ from hermes_state import SessionDB db = SessionDB(db_path=db_path) if db_path else SessionDB() - resolved = db.resolve_session_id(session_id) or session_id - meta = db.get_session(resolved) or {} - messages = db.get_messages_as_conversation(resolved) - return messages, meta + try: + resolved = db.resolve_session_id(session_id) or session_id + meta = db.get_session(resolved) or {} + messages = db.get_messages_as_conversation(resolved) + return messages, meta + finally: + try: + db.close() + except Exception: + logger.debug("Failed to close trace-upload SessionDB", exc_info=True) def upload_session_trace( diff --git a/agent/transports/chat_completions.py b/agent/transports/chat_completions.py index 275a66db32..f6f0cd5184 100644 --- a/agent/transports/chat_completions.py +++ b/agent/transports/chat_completions.py @@ -48,8 +48,15 @@ def _add_prompt_cache_key( tools: list[dict[str, Any]] | None, supports_prompt_cache_key: bool, session_id: str | None = None, + cache_scope_id: str | None = None, ) -> None: - """Add a content-addressed key only for an explicitly capable endpoint.""" + """Add a content-addressed key only for an explicitly capable endpoint. + + ``cache_scope_id``, when provided, is the rotation-stable logical scope + (compression-lineage root — agent/prompt_cache_scope.py) and takes + precedence over the physical ``session_id`` so the key survives + context-compression session rotation (#79017). + """ if not supports_prompt_cache_key: return @@ -70,7 +77,7 @@ def _add_prompt_cache_key( cache_key = _content_cache_key( _static_prompt_instructions(messages), tools, - _cache_scope_from_session_id(session_id), + _cache_scope_from_session_id(cache_scope_id or session_id), ) if cache_key: api_kwargs["prompt_cache_key"] = cache_key @@ -272,6 +279,25 @@ class ChatCompletionsTransport(ProviderTransport): break tool_calls = msg.get("tool_calls") if isinstance(tool_calls, list): + # Defense-in-depth: a strict OpenAI-compatible provider + # (e.g. onerouter / Qwen, DeepSeek v4) rejects an assistant + # message carrying ``tool_calls: []`` (empty array) with + # HTTP 400 "Empty tool_calls is not supported in message." + # The pre-API sanitizer in agent_runtime_helpers drops these, + # but only on the conversation_loop path — other routes can + # reach the wire without it. For every request that + # serializes through this transport (conversation loop and + # any caller using it), this is the last boundary, so + # normalize here. Requests built by fully separate payload + # paths (e.g. some auxiliary clients) never pass through + # this layer and are out of scope for it. (#58755 follow-up) + if ( + msg.get("role") == "assistant" + and "tool_calls" in msg + and not tool_calls + ): + needs_sanitize = True + break for tc in tool_calls: if isinstance(tc, dict) and ( "call_id" in tc @@ -282,6 +308,15 @@ class ChatCompletionsTransport(ProviderTransport): break if needs_sanitize: break + elif ( + isinstance(tool_calls, type(None)) + and msg.get("role") == "assistant" + and "tool_calls" in msg + ): + # Explicit ``tool_calls: null`` is equally invalid on strict + # providers — treat it like the empty-array case. + needs_sanitize = True + break if not needs_sanitize: return messages @@ -328,6 +363,19 @@ class ChatCompletionsTransport(ProviderTransport): tool_calls = msg.get("tool_calls") if isinstance(tool_calls, list): + # Strip empty/invalid tool_calls arrays at the transport + # layer (see detection above). Strict OpenAI-compatible + # providers reject ``tool_calls: []`` with HTTP 400; dropping + # the key keeps the message schema-valid. Matches the + # pre-API sanitizer's behaviour so all routes agree. + if ( + msg.get("role") == "assistant" + and "tool_calls" in msg + and not tool_calls + ): + out_msg = mutable_msg() + out_msg.pop("tool_calls", None) + continue copied_tool_calls: list[Any] | None = None for tc_idx, tc in enumerate(tool_calls): if isinstance(tc, dict): @@ -347,6 +395,14 @@ class ChatCompletionsTransport(ProviderTransport): copied_tool_calls[tc_idx] = copied_tc if copied_tool_calls is not None: mutable_msg()["tool_calls"] = copied_tool_calls + elif ( + isinstance(tool_calls, type(None)) + and msg.get("role") == "assistant" + and "tool_calls" in msg + ): + # Explicit ``tool_calls: null`` is invalid on strict + # providers — drop the key entirely. + mutable_msg().pop("tool_calls", None) return sanitized def convert_tools(self, tools: list[dict[str, Any]]) -> list[dict[str, Any]]: @@ -592,6 +648,7 @@ class ChatCompletionsTransport(ProviderTransport): supports_prompt_cache_key=bool(params.get("supports_prompt_cache_key")) or _is_openai_api_base_url(params.get("base_url")), session_id=params.get("session_id"), + cache_scope_id=params.get("cache_scope_id"), ) return api_kwargs @@ -742,6 +799,7 @@ class ChatCompletionsTransport(ProviderTransport): tools=api_kwargs.get("tools"), supports_prompt_cache_key=bool(getattr(profile, "supports_prompt_cache_key", False)), session_id=params.get("session_id"), + cache_scope_id=params.get("cache_scope_id"), ) return api_kwargs diff --git a/agent/transports/codex.py b/agent/transports/codex.py index 2676dd1fc0..bf73ba971f 100644 --- a/agent/transports/codex.py +++ b/agent/transports/codex.py @@ -173,6 +173,111 @@ def _content_cache_key( return f"pck_{digest}" +def _is_azure_foundry_responses(params: Dict[str, Any]) -> bool: + """Return True for Microsoft Foundry's OpenAI-compatible Responses API. + + Matched on the registered provider id first, then on the endpoint host. + Host matching goes through ``base_url_host_matches`` rather than a + substring test, so a path or query segment carrying the Foundry domain + (``https://proxy.example.com/.services.ai.azure.com/v1``) is not + misclassified as Foundry. + """ + from utils import base_url_host_matches + + provider = str(params.get("provider") or "").strip().lower() + if provider == "azure-foundry": + return True + + return base_url_host_matches( + str(params.get("base_url") or ""), "services.ai.azure.com" + ) + + +def _is_post_tool_replay(messages: Optional[List[Dict[str, Any]]]) -> bool: + """Return True when ``messages`` end on a tool result awaiting a follow-up. + + Azure Foundry only rejects the *post-tool follow-up* payload — the shape + where a prior assistant ``function_call`` and its ``function_call_output`` + are replayed alongside an encrypted ``reasoning`` item (HTTP 400 + invalid_payload). Detecting that shape here keeps reasoning suppression + scoped to the failing turn, so ordinary (non-tool) Foundry multi-turn + continuity is left unchanged. + + The test is on the *trailing* messages, not on the history as a whole. + Scanning the whole history for any tool call plus any tool result makes + the predicate sticky: one tool call early in a conversation would then + suppress reasoning on every later turn, including plain user follow-ups + that Foundry accepts. The rejected payload is specifically the turn whose + last item is a tool result, so that is what this matches: the final + non-system message is a ``tool`` result, and the assistant message that + issued its ``tool_call_id`` is present. + + Tool-call identity is resolved the same way + ``_chat_messages_to_responses_input`` resolves it, because the pairing + that matters is the one that reaches the wire. A stored tool call can + carry the function call id in ``call_id``, in ``id``, or in a composite + ``"call_x|fc_y"`` id, and a bare ``fc_``-prefixed ``id`` is a response + item id that the converter turns into ``call_``. Matching only + ``id`` would miss the ``id=fc_… / call_id=call_…`` shape that resumed + legacy sessions and host-fed histories still use, and let the rejected + payload through. + """ + from agent.codex_responses_adapter import _split_responses_tool_id + + def _pair_ids(raw: Any, explicit: Any = None) -> set: + """Every call id a stored tool id could pair on, converter-order.""" + embedded_call_id, item_id = _split_responses_tool_id(raw) + ids = {embedded_call_id} if embedded_call_id else set() + if isinstance(explicit, str) and explicit.strip(): + ids.add(explicit.strip()) + if not ids and isinstance(raw, str) and raw.strip(): + ids.add(raw.strip()) + if isinstance(item_id, str) and item_id.startswith("fc_") and item_id[3:]: + ids.add(f"call_{item_id[3:]}") + return ids + + trailing = set() + for msg in reversed(messages or ()): + if not isinstance(msg, dict): + return False + role = msg.get("role") + if role == "system": + continue + if role == "tool": + ids = _pair_ids(msg.get("tool_call_id")) + if not ids: + return False + trailing |= ids + continue + # First message before the trailing run of tool results. It must be + # the assistant turn that issued them for this to be the follow-up + # payload; a non-empty ``trailing`` is what proves the run existed. + if role != "assistant": + return False + return any( + trailing & _pair_ids(call.get("id"), call.get("call_id")) + for call in msg.get("tool_calls") or [] + if isinstance(call, dict) + ) + + return False + + +def _native_compaction_active(context_management: Any) -> bool: + """Is THIS request natively compacted? + + True only when the caller's eligibility gate + (``native_compaction.native_compaction_context_management``) produced a + non-empty payload. Every native-compaction side effect on the wire — + sending ``context_management``, replaying a ``type: "compaction"`` + checkpoint, restructuring the input around it — hangs off this one + predicate, so a checkpoint that outlives the gate (model swapped out of + the gpt-5.6 family, compression disabled, rejection kill switch, resumed + session) cannot keep reshaping requests on its own. + """ + return isinstance(context_management, list) and bool(context_management) + + class ResponsesApiTransport(ProviderTransport): """Transport for api_mode='codex_responses'. @@ -213,6 +318,9 @@ class ResponsesApiTransport(ProviderTransport): kwargs.get("replay_encrypted_reasoning", True) ), current_issuer_kind=issuer, + native_compaction_eligible=_native_compaction_active( + kwargs.get("context_management") + ), ) def convert_tools(self, tools: List[Dict[str, Any]]) -> Any: @@ -234,10 +342,16 @@ class ResponsesApiTransport(ProviderTransport): params: instructions: str — system prompt (extracted from messages[0] if not given) reasoning_config: dict | None — {effort, enabled} - session_id: str | None — transcript/session id; drives the xAI - x-grok-conv-id header and the Codex cache-scope headers, and is - the fallback prompt_cache_key when there is no static prefix to - content-address + session_id: str | None — transcript/session id; drives the Codex + ``session_id`` header, and is the cache-scope fallback when no + ``cache_scope_id`` is given + cache_scope_id: str | None — rotation-stable logical scope id + (compression-lineage root; see agent/prompt_cache_scope.py). + Preferred over session_id when deriving the prompt_cache_key + content hash and the xAI x-grok-conv-id header; the Codex + x-client-request-id header mirrors the resulting body key. + Keeps the cache warm across context-compression session + rotation (#79017) max_tokens: int | None — max_output_tokens timeout: float | None — per-request timeout forwarded to the SDK request_overrides: dict | None — extra kwargs merged in @@ -271,11 +385,28 @@ class ResponsesApiTransport(ProviderTransport): replay_encrypted_reasoning = bool( params.get("replay_encrypted_reasoning", True) ) + if replay_encrypted_reasoning and _is_azure_foundry_responses(params): + # Microsoft Foundry accepts the initial Responses function-call + # request and ordinary (non-tool) multi-turn continuity, but + # rejects the post-tool follow-up payload that carries prior + # encrypted reasoning items alongside function_call / + # function_call_output, with HTTP 400 invalid_payload. Scope the + # suppression to that follow-up turn: keep function_call / + # function_call_output continuity intact and drop only the + # encrypted reasoning replay for this endpoint. + if _is_post_tool_replay(payload_messages): + replay_encrypted_reasoning = False # Native server-side compaction (gpt-5.6 on direct OpenAI/Codex routes # only). The caller resolves eligibility via # agent.native_compaction.native_compaction_context_management(); # None means the field is never added to the request. context_management = params.get("context_management") + # Single source of truth for "this request is natively compacted": + # the same value decides whether the field goes out AND whether the + # converter may replay/prune around a compaction checkpoint. Keeping + # them derived from one expression is what stops a persisted + # checkpoint from restructuring the wire after the gate closes. + native_compaction_active = _native_compaction_active(context_management) # Resolve the issuing endpoint for this call. Stashed on the # transport so normalize_response can stamp it onto reasoning @@ -370,6 +501,7 @@ class ResponsesApiTransport(ProviderTransport): is_github_responses=is_github_responses, replay_encrypted_reasoning=replay_encrypted_reasoning, current_issuer_kind=issuer_kind, + native_compaction_eligible=native_compaction_active, ), "store": False, } @@ -377,7 +509,7 @@ class ResponsesApiTransport(ProviderTransport): kwargs["tools"] = response_tools kwargs["tool_choice"] = "auto" kwargs["parallel_tool_calls"] = True - if isinstance(context_management, list) and context_management: + if native_compaction_active: kwargs["context_management"] = context_management session_id = params.get("session_id") @@ -386,10 +518,18 @@ class ResponsesApiTransport(ProviderTransport): # recurring cron jobs carry a per-fire timestamp in session_id # (cron__) that made every run cache-cold, so the scope strips # that suffix (see _cache_scope_from_session_id). session_id is left - # untouched for transcript isolation and the cache-scope routing - # headers below. Falls back to session_id when there is no static - # content to hash. - _cache_scope = _cache_scope_from_session_id(session_id) + # untouched for transcript isolation (the Codex ``session_id`` header + # below). Falls back to session_id when there is no static content to + # hash. + # + # cache_scope_id, when provided, is the rotation-stable logical scope + # (compression-lineage root — agent/prompt_cache_scope.py): legacy + # ``compression.in_place: false`` compaction rotates session_id + # mid-conversation, and scoping by the physical id went cache-cold at + # every rotation boundary (#79017). + _cache_scope = _cache_scope_from_session_id( + params.get("cache_scope_id") or session_id + ) cache_key = _content_cache_key( instructions, response_tools, _cache_scope ) or _cache_scope diff --git a/agent/turn_context.py b/agent/turn_context.py index eef1ecc8d8..84ef44779e 100644 --- a/agent/turn_context.py +++ b/agent/turn_context.py @@ -42,6 +42,7 @@ from agent.context_engine import automatic_compaction_status_message from agent.iteration_budget import IterationBudget from agent.memory_manager import build_memory_context_block from agent.memory_provider import is_trivial_prompt +from agent.message_metadata import append_message, stamp_message_timestamp from agent.model_metadata import ( estimate_messages_tokens_rough, estimate_request_tokens_rough, @@ -493,6 +494,17 @@ def build_turn_context( # after primary restoration has settled the runtime. try: from agent.auxiliary_client import set_runtime_main + from agent.prompt_cache_scope import resolve_prompt_cache_scope_safe + # Rotation-stable prompt-cache scope. Memoized per segment on the + # agent, so this is a DB walk at most once per segment — except a + # brand-new session whose row lands later in turn setup + # (_ensure_db_session); that first turn falls back to the physical + # id here and the first build_api_kwargs re-resolves. Stays valid + # through a mid-turn compression rotation because the lineage root + # is by definition rotation-invariant (#79017). Resolved with the + # never-raising variant OUTSIDE the argument list, so a resolution + # failure can only lose the scope — never the whole runtime binding. + _cache_scope = resolve_prompt_cache_scope_safe(agent) or "" set_runtime_main( getattr(agent, "provider", "") or "", getattr(agent, "model", "") or "", @@ -502,6 +514,7 @@ def build_turn_context( api_mode=getattr(agent, "api_mode", "") or "", auth_mode=getattr(agent, "auth_mode", "") or "", session_id=getattr(agent, "session_id", "") or "", + cache_scope=_cache_scope, ) except Exception: pass @@ -633,9 +646,15 @@ def build_turn_context( # the same dict and any close-path durable marker. user_msg["content"] = user_message else: - user_msg = {"role": "user", "content": user_message} + user_msg = stamp_message_timestamp( + {"role": "user", "content": user_message}, + timestamp=persist_user_timestamp, + ) if isinstance(pending_cli_message, dict): agent._pending_cli_user_message = None + # CLI input is stamped when staged. Gateway input may carry the platform + # event time. Preserve either value and cover any legacy unstamped handoff. + stamp_message_timestamp(user_msg, timestamp=persist_user_timestamp) # Hydrate todo store from conversation history. if conversation_history and not agent._todo_store.has_items(): @@ -667,7 +686,7 @@ def build_turn_context( if persist_user_display_metadata: user_msg["display_metadata"] = persist_user_display_metadata - messages.append(user_msg) + append_message(messages, user_msg) current_turn_user_idx = len(messages) - 1 agent._persist_user_message_idx = current_turn_user_idx diff --git a/agent/turn_finalizer.py b/agent/turn_finalizer.py index de31bbabd4..402e1b6a5d 100644 --- a/agent/turn_finalizer.py +++ b/agent/turn_finalizer.py @@ -26,6 +26,7 @@ import os from agent.codex_responses_adapter import _summarize_user_message_for_log from agent.message_content import flatten_message_text +from agent.message_metadata import append_message, stamp_message_timestamp from agent.message_sanitization import _sanitize_surrogates @@ -315,7 +316,10 @@ def finalize_turn( if _tail_role != "assistant": # Tail is not an assistant row — append the final response # so the durable turn closes with the answer (#43849/#44100). - messages.append({"role": "assistant", "content": final_response}) + append_message( + messages, + {"role": "assistant", "content": final_response}, + ) elif isinstance(_tail, dict) and _tail.get("content") != final_response and _is_pure_tool_call_tail(_tail): # The tail IS an assistant row, but a *pure tool-call turn*: # tool_calls with no text of its own. The role check alone @@ -332,6 +336,10 @@ def finalize_turn( # candidate collapse — the provisional answer was persisted and # reused as the terminal response, #65919 §7). _tail["content"] = final_response + # The normal assistant builder already stamps this row. Cover + # legacy/exceptional pure-tool tails before they become a + # delivered final response. + stamp_message_timestamp(_tail) # The row may have already been flushed to SQLite by the # incremental tool-call persist (conversation_loop.py:4990), # which stamps ``_DB_PERSISTED_MARKER`` so subsequent flushes @@ -699,8 +707,8 @@ def finalize_turn( "health (`hermes doctor`), then send your message again" ) # Machine-readable cause for the gateway/desktop: exactly - # 'session_persistence_failed:'. Never clobber a - # failure_reason another path already stamped on this result. + # 'session_persistence_failed:'. + # Never clobber a failure_reason another path already stamped. if "failure_reason" not in result: _cause = getattr(agent, "_last_persistence_error_cause", None) result["failure_reason"] = ( diff --git a/agent/usage_pricing.py b/agent/usage_pricing.py index 592f6742b6..c955bdc7f0 100644 --- a/agent/usage_pricing.py +++ b/agent/usage_pricing.py @@ -8,7 +8,7 @@ from decimal import Decimal from typing import Any, Dict, Literal, Optional from agent.model_metadata import fetch_endpoint_model_metadata, fetch_model_metadata -from utils import base_url_host_matches +from utils import base_url_host_matches, base_url_hostname logger = logging.getLogger(__name__) @@ -18,6 +18,46 @@ _ZERO = Decimal("0") _ONE_MILLION = Decimal("1000000") _NOUS_DEFAULT_BASE_URL = "https://inference-api.nousresearch.com/v1" +# Sub-cent cost threshold: below $0.01, render at 4 decimal places so +# the display is non-zero (e.g. $0.0046 instead of $0.00). See #79220. +_SUBCENT_THRESHOLD = Decimal("0.01") + +# Attached to every CostResult with status="included" so consumers can +# distinguish "free because subscription" from "free because $0 pricing". +_INCLUDED_NOTE = "subscription-included; no provider invoice for usage" + + +def format_cost_label(amount: Decimal) -> str: + """Format a cost amount as a display label. + + Scales precision to magnitude: + - Zero → "$0.00" + - Sub-cent (< $0.01) → "~$0.0046" (4 dp; amounts that ROUND to + 0.0000 at 4 dp — i.e. at or below $0.00005 under banker's + rounding — fall back to "~$<0.0001" so the label never reads + as zero) + - Normal → "~$1.23" (2 dp) + + This fixes #79220 where sub-cent per-turn costs on cheap models + (DeepSeek, etc.) rendered as "$0.00" despite amount_usd carrying + full Decimal precision. + + Shared by per-response cost labels (estimate_usage_cost) and the + insights cost-bucket formatters — keep both surfaces on this one + implementation so sub-cent honesty can't regress on one of them. + """ + if amount == _ZERO: + return "$0.00" + if amount < _SUBCENT_THRESHOLD: + label = f"~${amount:.4f}" + # A positive amount that rounds to 0.0000 at 4 dp would render + # "~$0.0000" — a zero-looking label, the exact #79220 dishonesty. + # Comparing the rendered label checks the truth directly (a naive + # `< 0.00005` threshold misses the exact boundary under + # ROUND_HALF_EVEN). + return label if label != "~$0.0000" else "~$<0.0001" + return f"~${amount:.2f}" + CostStatus = Literal["actual", "estimated", "included", "unknown"] CostSource = Literal[ "provider_cost_api", @@ -990,6 +1030,29 @@ def _to_int(value: Any) -> int: return 0 +def _usage_get(obj: Any, name: str, default: Any = 0) -> Any: + """Read a field from a usage object that may be a dict or an attribute object. + + The Responses API can return usage as either a typed SDK object (accessible + via ``getattr``) or a plain ``dict`` (from JSON deserialisation). Using + ``getattr`` on a dict silently yields the default, zeroing out all token + counts. This helper normalises access so both shapes work transparently. + """ + if isinstance(obj, dict): + return obj.get(name, default) + return getattr(obj, name, default) + + +def _usage_count(value: Any) -> int: + """Coerce a usage counter to a non-negative integer. + + Providers occasionally emit malformed negative counters; clamp them to 0 + so a bad field cannot corrupt session accounting (#85706). + """ + return max(0, _to_int(value)) + + + def resolve_billing_route( model_name: str, provider: Optional[str] = None, @@ -1035,7 +1098,7 @@ def resolve_billing_route( # Fireworks model ids look like accounts/fireworks/models/; # rsplit("/", 1)[-1] yields just which is what the dict keys on. return BillingRoute(provider="fireworks", model=model.rsplit("/", 1)[-1], base_url=base_url or "", billing_mode="official_docs_snapshot") - if provider_name in {"custom", "local"} or (base and "localhost" in base): + if provider_name in {"custom", "local"} or (base and base_url_hostname(base) in ("localhost", "127.0.0.1")): return BillingRoute(provider=provider_name or "custom", model=model, base_url=base_url or "", billing_mode="unknown") return BillingRoute(provider=provider_name or "unknown", model=model.split("/")[-1] if model else "", base_url=base_url or "", billing_mode="unknown") @@ -1229,47 +1292,89 @@ def normalize_usage( mode = (api_mode or "").strip().lower() if mode == "anthropic_messages" or provider_name == "anthropic": - input_tokens = _to_int(getattr(response_usage, "input_tokens", 0)) - output_tokens = _to_int(getattr(response_usage, "output_tokens", 0)) - cache_read_tokens = _to_int(getattr(response_usage, "cache_read_input_tokens", 0)) - cache_write_tokens = _to_int(getattr(response_usage, "cache_creation_input_tokens", 0)) - elif mode == "codex_responses": - input_total = _to_int(getattr(response_usage, "input_tokens", 0)) - output_tokens = _to_int(getattr(response_usage, "output_tokens", 0)) - details = getattr(response_usage, "input_tokens_details", None) - cache_read_tokens = _to_int(getattr(details, "cached_tokens", 0) if details else 0) - cache_write_tokens = _to_int( - getattr(details, "cache_creation_tokens", 0) if details else 0 + input_tokens = _usage_count(_usage_get(response_usage, "input_tokens", 0)) + output_tokens = _usage_count(_usage_get(response_usage, "output_tokens", 0)) + cache_read_tokens = _usage_count(_usage_get(response_usage, "cache_read_input_tokens", 0)) + cache_write_tokens = _usage_count( + _usage_get(response_usage, "cache_creation_input_tokens", 0) ) + elif mode == "codex_responses": + input_total = _usage_count(_usage_get(response_usage, "input_tokens", 0)) + output_tokens = _usage_count(_usage_get(response_usage, "output_tokens", 0)) + details = _usage_get(response_usage, "input_tokens_details", None) + cache_read_tokens = _usage_count( + _usage_get(details, "cached_tokens", 0) if details else 0 + ) + # OpenAI's documented field for GPT-5.6+ explicit cache writes is + # `cache_write_tokens` (billed at 1.25x); `cache_creation_tokens` is + # kept as a fallback for older/alternate Responses-compatible + # endpoints (#70543). + cache_write_tokens = _usage_count( + _usage_get(details, "cache_write_tokens", 0) if details else 0 + ) + if not cache_write_tokens: + cache_write_tokens = _usage_count( + _usage_get(details, "cache_creation_tokens", 0) if details else 0 + ) input_tokens = max(0, input_total - cache_read_tokens - cache_write_tokens) else: - prompt_total = _to_int(getattr(response_usage, "prompt_tokens", 0)) - output_tokens = _to_int(getattr(response_usage, "completion_tokens", 0)) - details = getattr(response_usage, "prompt_tokens_details", None) + # OpenAI-style names first; fall back to Anthropic-style + # (input_tokens/output_tokens). Local OpenAI-compatible servers like + # mlx_vlm.server emit the Anthropic names in chat_completions responses, + # and the OpenAI Python client preserves them as extra attributes. + prompt_total = _usage_count( + _usage_get(response_usage, "prompt_tokens", 0) + ) or _usage_count(_usage_get(response_usage, "input_tokens", 0)) + output_tokens = _usage_count( + _usage_get(response_usage, "completion_tokens", 0) + ) or _usage_count(_usage_get(response_usage, "output_tokens", 0)) + details = _usage_get(response_usage, "prompt_tokens_details", None) # Primary: OpenAI-style prompt_tokens_details. Fallback: Anthropic-style # top-level fields that some OpenAI-compatible proxies (OpenRouter, Vercel # AI Gateway, Cline) expose when routing Claude models — without this # fallback, cache writes are undercounted as 0 and cache reads can be # missed when the proxy only surfaces them at the top level. # Port of cline/cline#10266. - cache_read_tokens = _to_int(getattr(details, "cached_tokens", 0) if details else 0) + cache_read_tokens = _usage_count( + _usage_get(details, "cached_tokens", 0) if details else 0 + ) if not cache_read_tokens: - cache_read_tokens = _to_int(getattr(response_usage, "cache_read_input_tokens", 0)) + cache_read_tokens = _usage_count( + _usage_get(response_usage, "cache_read_input_tokens", 0) + ) if not cache_read_tokens: # DeepSeek's native API (api.deepseek.com) reports context-cache # hits as top-level prompt_cache_hit_tokens (+ the complementary # prompt_cache_miss_tokens; prompt_tokens = hit + miss), not the # OpenAI nested shape. Without this, direct DeepSeek sessions # always showed 0 cache-hit tokens (#61871). - cache_read_tokens = _to_int( - getattr(response_usage, "prompt_cache_hit_tokens", 0) + cache_read_tokens = _usage_count( + _usage_get(response_usage, "prompt_cache_hit_tokens", 0) ) - cache_write_tokens = _to_int( - getattr(details, "cache_write_tokens", 0) if details else 0 + if not cache_read_tokens: + # Kimi/Moonshot's native API (api.moonshot.cn / .ai) reports + # context-cache hits as a top-level usage.cached_tokens, not the + # OpenAI nested prompt_tokens_details.cached_tokens shape. Without + # this, direct Kimi sessions always showed 0 cache-hit tokens and + # the hits were billed at the full input rate (#65722). + cache_read_tokens = _usage_count( + _usage_get(response_usage, "cached_tokens", 0) + ) + cache_write_tokens = _usage_count( + _usage_get(details, "cache_write_tokens", 0) if details else 0 ) if not cache_write_tokens: - cache_write_tokens = _to_int( - getattr(response_usage, "cache_creation_input_tokens", 0) + cache_write_tokens = _usage_count( + _usage_get(details, "cache_creation_input_tokens", 0) + if details else 0 + ) + if not cache_write_tokens: + cache_write_tokens = _usage_count( + _usage_get(response_usage, "cache_creation_input_tokens", 0) + ) + if not cache_write_tokens: + cache_write_tokens = _usage_count( + _usage_get(response_usage, "cache_write_tokens", 0) ) input_tokens = max(0, prompt_total - cache_read_tokens - cache_write_tokens) @@ -1281,14 +1386,14 @@ def normalize_usage( # hidden thinking was invisible in session accounting even though it # dominates output spend on models like deepseek-v4-flash (measured: # single calls burning 21K reasoning tokens to emit 500 visible tokens). - output_details = getattr(response_usage, "output_tokens_details", None) + output_details = _usage_get(response_usage, "output_tokens_details", None) if output_details: - reasoning_tokens = _to_int(getattr(output_details, "reasoning_tokens", 0)) + reasoning_tokens = _usage_count(_usage_get(output_details, "reasoning_tokens", 0)) if not reasoning_tokens: - completion_details = getattr(response_usage, "completion_tokens_details", None) + completion_details = _usage_get(response_usage, "completion_tokens_details", None) if completion_details: - reasoning_tokens = _to_int( - getattr(completion_details, "reasoning_tokens", 0) + reasoning_tokens = _usage_count( + _usage_get(completion_details, "reasoning_tokens", 0) ) # Cache observability for MiniMax's Anthropic wire: on MiniMax-M3, @@ -1334,6 +1439,7 @@ def estimate_usage_cost( source="none", label="included", pricing_version="included-route", + notes=(_INCLUDED_NOTE,), ) entry = get_pricing_entry(model_name, provider=provider, base_url=base_url, api_key=api_key) @@ -1378,10 +1484,11 @@ def estimate_usage_cost( amount += Decimal(usage.request_count) * entry.request_cost status: CostStatus = "estimated" - label = f"~${amount:.2f}" + label = format_cost_label(amount) if entry.source == "none" and amount == _ZERO: status = "included" label = "included" + notes.append(_INCLUDED_NOTE) if route.provider == "openrouter": notes.append("OpenRouter cost is estimated from the models API until reconciled.") diff --git a/apps/desktop/e2e/mock-server.ts b/apps/desktop/e2e/mock-server.ts index ce4665d177..8de1af8aa4 100644 --- a/apps/desktop/e2e/mock-server.ts +++ b/apps/desktop/e2e/mock-server.ts @@ -120,6 +120,9 @@ let _correctionSwitchIndex = 0 /** Per-server counter for the verify-on-stop script. */ let _verificationStopIndex = 0 +/** Per-server counter for the task-panel warm-resume script. */ +let _taskPanelResumeIndex = 0 + /** User messages received by the mock, for E2E assertions on real submits. */ const _receivedUserTexts: string[] = [] @@ -131,6 +134,7 @@ function resetScriptIndex(): void { _queueStopIndex = 0 _correctionSwitchIndex = 0 _verificationStopIndex = 0 + _taskPanelResumeIndex = 0 _receivedUserTexts.length = 0 } @@ -295,6 +299,41 @@ export const VERIFICATION_STOP_TEXT = 'I cannot provide fresh verification evide export const BLOCKING_CLARIFY_TRIGGER = 'E2E_BLOCKING_CLARIFY_TRIGGER' export const BLOCKING_CLARIFY_QUESTION = 'Keep this test turn running?' +/** + * A long live response with a five-row todo card, held open by a foreground tool. + * The transcript is deliberately taller than the viewport so warm-session + * tests can detect when re-opening the session leaves it above the true bottom. + */ +export const TASK_PANEL_RESUME_TRIGGER = 'E2E_TASK_PANEL_RESUME_TRIGGER' +export const TASK_PANEL_RESUME_TEXT = Array.from( + { length: 24 }, + (_, index) => `Task-panel clearance line ${index + 1}: inspect the restored working session geometry.`, +).join('\n\n') + +const TASK_PANEL_RESUME_SCRIPT: ScriptedTurn[] = [ + { + text: TASK_PANEL_RESUME_TEXT, + toolCalls: [ + { + name: 'todo', + args: { + todos: [ + { id: 'design', content: 'Design the restored layout', status: 'completed' }, + { id: 'implement', content: 'Implement the measured clearance', status: 'in_progress' }, + { id: 'verify', content: 'Verify the latest message stays visible', status: 'pending' }, + { id: 'review', content: 'Review the visual regression', status: 'pending' }, + { id: 'ship', content: 'Ship the focused fix', status: 'pending' }, + ], + }, + }, + { + name: 'terminal', + args: { command: 'sleep 60' }, + }, + ], + }, +] + const BLOCKING_CLARIFY_TURN: ScriptedTurn = { text: '', toolCalls: [{ name: 'clarify', args: { question: BLOCKING_CLARIFY_QUESTION, choices: ['Yes', 'No'] } }], @@ -414,6 +453,7 @@ export function startMockServer(options: MockServerOptions = {}): Promise typeof message?.content === 'string' && message.content.includes(VERIFICATION_STOP_TRIGGER), ) @@ -421,6 +461,29 @@ export function startMockServer(options: MockServerOptions = {}): Promise typeof message?.content === 'string' && message.content.includes(CORRECTION_SWITCH_TRIGGER), ) + if (isTaskPanelResumeTrigger) { + const turn = + TASK_PANEL_RESUME_SCRIPT[_taskPanelResumeIndex] ?? + TASK_PANEL_RESUME_SCRIPT[TASK_PANEL_RESUME_SCRIPT.length - 1] + _taskPanelResumeIndex++ + const respond = () => { + if (stream) { + streamScriptedTurn(res, model, turn) + } else { + nonStreamingScriptedTurn(res, model, turn) + } + } + + if (holdThisCompletion) { + heldCompletionCount++ + resolveHeldStreamStarted?.() + void heldStreamReleased.then(respond) + } else { + respond() + } + return + } + if (includesBlockingClarifyTrigger(parsed.messages)) { if (stream) { streamScriptedTurn(res, model, BLOCKING_CLARIFY_TURN) diff --git a/apps/desktop/e2e/sidebar-states.spec.ts b/apps/desktop/e2e/sidebar-states.spec.ts index 6d8c0c2c9c..29e12debd2 100644 --- a/apps/desktop/e2e/sidebar-states.spec.ts +++ b/apps/desktop/e2e/sidebar-states.spec.ts @@ -25,6 +25,8 @@ import { /** Background-running dot aria-label (from i18n en.ts). */ const BG_DOT_LABEL = 'Background task running' +/** Foreground turn-running dot aria-label. */ +const SESSION_RUNNING_DOT_LABEL = 'Session running' /** Finished-unread dot aria-label. */ const UNREAD_DOT_LABEL = 'Finished — unread' @@ -219,12 +221,20 @@ test.describe('sidebar states — cross-session dot transition', () => { ) .toBeGreaterThan(0) - // Wait for the final answer (turn completes, but bg process still running). + // The final answer text streams before message.complete, so text visibility + // alone is not a completion barrier. Wait for the foreground-running state + // to clear before asserting the background-process state. await page.waitForFunction( (text) => (document.body.textContent ?? '').includes(text), SIDEBAR_CROSS_TEXTS.finalText, { timeout: 90_000 }, ) + await expect + .poll( + () => page.locator(`[aria-label="${SESSION_RUNNING_DOT_LABEL}"]`).count(), + { timeout: 30_000, message: 'session running dot should disappear after turn completes' }, + ) + .toBe(0) // The background dot must still be visible: the turn is done but the // process is held open by the sentinel, so this is a stable state rather diff --git a/apps/desktop/e2e/task-panel-clearance.spec.ts b/apps/desktop/e2e/task-panel-clearance.spec.ts new file mode 100644 index 0000000000..19cf8d9541 --- /dev/null +++ b/apps/desktop/e2e/task-panel-clearance.spec.ts @@ -0,0 +1,145 @@ +/** + * Regression coverage for returning to a working session as its task panel + * expands. The transcript must reconcile to the composer's full measured + * height without needing a manual scroll to repair the position. + */ + +import { expect, test, type Page } from './test' + +import { type MockBackendFixture, setupMockBackend, waitForAppReady } from './fixtures' +import { TASK_PANEL_RESUME_TRIGGER } from './mock-server' + +const SURFACE = '[data-composer-target]:visible' +const PROMPT = `${TASK_PANEL_RESUME_TRIGGER}: keep the task panel expanded while this session is reopened.` + +function activeSurface(page: Page) { + return page.locator(SURFACE).last() +} + +async function send(page: Page, text: string): Promise { + const composer = activeSurface(page).locator('[contenteditable="true"]').first() + + await composer.waitFor({ state: 'visible', timeout: 15_000 }) + await composer.click() + await composer.type(text, { delay: 5 }) + await page.keyboard.press('Enter') +} + +async function openFreshDraft(page: Page): Promise { + await page.locator('[data-slot="sidebar"] button[aria-label="New session"]').first().click() + await expect(activeSurface(page).locator('[data-slot="aui_thread-viewport"]')).not.toContainText(PROMPT) + await page.waitForTimeout(1_000) +} + +async function reopenWorkingSession(page: Page): Promise { + const sidebar = page.locator('[data-slot="sidebar"]') + const row = sidebar.getByRole('button', { name: /^(?:Session running|Needs your input|Working)\b/ }).first() + + await row.waitFor({ state: 'visible', timeout: 30_000 }) + await row.click() + await expect(activeSurface(page).locator('[data-slot="aui_thread-viewport"]')).toContainText( + 'Task-panel clearance line 24', + { timeout: 30_000 }, + ) +} + +interface ClearanceMetrics { + composerHeight: number + distanceFromBottom: number + latestMessageBottom: number + statusPanelTop: number + viewportHeight: number +} + +async function clearanceMetrics(page: Page): Promise { + return activeSurface(page).evaluate(surface => { + const chatSurface = surface.closest('[data-chat-surface]')! + const viewport = surface.querySelector('[data-slot="aui_thread-viewport"]')! + const latest = Array.from(surface.querySelectorAll('[data-role="assistant"]')).at(-1)! + const status = surface.querySelector('[data-slot="composer-status-stack"]')! + const styles = getComputedStyle(chatSurface) + + return { + composerHeight: Number.parseFloat(styles.getPropertyValue('--composer-measured-height')), + distanceFromBottom: viewport.scrollHeight - viewport.clientHeight - viewport.scrollTop, + latestMessageBottom: latest.getBoundingClientRect().bottom, + statusPanelTop: status.getBoundingClientRect().top, + viewportHeight: viewport.clientHeight, + } + }) +} + +test.describe('working-session task-panel clearance', () => { + let fixture: MockBackendFixture | null = null + + test.beforeEach(async () => { + fixture = await setupMockBackend({ + mockServer: { holdFirstCompletionContaining: TASK_PANEL_RESUME_TRIGGER }, + }) + await waitForAppReady(fixture, 120_000) + }) + + test.afterEach(async () => { + await fixture?.cleanup() + fixture = null + }) + + test('window focus reanchors a working session above the expanded task panel', async ({}, testInfo) => { + const page = fixture!.page + + await send(page, PROMPT) + await fixture!.mock.waitForHeldCompletion() + await openFreshDraft(page) + + // Re-open while the long response is still streaming. Its todo call lands + // afterward, so the already-visible composer grows only after the initial + // session-load scroll settle has finished. + fixture!.mock.releaseHeldStream() + await page.waitForTimeout(1_000) + await reopenWorkingSession(page) + await expect(activeSurface(page).getByText('Tasks 1/5')).toBeVisible({ timeout: 30_000 }) + + // Reproduce the stale geometry at the foreground boundary. Active turns + // disable Chromium's background throttling, so visibility can stay `visible` + // and window focus is the only foreground edge that can repair it. + await page.waitForTimeout(750) + const staleState = await activeSurface(page) + .locator('[data-slot="aui_thread-viewport"]') + .evaluate(viewport => { + // Grow scrollHeight before the observed thread-content node. This + // shifts the transcript behind the dock without resizing the observed + // node or synthesizing a user scroll (which must escape the lock). + const staleClearance = document.createElement('div') + staleClearance.style.height = '160px' + staleClearance.setAttribute('aria-hidden', 'true') + viewport.prepend(staleClearance) + + const distance = viewport.scrollHeight - viewport.clientHeight - viewport.scrollTop + const surface = viewport.closest('[data-composer-target]')! + const latest = Array.from(surface.querySelectorAll('[data-role="assistant"]')).at(-1)! + const status = surface.querySelector('[data-slot="composer-status-stack"]')! + + window.dispatchEvent(new Event('focus')) + + return { + distance, + following: viewport.dataset.following, + latestMessageBottom: latest.getBoundingClientRect().bottom, + statusPanelTop: status.getBoundingClientRect().top, + visibility: document.visibilityState, + } + }) + + expect(staleState.visibility, JSON.stringify(staleState)).toBe('visible') + expect(staleState.following, JSON.stringify(staleState)).toBe('true') + expect(staleState.distance).toBeGreaterThan(100) + expect(staleState.latestMessageBottom, JSON.stringify(staleState)).toBeGreaterThan(staleState.statusPanelTop) + await page.waitForTimeout(1_000) + const metrics = await clearanceMetrics(page) + await page.screenshot({ path: testInfo.outputPath('task-panel-after-resume.png') }) + + expect(metrics.composerHeight, JSON.stringify(metrics)).toBeGreaterThanOrEqual(190) + expect(metrics.distanceFromBottom, JSON.stringify(metrics)).toBeLessThan(staleState.distance / 2) + expect(metrics.latestMessageBottom, JSON.stringify(metrics)).toBeLessThanOrEqual(metrics.statusPanelTop) + }) +}) diff --git a/apps/desktop/e2e/tile-unread-bug.spec.ts b/apps/desktop/e2e/tile-unread-bug.spec.ts index fa614a21d2..7a676798f5 100644 --- a/apps/desktop/e2e/tile-unread-bug.spec.ts +++ b/apps/desktop/e2e/tile-unread-bug.spec.ts @@ -33,6 +33,8 @@ import { const UNREAD_DOT_LABEL = 'Finished — unread' /** Background-running dot aria-label. */ const BG_DOT_LABEL = 'Background task running' +/** Foreground turn-running dot aria-label. */ +const SESSION_RUNNING_DOT_LABEL = 'Session running' /** Locate a session's sidebar row by its preview text. */ function sessionRow(page: import('@playwright/test').Page, text: string) { @@ -65,12 +67,20 @@ async function startTurnAndSwitchAway(page: import('@playwright/test').Page) { ) .toBeGreaterThan(0) - // Wait for the turn to complete (final answer visible). + // The final answer text streams before message.complete, so text visibility + // alone is not a completion barrier. Wait for the foreground-running state + // to clear before asserting the background-process state. await page.waitForFunction( (text) => (document.body.textContent ?? '').includes(text), SIDEBAR_CROSS_TEXTS.finalText, { timeout: 90_000 }, ) + await expect + .poll( + () => page.locator(`[aria-label="${SESSION_RUNNING_DOT_LABEL}"]`).count(), + { timeout: 30_000, message: 'session running dot should disappear after turn completes' }, + ) + .toBe(0) // The background dot must still be visible: the turn is done but the // process is held open by the sentinel, so this is a stable state rather diff --git a/apps/desktop/e2e/unread-dot-restart.spec.ts b/apps/desktop/e2e/unread-dot-restart.spec.ts new file mode 100644 index 0000000000..9ef8c3d9e2 --- /dev/null +++ b/apps/desktop/e2e/unread-dot-restart.spec.ts @@ -0,0 +1,162 @@ +/** + * E2E test for PERSISTED unread state — the green "Finished — unread" dot + * must survive an app restart. + * + * Regression coverage for the reset-on-restart bug: the unread flag used to + * live only in the in-memory `$unreadFinishedSessionIds` atom, so closing and + * reopening the desktop app grayed out every green dot. The persisted layer + * (src/store/session-unread.ts) now rebuilds the dot from localStorage-backed + * finish markers + seen-count watermarks. + * + * The scenario uses TWO sessions on purpose: with a single session the app + * can reopen straight into it after a restart, which acks the session (the + * user is looking at it) and would mask the dot. Session A finishes in the + * background while session B is the selected one; the restart reopens into + * B, so A's dot is observable. + * + * 1. Session A: start a turn, hold its stream open. + * 2. Session B: new session, send a message, let it finish while SELECTED. + * 3. Release A's stream → its background finish paints A's green dot. + * 4. QUIT the app, relaunch on the SAME sandbox → A's dot must still be + * there (previously it was lost). + * 5. Open session A → dot clears. + * 6. Restart once more → the cleared state must persist too (no zombie dot). + * + * Prerequisite: `npm run build` must have been run so dist/ exists. + */ + +import { expect, type Page, test } from '@playwright/test' +import { type ElectronApplication } from '@playwright/test' + +import { + buildAppEnv, + launchDesktop, + type MockBackendFixture, + setupMockBackend, + waitForAppReady, +} from './fixtures' +import { restartMockServer } from './mock-server' + +/** Finished-unread dot aria-label (from i18n en.ts). */ +const UNREAD_DOT_LABEL = 'Finished — unread' + +/** Held prompt — the mock pauses this stream until we release it, so the + * turn is deterministically still running when we switch away. */ +const HELD_PROMPT = 'E2E unread restart: hold this stream until released.' +/** Second session's prompt — completes normally while selected. */ +const SECOND_PROMPT = 'E2E unread restart: second session, read while open.' + +const unreadDots = (page: Page) => page.locator(`[aria-label="${UNREAD_DOT_LABEL}"]`) + +async function sendMessage(page: Page, text: string): Promise { + const composer = page.locator('[contenteditable="true"]').first() + await composer.waitFor({ state: 'visible', timeout: 15_000 }) + await composer.click() + await composer.type(text, { delay: 5 }) + await page.keyboard.press('Enter') +} + +test.describe('unread dot survives app restart', () => { + test.describe.configure({ mode: 'serial' }) + // Three full app boots in one scenario — give it more than the global 90s. + test.setTimeout(300_000) + + let fixture: MockBackendFixture + let app: ElectronApplication + let page: Page + + test.beforeAll(async () => { + restartMockServer() + fixture = await setupMockBackend({ + mockServer: { holdFirstStreamForPrompt: HELD_PROMPT }, + }) + app = fixture.app + page = fixture.page + await waitForAppReady(fixture, 120_000) + }) + + test.afterAll(async () => { + // The fixture's own app handle may already be closed by the restart + // steps — close whatever is current, then drop the sandbox + mock. + try { + await app?.close() + } catch { + // already closed + } + + fixture?.mock.close() + fixture?.sandbox.cleanup() + }) + + /** Relaunch the desktop app against the SAME sandbox (same userData → + * same localStorage, same HERMES_HOME → same session store). */ + async function restartApp(): Promise { + await app.close() + + const relaunched = await launchDesktop(buildAppEnv(fixture.sandbox)) + app = relaunched.app + page = relaunched.page + await waitForAppReady({ ...fixture, app, page }, 120_000) + } + + test('green dot persists across restart and its clear persists too', async () => { + // ── 1. Session A: start a turn whose stream the mock holds open ──── + await sendMessage(page, HELD_PROMPT) + await fixture.mock.waitForHeldStream() + + // ── 2. Session B: complete a turn while SELECTED (stays read) ────── + await page.locator('button:has-text("New session")').first().click() + await sendMessage(page, SECOND_PROMPT) + + // B's reply lands in the open transcript — this session is "read". + await page.waitForFunction( + () => (document.body.textContent ?? '').includes('Hello from the mock inference server'), + undefined, + { timeout: 30_000 }, + ) + + // ── 3. Release A's stream → background finish paints A's dot ────── + fixture.mock.releaseHeldStream() + + await expect + .poll(() => unreadDots(page).count(), { + timeout: 30_000, + message: 'green unread dot should appear after the background finish', + }) + .toBeGreaterThan(0) + + // ── 4. Restart the app — the dot must survive ────────────────────── + // The app reopens into session B (or a fresh draft), NOT session A, so + // A's dot is observable rather than being acked by the route restore. + await restartApp() + + await expect + .poll(() => unreadDots(page).count(), { + timeout: 60_000, + message: 'green unread dot should be rebuilt from persisted state after restart', + }) + .toBeGreaterThan(0) + + // ── 5. Open session A — the dot clears ───────────────────────────── + // The dot sits inside A's sidebar row button; click that row. + await unreadDots(page) + .first() + .locator('xpath=ancestor::button[1]') + .click() + + await expect + .poll(() => unreadDots(page).count(), { + timeout: 15_000, + message: 'opening the session should clear its unread dot', + }) + .toBe(0) + + // ── 6. Restart again — the CLEARED state must persist as well ────── + await restartApp() + + // Give the sidebar a moment to load rows, then assert no dot returns. + await page.waitForSelector('[data-slot="sidebar"]', { timeout: 60_000 }) + await page.waitForTimeout(5_000) + expect(await unreadDots(page).count(), 'acked session must stay read after restart').toBe(0) + }) +}) diff --git a/apps/desktop/electron/backend-ownership.test.ts b/apps/desktop/electron/backend-ownership.test.ts new file mode 100644 index 0000000000..b2afbae7b0 --- /dev/null +++ b/apps/desktop/electron/backend-ownership.test.ts @@ -0,0 +1,223 @@ +import assert from 'node:assert/strict' + +import { test, vi } from 'vitest' + +import { + backendCommandMatches, + type BackendIdentity, + createBackendOwnership, + createBackendShutdownCoordinator, + parseBackendOwnership +} from './backend-ownership' + +function memoryStore(initial = '') { + let contents = initial + + return { + read: () => contents, + value: () => contents, + write: (next: string) => { + contents = next + } + } +} + +function identity(overrides: Partial = {}): BackendIdentity { + return { + nonce: 'nonce-42', + pid: 42, + profile: 'default', + startMarker: 'os-start-123', + ...overrides + } +} + +function ownershipEntry(overrides: Partial = {}) { + return { command: 'hermes serve --port 0', ...identity(overrides) } +} + +function stored(entries: object[]): string { + return JSON.stringify({ backends: entries }) +} + +function deferred() { + let resolve!: () => void + + const promise = new Promise(done => { + resolve = done + }) + + return { promise, resolve } +} + +function createOwnership(store = memoryStore(), overrides: Partial[0]> = {}) { + return createBackendOwnership({ + matchesIdentity: async () => true, + stop: () => {}, + store, + ...overrides + }) +} + +test('claim persists the caller-supplied exact identity before resolving', async () => { + const store = memoryStore() + const ownership = createOwnership(store) + const claim = ownershipEntry() + + assert.deepEqual(await ownership.claim(claim), claim) + assert.deepEqual(parseBackendOwnership(store.value()), [claim]) +}) + +test('incomplete claims and persisted records are rejected', async () => { + const store = memoryStore( + stored([ + ownershipEntry(), + { ...ownershipEntry({ pid: 43 }), startMarker: '' }, + { ...ownershipEntry({ pid: 44 }), nonce: undefined }, + { ...ownershipEntry({ pid: 45 }), profile: undefined } + ]) + ) + + const ownership = createOwnership(store) + + await assert.rejects(ownership.claim({ ...ownershipEntry(), startMarker: '' }), /complete process identity/) + assert.deepEqual(parseBackendOwnership(store.value()), [ownershipEntry()]) +}) + +test('failed persistence awaits asynchronous cleanup of the exact identity', async () => { + const cleanup = deferred() + const stop = vi.fn(() => cleanup.promise) + const expected = new Error('disk full') + const claim = ownershipEntry({ pid: 43 }) + + const ownership = createOwnership(memoryStore(), { + stop, + store: { + read: () => null, + write: () => { + throw expected + } + } + }) + + let rejected = false + + const result = ownership.claim(claim).catch(error => { + rejected = true + throw error + }) + + await Promise.resolve() + assert.equal(rejected, false) + assert.deepEqual(stop.mock.calls, [[claim]]) + + cleanup.resolve() + await assert.rejects(result, expected) + assert.equal(rejected, true) +}) + +test('startup reap drops a confirmed PID reuse mismatch without stopping it', async () => { + const entry = ownershipEntry() + const store = memoryStore(stored([entry])) + const matchesIdentity = vi.fn(async () => false) + const stop = vi.fn() + const ownership = createOwnership(store, { matchesIdentity, stop }) + + assert.deepEqual(await ownership.reapOrphans(), []) + assert.deepEqual(matchesIdentity.mock.calls, [[entry]]) + assert.equal(stop.mock.calls.length, 0) + assert.deepEqual(parseBackendOwnership(store.value()), []) +}) + +test('startup reap preserves records when exact identity probing is uncertain or fails', async () => { + const uncertain = ownershipEntry({ pid: 50, nonce: 'uncertain' }) + const failed = ownershipEntry({ pid: 51, nonce: 'failed' }) + const store = memoryStore(stored([uncertain, failed])) + const stop = vi.fn() + + const ownership = createOwnership(store, { + matchesIdentity: async entry => { + if (entry.pid === failed.pid) { + throw new Error('process table unavailable') + } + + return undefined + }, + stop + }) + + assert.deepEqual(await ownership.reapOrphans(), []) + assert.equal(stop.mock.calls.length, 0) + assert.deepEqual(parseBackendOwnership(store.value()), [uncertain, failed]) +}) + +test('startup reap passes the full confirmed identity to stop', async () => { + const entry = ownershipEntry({ pid: 52 }) + const store = memoryStore(stored([entry])) + const stop = vi.fn() + const ownership = createOwnership(store, { stop }) + + assert.deepEqual(await ownership.reapOrphans(), [52]) + assert.deepEqual(stop.mock.calls, [[entry]]) + assert.deepEqual(parseBackendOwnership(store.value()), []) +}) + +test('startup reap preserves failed stops for the next launch', async () => { + const entry = ownershipEntry({ pid: 53 }) + const store = memoryStore(stored([entry])) + + const ownership = createOwnership(store, { + stop: () => { + throw new Error('permission denied') + } + }) + + assert.deepEqual(await ownership.reapOrphans(), []) + assert.deepEqual(parseBackendOwnership(store.value()), [entry]) +}) + +test('release removes only the exact identity rather than every record for its PID', () => { + const oldProcess = ownershipEntry({ nonce: 'old', startMarker: 'start-old' }) + const reusedPid = ownershipEntry({ nonce: 'new', startMarker: 'start-new' }) + const store = memoryStore(stored([oldProcess, reusedPid])) + const ownership = createOwnership(store) + + ownership.release(oldProcess) + + assert.deepEqual(parseBackendOwnership(store.value()), [reusedPid]) +}) + +test('backend identity check matches only serve and dashboard invocation shapes', () => { + assert.equal(backendCommandMatches('/venv/bin/hermes serve --port 0'), true) + assert.equal(backendCommandMatches('python -m hermes_cli.main dashboard --no-open'), true) + assert.equal(backendCommandMatches('/venv/bin/hermes --profile work serve --port 0'), true) + assert.equal(backendCommandMatches('"C:\\Hermes Runtime\\hermes.exe" dashboard --no-open'), true) + assert.equal(backendCommandMatches('hermes chat --query serve'), false) + assert.equal(backendCommandMatches('unrelated dashboard'), false) +}) + +test('shutdown coordinator returns one promise and awaits teardown exactly once', async () => { + const completion = deferred() + const teardown = vi.fn(() => completion.promise) + const coordinator = createBackendShutdownCoordinator(teardown) + + const first = coordinator.run() + const second = coordinator.run() + + assert.equal(first, second) + assert.equal(coordinator.hasStarted(), true) + await Promise.resolve() + assert.equal(teardown.mock.calls.length, 1) + + let finished = false + first.then(() => { + finished = true + }) + await Promise.resolve() + assert.equal(finished, false) + + completion.resolve() + await second + assert.equal(finished, true) + assert.equal(coordinator.run(), first) +}) diff --git a/apps/desktop/electron/backend-ownership.ts b/apps/desktop/electron/backend-ownership.ts new file mode 100644 index 0000000000..feb50fb8e6 --- /dev/null +++ b/apps/desktop/electron/backend-ownership.ts @@ -0,0 +1,227 @@ +export interface BackendIdentity { + nonce: string + pid: number + profile: string + startMarker: string +} + +export interface BackendOwnershipEntry extends BackendIdentity { + command?: string +} + +export interface BackendOwnershipStore { + read: () => string | null + write: (contents: string) => void +} + +export interface BackendOwnershipDeps { + matchesIdentity: (identity: BackendIdentity) => Promise + stop: (identity: BackendIdentity) => Promise | void + store: BackendOwnershipStore +} + +export interface BackendClaim extends BackendIdentity { + command?: string +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.length > 0 +} + +function isCompleteIdentity(value: unknown): value is BackendIdentity { + if (!value || typeof value !== 'object') { + return false + } + + const candidate = value as Partial + + return ( + Number.isInteger(candidate.pid) && + Number(candidate.pid) > 0 && + isNonEmptyString(candidate.startMarker) && + isNonEmptyString(candidate.nonce) && + isNonEmptyString(candidate.profile) + ) +} + +function identitiesMatch(left: BackendIdentity, right: BackendIdentity): boolean { + return ( + left.pid === right.pid && + left.startMarker === right.startMarker && + left.nonce === right.nonce && + left.profile === right.profile + ) +} + +export function parseBackendOwnership(contents: unknown): BackendOwnershipEntry[] { + let parsed: unknown + + try { + parsed = JSON.parse(String(contents ?? '')) + } catch { + return [] + } + + const values = Array.isArray(parsed) + ? parsed + : parsed && typeof parsed === 'object' && Array.isArray((parsed as { backends?: unknown }).backends) + ? (parsed as { backends: unknown[] }).backends + : [] + + const entries: BackendOwnershipEntry[] = [] + + for (const value of values) { + if (!isCompleteIdentity(value)) { + continue + } + + const candidate = value as BackendOwnershipEntry + + const entry: BackendOwnershipEntry = { + nonce: candidate.nonce, + pid: candidate.pid, + profile: candidate.profile, + startMarker: candidate.startMarker + } + + if (typeof candidate.command === 'string') { + entry.command = candidate.command + } + + if (!entries.some(existing => identitiesMatch(existing, entry))) { + entries.push(entry) + } + } + + return entries +} + +export function serializeBackendOwnership(entries: BackendOwnershipEntry[]): string { + return `${JSON.stringify({ backends: entries }, null, 2)}\n` +} + +/** + * Persistent ownership for local backend roots. + * + * Claiming is asynchronous so a failed persistence transaction can await child + * cleanup before reporting failure to the caller. + */ +export function createBackendOwnership(deps: BackendOwnershipDeps) { + const read = () => parseBackendOwnership(deps.store.read()) + const write = (entries: BackendOwnershipEntry[]) => deps.store.write(serializeBackendOwnership(entries)) + + return { + async claim(claim: BackendClaim): Promise { + if (!isCompleteIdentity(claim)) { + throw new Error('Cannot own a backend without a complete process identity.') + } + + const entry: BackendOwnershipEntry = { + nonce: claim.nonce, + pid: claim.pid, + profile: claim.profile, + startMarker: claim.startMarker + } + + if (typeof claim.command === 'string') { + entry.command = claim.command + } + + try { + const entries = read().filter(candidate => candidate.pid !== entry.pid) + write([...entries, entry]) + } catch (error) { + try { + await deps.stop(entry) + } catch { + // Persistence remains the claim failure even if cleanup also fails. + } + + throw error + } + + return entry + }, + + release(identity: BackendIdentity): void { + if (!isCompleteIdentity(identity)) { + throw new Error('Cannot release a backend without a complete process identity.') + } + + const entries = read() + const next = entries.filter(entry => !identitiesMatch(entry, identity)) + + if (next.length !== entries.length) { + write(next) + } + }, + + async reapOrphans(): Promise { + const entries = read() + const survivors: BackendOwnershipEntry[] = [] + const reaped: number[] = [] + + for (const entry of entries) { + let matches: boolean | undefined + + try { + matches = await deps.matchesIdentity(entry) + } catch { + survivors.push(entry) + + continue + } + + if (matches === false) { + continue + } + + if (matches !== true) { + survivors.push(entry) + + continue + } + + try { + await deps.stop(entry) + reaped.push(entry.pid) + } catch { + // Preserve failed ownership so a later startup can retry it. + survivors.push(entry) + } + } + + write(survivors) + + return reaped + }, + + clear(): void { + write([]) + } + } +} + +export function backendCommandMatches(command: unknown): boolean { + return /(?:^|[\s/\\"])(?:hermes(?:\.exe)?|hermes_cli\.main|hermes_cli[/\\]main\.py)"?(?:\s+(?:--profile|-p)\s+\S+)?\s+(?:serve|dashboard)(?:\s|$)/i.test( + String(command ?? '') + ) +} + +/** Coordinates all quit paths so asynchronous backend teardown runs once. */ +export function createBackendShutdownCoordinator(teardown: () => Promise | void) { + let completion: Promise | undefined + + return { + run(): Promise { + if (!completion) { + completion = Promise.resolve().then(teardown) + } + + return completion + }, + hasStarted(): boolean { + return completion !== undefined + } + } +} diff --git a/apps/desktop/electron/connection-config.test.ts b/apps/desktop/electron/connection-config.test.ts index a306bec5ca..b6451cad5d 100644 --- a/apps/desktop/electron/connection-config.test.ts +++ b/apps/desktop/electron/connection-config.test.ts @@ -21,6 +21,7 @@ import { buildGatewayWsUrlWithTicket, connectionScopeKey, cookiesHaveLiveSession, + cookiesHavePrivyAccessToken, cookiesHavePrivySession, cookiesHaveSession, gatewayTicketFailure, @@ -571,6 +572,42 @@ test('cookiesHavePrivySession is false for unrelated cookies and non-arrays', () assert.equal(cookiesHavePrivySession([]), false) }) +test('cookiesHavePrivySession treats refresh-token material as a (renewable) session', () => { + // #73495: after a restart the ~1h `privy-token` is often gone while the + // 30-day renewal cookies survive. That jar is still SIGNED IN (renewable), + // so the session check must accept it — the access check below is what + // distinguishes "can discovery succeed right now". + assert.equal(cookiesHavePrivySession([{ name: 'privy-refresh-token', value: 'x' }]), true) +}) + +// --- cookiesHavePrivyAccessToken (short-lived access state for /api/agents) --- + +test('cookiesHavePrivyAccessToken detects privy-token and its secured prefixes', () => { + assert.equal(cookiesHavePrivyAccessToken([{ name: 'privy-token', value: 'jwt' }]), true) + assert.equal(cookiesHavePrivyAccessToken([{ name: '__Host-privy-token', value: 'x' }]), true) + assert.equal(cookiesHavePrivyAccessToken([{ name: '__Secure-privy-token', value: 'x' }]), true) +}) + +test('cookiesHavePrivyAccessToken rejects renewal-only jars (the #73495 cold-start state)', () => { + // Session/refresh material present, access token absent: signed in but + // discovery would 401 → the silent-renewal path must trigger, not re-login. + const renewalOnly = [ + { name: 'privy-session', value: 'x' }, + { name: 'privy-refresh-token', value: 'x' } + ] + + assert.equal(cookiesHavePrivySession(renewalOnly), true) + assert.equal(cookiesHavePrivyAccessToken(renewalOnly), false) +}) + +test('cookiesHavePrivyAccessToken is false for empty values, gateway cookies, and non-arrays', () => { + assert.equal(cookiesHavePrivyAccessToken([{ name: 'privy-token', value: '' }]), false) + assert.equal(cookiesHavePrivyAccessToken([{ name: 'hermes_session_at', value: 'x' }]), false) + assert.equal(cookiesHavePrivyAccessToken(null), false) + assert.equal(cookiesHavePrivyAccessToken(undefined), false) + assert.equal(cookiesHavePrivyAccessToken([]), false) +}) + // --- tokenPreview --- test('tokenPreview returns null for empty', () => { diff --git a/apps/desktop/electron/connection-config.ts b/apps/desktop/electron/connection-config.ts index 4644008d48..c3a676785d 100644 --- a/apps/desktop/electron/connection-config.ts +++ b/apps/desktop/electron/connection-config.ts @@ -44,7 +44,21 @@ const RT_COOKIE_VARIANTS = ['__Host-hermes_session_rt', '__Secure-hermes_session // sign-in / discovery liveness must look for the Privy cookie, NOT the gateway // cookies above. `privy-token` is the access token (the required signal); // variants cover the secured-prefix forms and the older `privy-session` name. -const PRIVY_SESSION_COOKIE_VARIANTS = ['__Host-privy-token', '__Secure-privy-token', 'privy-token', 'privy-session'] +const PRIVY_SESSION_COOKIE_VARIANTS = [ + '__Host-privy-token', + '__Secure-privy-token', + 'privy-token', + 'privy-session', + 'privy-refresh-token' +] + +// The short-lived Privy ACCESS token only — the credential `/api/agents` +// actually validates. `privy-session` / `privy-refresh-token` are long-lived +// renewal material: their presence means the session is RENEWABLE (signed in, +// no interactive login needed), but discovery still 401s until a fresh +// `privy-token` is minted. Distinguishing the two is what lets a cold start +// silently renew instead of demanding a re-login (#73495). +const PRIVY_ACCESS_COOKIE_VARIANTS = ['__Host-privy-token', '__Secure-privy-token', 'privy-token'] // Keep this aligned with hermes_cli.profiles.validate_profile_name(). `default` // is the built-in root alias; these names cannot be created as profiles. const RESERVED_REMOTE_PROFILES = new Set(['hermes', 'test', 'tmp', 'root', 'sudo']) @@ -557,6 +571,22 @@ function cookiesHavePrivySession(cookies) { return cookies.some(c => c && c.value && PRIVY_SESSION_COOKIE_VARIANTS.includes(c.name)) } +/** + * True only when the short-lived Privy ACCESS token (`privy-token`) is present + * — the exact cookie `/api/agents` validates. A jar can satisfy + * `cookiesHavePrivySession` (renewable session: `privy-session` / + * `privy-refresh-token`) while failing this check; that gap is the cold-start + * "Signed in" + "No agents found" contradiction, and the signal that a silent + * renewal (not an interactive re-login) is the right recovery (#73495). + */ +function cookiesHavePrivyAccessToken(cookies) { + if (!Array.isArray(cookies)) { + return false + } + + return cookies.some(c => c && c.value && PRIVY_ACCESS_COOKIE_VARIANTS.includes(c.name)) +} + export { AT_COOKIE_VARIANTS, authModeFromStatus, @@ -564,6 +594,7 @@ export { buildGatewayWsUrlWithTicket, connectionScopeKey, cookiesHaveLiveSession, + cookiesHavePrivyAccessToken, cookiesHavePrivySession, cookiesHaveSession, gatewayTicketFailure, @@ -576,6 +607,7 @@ export { normalizeSshConfig, normAuthMode, pathWithGlobalRemoteProfile, + PRIVY_ACCESS_COOKIE_VARIANTS, PRIVY_SESSION_COOKIE_VARIANTS, profileHasRemoteConnection, profileRemoteOverride, diff --git a/apps/desktop/electron/connection-registry.test.ts b/apps/desktop/electron/connection-registry.test.ts new file mode 100644 index 0000000000..5e41b61187 --- /dev/null +++ b/apps/desktop/electron/connection-registry.test.ts @@ -0,0 +1,442 @@ +/** + * Tests for electron/connection-registry.ts — the v2 multi-connection + * registry: label rules (required, unique, @handle disambiguation), input + * validation, registry normalization from disk, the v1→v2 migration, and the + * pure upsert/remove/set-primary operations. + */ + +import assert from 'node:assert/strict' + +import { test } from 'vitest' + +import type { ConnectionRegistry } from './connection-registry' +import { + agentHandle, + backendScopeKey, + backendScopePrefix, + connectionIdForLabel, + labelKey, + labelSlug, + LOCAL_CONNECTION_ID, + mergeConnectionInput, + migrateV1ToRegistry, + normalizeConnectionInput, + normalizeRegistry, + REGISTRY_VERSION, + removeConnection, + setPrimaryConnection, + uniqueLabel, + upsertConnection +} from './connection-registry' + +function emptyRegistry(): ConnectionRegistry { + return normalizeRegistry(null) +} + +// --- labels, slugs, handles --- + +test('labelKey is case-insensitive and trimmed', () => { + assert.equal(labelKey(' Homelab '), 'homelab') + assert.equal(labelKey('HOMELAB'), labelKey('homelab')) +}) + +test('labelSlug kebab-cases and never returns empty for non-empty input', () => { + assert.equal(labelSlug('Work Laptop'), 'work-laptop') + assert.equal(labelSlug('Spark Box #2'), 'spark-box-2') + assert.equal(labelSlug('!!!'), 'connection') +}) + +test('agentHandle bare when unique, @name-device shape when duplicated', () => { + assert.equal(agentHandle('research', 'Homelab', false), 'research') + assert.equal(agentHandle('research', 'Homelab', true), 'research-homelab') + assert.equal(agentHandle('research', 'Work Laptop', true), 'research-work-laptop') + assert.equal(agentHandle('', 'Homelab', false), 'default') +}) + +test('connectionIdForLabel suffixes on collision and never mints "local"', () => { + assert.equal(connectionIdForLabel('Homelab', []), 'homelab') + assert.equal(connectionIdForLabel('Homelab', ['homelab']), 'homelab-2') + assert.equal(connectionIdForLabel('Homelab', ['homelab', 'homelab-2']), 'homelab-3') + assert.equal(connectionIdForLabel('Local', []), 'local-2') +}) + +test('uniqueLabel counts up (never "X 2 2") and clamps long candidates', () => { + assert.equal(uniqueLabel('Homelab', []), 'Homelab') + assert.equal(uniqueLabel('Homelab', ['Homelab']), 'Homelab 2') + assert.equal(uniqueLabel('Homelab', ['Homelab', 'Homelab 2']), 'Homelab 3') + // Case-insensitive collision detection. + assert.equal(uniqueLabel('homelab', ['HOMELAB']), 'homelab 2') + + const long = 'x'.repeat(300) + assert.ok(uniqueLabel(long, []).length <= 64) + assert.ok(uniqueLabel(long, [uniqueLabel(long, [])]).length <= 64) +}) + +// --- backendScopeKey (composite pool keys) --- + +test('backendScopeKey: local/empty connection keeps the bare profile key', () => { + assert.equal(backendScopeKey(null, 'research'), 'research') + assert.equal(backendScopeKey('', 'research'), 'research') + assert.equal(backendScopeKey(LOCAL_CONNECTION_ID, 'research'), 'research') + assert.equal(backendScopeKey('local', ''), 'default') + assert.equal(backendScopeKey(undefined, undefined), 'default') +}) + +test('backendScopeKey: non-local connections get an unambiguous composite', () => { + assert.equal(backendScopeKey('homelab', 'research'), 'conn:homelab::research') + assert.equal(backendScopeKey('homelab', ''), 'conn:homelab::default') + // Composite keys can never collide with a plain profile name, and the + // prefix helper matches exactly the keys the connection owns. + assert.ok(backendScopeKey('homelab', 'research').startsWith(backendScopePrefix('homelab'))) + assert.ok(!backendScopeKey('homelab-2', 'research').startsWith(backendScopePrefix('homelab'))) + assert.ok(!'research'.startsWith(backendScopePrefix('homelab'))) +}) + +// --- normalizeConnectionInput --- + +test('save rejects the reserved "local" id on non-local kinds', () => { + assert.throws( + () => + normalizeConnectionInput({ id: 'local', kind: 'remote', label: 'Sneaky', url: 'http://x:1' }, emptyRegistry()), + /reserved/ + ) +}) + +test('token only persists on token-auth remotes; oauth/cloud drop it', () => { + const registry = emptyRegistry() + + const tokenAuth = normalizeConnectionInput( + { kind: 'remote', label: 'A', url: 'http://a:1', authMode: 'token', token: { enc: 'x' } }, + registry + ) + + assert.deepEqual(tokenAuth.token, { enc: 'x' }) + + const oauth = normalizeConnectionInput( + { kind: 'remote', label: 'B', url: 'http://b:1', authMode: 'oauth', token: { enc: 'x' } }, + registry + ) + + assert.equal(oauth.token, undefined) + + const cloud = normalizeConnectionInput( + { kind: 'cloud', label: 'C', url: 'https://c.hermes.cloud', authMode: 'oauth', token: { enc: 'x' } }, + registry + ) + + assert.equal(cloud.token, undefined) +}) + +// --- mergeConnectionInput (edit inheritance) --- + +test('merge preserves fields the editor does not carry (org, ssh extras)', () => { + const cloud = { + authMode: 'oauth' as const, + id: 'c', + kind: 'cloud' as const, + label: 'Cloud', + org: 'nous', + url: 'https://a.cloud' + } + + const renamed = mergeConnectionInput({ id: 'c', kind: 'cloud', label: 'Renamed', url: 'https://a.cloud' }, cloud) + + assert.equal(renamed.org, 'nous') + + const ssh = { + host: 'homelab.lan', + id: 's', + keyPath: '/k/id', + kind: 'ssh' as const, + label: 'Box', + port: 2222, + remoteHermesPath: '/opt/hermes', + remoteProfile: 'research', + user: 'k' + } + + const labelOnly = mergeConnectionInput({ id: 's', kind: 'ssh', label: 'Renamed box' }, ssh) + + assert.equal(labelOnly.remoteHermesPath, '/opt/hermes') + assert.equal(labelOnly.remoteProfile, 'research') + assert.equal(labelOnly.host, 'homelab.lan') + assert.equal(labelOnly.user, 'k') + assert.equal(labelOnly.port, 2222) +}) + +test('merge: a supplied ssh host string beats stored user/port', () => { + const ssh = { host: 'spark1', id: 's', kind: 'ssh' as const, label: 'Spark', port: 2222, user: 'tek' } + const merged = mergeConnectionInput({ host: 'admin@newbox:2200', id: 's', kind: 'ssh', label: 'Spark' }, ssh) + + // Stored user/port must NOT ride along — the host string is authoritative. + assert.equal(merged.user, undefined) + assert.equal(merged.port, undefined) + + const entry = normalizeConnectionInput(merged, emptyRegistry()) + + assert.equal(entry.host, 'newbox') + assert.equal(entry.user, 'admin') + assert.equal(entry.port, 2200) +}) + +test('save rejects a missing label with a device-name message', () => { + assert.throws( + () => normalizeConnectionInput({ kind: 'remote', label: ' ', url: 'http://10.0.0.5:9119' }, emptyRegistry()), + /device name/ + ) +}) + +test('save rejects a duplicate label case-insensitively', () => { + let registry = emptyRegistry() + registry = upsertConnection( + registry, + normalizeConnectionInput({ kind: 'remote', label: 'Homelab', url: 'http://10.0.0.5:9119' }, registry) + ) + + assert.throws( + () => normalizeConnectionInput({ kind: 'remote', label: ' homelab ', url: 'http://10.0.0.9:9119' }, registry), + /must be unique/ + ) +}) + +test('editing an entry does not collide with its own label', () => { + let registry = emptyRegistry() + const entry = normalizeConnectionInput({ kind: 'remote', label: 'Homelab', url: 'http://10.0.0.5:9119' }, registry) + registry = upsertConnection(registry, entry) + + const edited = normalizeConnectionInput( + { id: entry.id, kind: 'remote', label: 'Homelab', url: 'http://10.0.0.6:9119' }, + registry + ) + + assert.equal(edited.id, entry.id) + assert.equal(edited.url, 'http://10.0.0.6:9119') +}) + +test('remote input normalizes URL and auth mode; cloud keeps org', () => { + const registry = emptyRegistry() + + const remote = normalizeConnectionInput( + { kind: 'remote', label: 'LAN box', url: '10.0.0.5:9119', authMode: 'weird' }, + registry + ) + + assert.equal(remote.url, 'http://10.0.0.5:9119') + assert.equal(remote.authMode, 'token') + + const cloud = normalizeConnectionInput( + { kind: 'cloud', label: 'Cloud', url: 'https://foo.hermes.cloud', authMode: 'oauth', org: 'nous' }, + registry + ) + + assert.equal(cloud.kind, 'cloud') + assert.equal(cloud.org, 'nous') + assert.equal(cloud.authMode, 'oauth') +}) + +test('ssh input requires a host; local input only carries the label', () => { + const registry = emptyRegistry() + + assert.throws(() => normalizeConnectionInput({ kind: 'ssh', label: 'Spark', host: ' ' }, registry), /host/) + + const ssh = normalizeConnectionInput({ kind: 'ssh', label: 'Spark', host: 'tek@spark1:2222' }, registry) + + assert.equal(ssh.host, 'spark1') + assert.equal(ssh.user, 'tek') + assert.equal(ssh.port, 2222) + + const local = normalizeConnectionInput({ kind: 'local', label: 'My MacBook' }, registry) + + assert.equal(local.id, LOCAL_CONNECTION_ID) + assert.deepEqual(Object.keys(local).sort(), ['id', 'kind', 'label']) +}) + +// --- normalizeRegistry --- + +test('normalizeRegistry degrades junk to a local-only registry', () => { + for (const junk of [null, undefined, 42, 'nope', { connections: 'zzz' }, { version: 99 }]) { + const registry = normalizeRegistry(junk) + + assert.equal(registry.version, REGISTRY_VERSION) + assert.equal(registry.primary, LOCAL_CONNECTION_ID) + assert.equal(registry.connections.length, 1) + assert.equal(registry.connections[0].kind, 'local') + } +}) + +test('normalizeRegistry guarantees local, dedupes labels, fixes dangling primary', () => { + const registry = normalizeRegistry({ + version: 2, + primary: 'ghost', + connections: [ + { id: 'a', kind: 'remote', label: 'Homelab', url: 'http://10.0.0.5:9119' }, + { id: 'b', kind: 'remote', label: 'homelab', url: 'http://10.0.0.6:9119' }, + { id: 'c', kind: 'remote', label: 'No URL entry' }, + { kind: 'nonsense', label: 'x' } + ] + }) + + assert.equal(registry.primary, LOCAL_CONNECTION_ID) + assert.ok(registry.connections.some(c => c.kind === 'local')) + + const labels = registry.connections.map(c => labelKey(c.label)) + + assert.equal(new Set(labels).size, labels.length) + // The url-less remote entry is dropped, the junk kind is dropped. + assert.equal(registry.connections.filter(c => c.kind === 'remote').length, 2) +}) + +test('normalizeRegistry round-trips a valid registry unchanged in shape', () => { + const input = { + version: 2, + primary: 'homelab', + connections: [ + { id: 'local', kind: 'local', label: 'This device' }, + { + id: 'homelab', + kind: 'remote', + label: 'Homelab', + url: 'http://10.0.0.5:9119', + authMode: 'token', + token: { v: 1 } + }, + { + id: 'cloud-1', + kind: 'cloud', + label: 'Hermes Cloud', + url: 'https://a.hermes.cloud', + authMode: 'oauth', + org: 'nous' + }, + { id: 'spark', kind: 'ssh', label: 'Spark', host: 'spark1', user: 'tek', port: 2222 } + ] + } + + const registry = normalizeRegistry(input) + + assert.equal(registry.primary, 'homelab') + assert.equal(registry.connections.length, 4) + assert.deepEqual( + registry.connections.map(c => c.id), + ['local', 'homelab', 'cloud-1', 'spark'] + ) + assert.deepEqual(registry.connections[1].token, { v: 1 }) + assert.equal(registry.connections[3].port, 2222) +}) + +// --- v1 → v2 migration --- + +test('migrate: v1 local-only config → local-only registry', () => { + const registry = migrateV1ToRegistry({ mode: 'local', remote: {}, profiles: {} }) + + assert.equal(registry.primary, LOCAL_CONNECTION_ID) + assert.equal(registry.connections.length, 1) +}) + +test('migrate: v1 global remote becomes a labeled entry and the primary', () => { + const registry = migrateV1ToRegistry({ + mode: 'remote', + remote: { url: 'http://homelab.lan:9119', authMode: 'token', token: { enc: 'x' } } + }) + + const remote = registry.connections.find(c => c.kind === 'remote') + + assert.ok(remote) + assert.equal(registry.primary, remote.id) + assert.equal(remote.label, 'homelab.lan:9119') + assert.deepEqual(remote.token, { enc: 'x' }) +}) + +test('migrate: v1 cloud keeps cloud provenance + org', () => { + const registry = migrateV1ToRegistry({ + mode: 'cloud', + remote: { url: 'https://a.hermes.cloud', authMode: 'oauth', org: 'nous' } + }) + + const cloud = registry.connections.find(c => c.kind === 'cloud') + + assert.ok(cloud) + assert.equal(registry.primary, cloud.id) + assert.equal(cloud.org, 'nous') +}) + +test('migrate: per-profile overrides become extra sources, deduped by URL', () => { + const registry = migrateV1ToRegistry({ + mode: 'remote', + remote: { url: 'http://homelab.lan:9119', authMode: 'token', token: { enc: 'x' } }, + profiles: { + research: { mode: 'remote', url: 'http://homelab.lan:9119', authMode: 'token', token: { enc: 'x' } }, + coder: { mode: 'remote', url: 'http://other.lan:9119', authMode: 'token', token: { enc: 'y' } }, + sparky: { mode: 'ssh', host: 'spark1', user: 'tek' }, + plain: { mode: 'local', savedSsh: { mode: 'ssh', host: 'spark1', user: 'tek' } } + } + }) + + // homelab (global+research deduped), other.lan, spark ssh (override+savedSsh deduped), local + assert.equal(registry.connections.length, 4) + assert.equal(registry.connections.filter(c => c.kind === 'remote').length, 2) + assert.equal(registry.connections.filter(c => c.kind === 'ssh').length, 1) +}) + +test('migrate: v1 global ssh becomes the primary', () => { + const registry = migrateV1ToRegistry({ + mode: 'ssh', + remote: { mode: 'ssh', host: 'spark1', user: 'tek', port: 2222 } + }) + + const ssh = registry.connections.find(c => c.kind === 'ssh') + + assert.ok(ssh) + assert.equal(registry.primary, ssh.id) + assert.equal(ssh.label, 'spark1') +}) + +test('migrate: duplicate host labels are suffixed, not dropped', () => { + const registry = migrateV1ToRegistry({ + mode: 'remote', + remote: { url: 'http://box.lan:9119', authMode: 'token', token: {} }, + profiles: { + a: { mode: 'ssh', host: 'box.lan' } + } + }) + + const labels = registry.connections.map(c => labelKey(c.label)) + + assert.equal(new Set(labels).size, labels.length) + assert.equal(registry.connections.length, 3) +}) + +// --- registry operations --- + +test('removeConnection: local refuses, primary retargets to local', () => { + let registry = emptyRegistry() + const entry = normalizeConnectionInput({ kind: 'remote', label: 'Homelab', url: 'http://10.0.0.5:9119' }, registry) + registry = upsertConnection(registry, entry) + registry = setPrimaryConnection(registry, entry.id) + + assert.throws(() => removeConnection(registry, LOCAL_CONNECTION_ID), /cannot be removed/) + + const after = removeConnection(registry, entry.id) + + assert.equal(after.primary, LOCAL_CONNECTION_ID) + assert.equal(after.connections.length, 1) + // Removing an unknown id is a no-op, not an error. + assert.equal(removeConnection(after, 'ghost'), after) +}) + +test('setPrimaryConnection validates the target id', () => { + const registry = emptyRegistry() + + assert.throws(() => setPrimaryConnection(registry, 'ghost'), /No connection/) + assert.equal(setPrimaryConnection(registry, LOCAL_CONNECTION_ID).primary, LOCAL_CONNECTION_ID) +}) + +test('upsertConnection replaces by id and appends new ids', () => { + let registry = emptyRegistry() + const a = normalizeConnectionInput({ kind: 'remote', label: 'A', url: 'http://a:1' }, registry) + registry = upsertConnection(registry, a) + registry = upsertConnection(registry, { ...a, url: 'http://a:2' }) + + assert.equal(registry.connections.filter(c => c.id === a.id).length, 1) + assert.equal(registry.connections.find(c => c.id === a.id)?.url, 'http://a:2') +}) diff --git a/apps/desktop/electron/connection-registry.ts b/apps/desktop/electron/connection-registry.ts new file mode 100644 index 0000000000..9752aa2ca0 --- /dev/null +++ b/apps/desktop/electron/connection-registry.ts @@ -0,0 +1,614 @@ +/** + * connection-registry.ts + * + * Pure, electron-free helpers for the desktop's multi-connection registry — + * the v2 successor to the single global `mode` + `remote` block in + * connection.json. The registry is a named list of agent SOURCES (local + * runtime, remote gateways, Hermes Cloud instances, SSH hosts) that are all + * registered at once; routing/pooling changes that consume the registry land + * separately, so this module is deliberately storage-shaped, not + * transport-shaped. + * + * Design rules (agreed with Teknium, Aug 2026): + * - Every connection carries a REQUIRED, registry-unique `label` (the + * "device name"). Uniqueness is case-insensitive so `Homelab` and + * `homelab` can't coexist and produce two identical badges. + * - When two sources expose the same profile name, surfaces disambiguate as + * `@-` — `agentHandle()` is the one place that rule + * lives. + * - The registry ALWAYS contains exactly one `local` connection (the app's + * own runtime). It cannot be removed; it is the default primary. + * - `primary` designates the connection that owns the window backend (boot + * overlay, install/update machinery). Removing the primary retargets to + * the local entry rather than leaving a dangling id. + * + * Kept standalone (no `import 'electron'`) so it unit-tests with `node --test` + * — same pattern as connection-config.ts / backend-probes.ts. main.ts wires + * these into the IPC layer and owns file I/O + secret encryption. + */ + +import { + hostLabelFromBaseUrl, + modeIsRemoteLike, + normalizeRemoteBaseUrl, + normalizeSshConfig, + normAuthMode +} from './connection-config' + +export const REGISTRY_VERSION = 2 + +export const LOCAL_CONNECTION_ID = 'local' + +/** Connection kinds. 'cloud' is remote-shaped (see modeIsRemoteLike) but keeps + * its provenance so the UI can render the right card and updates can skip + * platform-managed instances. */ +export type ConnectionKind = 'cloud' | 'local' | 'remote' | 'ssh' + +export interface RegistryConnection { + id: string + kind: ConnectionKind + /** Required, unique (case-insensitive) display name — the "device name". */ + label: string + /** remote/cloud: normalized base URL. */ + url?: string + /** remote/cloud: 'token' | 'oauth'. */ + authMode?: 'oauth' | 'token' + /** remote: encrypted token envelope (opaque here; main.ts encrypts/decrypts). */ + token?: unknown + /** cloud: portal org slug/id the instance was discovered under. */ + org?: string + /** ssh fields (normalizeSshConfig shapes). */ + host?: string + user?: string + port?: number + keyPath?: string + remoteHermesPath?: string + remoteProfile?: string +} + +export interface ConnectionRegistry { + version: typeof REGISTRY_VERSION + /** id of the connection that owns the window/primary backend. */ + primary: string + connections: RegistryConnection[] +} + +// ── Labels and ids ────────────────────────────────────────────────────────── + +const LABEL_MAX = 64 + +/** Canonical comparison key for label uniqueness. */ +export function labelKey(label: string): string { + return String(label || '') + .trim() + .toLowerCase() +} + +/** + * Derive a registry-unique label from a candidate: clamps to LABEL_MAX (a + * migrated URL host can exceed it, which would fail validation on any later + * edit) and suffixes " 2" / " 3" / … on collision. The single home of the + * label-dedup rule — normalizeRegistry and the migration both use it. + */ +export function uniqueLabel(candidate: string, taken: Iterable): string { + const used = new Set([...taken].map(labelKey)) + + // Reserve room for a collision suffix so the suffixed form stays in-bounds. + const base = String(candidate || '') + .trim() + .slice(0, LABEL_MAX - 4) + + if (!used.has(labelKey(base))) { + return base + } + + for (let n = 2; ; n += 1) { + const suffixed = `${base} ${n}` + + if (!used.has(labelKey(suffixed))) { + return suffixed + } + } +} + +/** Kebab-slug of a label for ids and @handles. Never empty for a non-empty label. */ +export function labelSlug(label: string): string { + const slug = String(label || '') + .trim() + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 48) + + return slug || 'connection' +} + +/** + * The one place the duplicate-agent naming rule lives: a profile that exists + * on several registered sources renders as `@-`; + * a profile unique across the roster keeps its bare name. + */ +export function agentHandle(profile: string, connectionLabel: string, duplicated: boolean): string { + const name = String(profile || '').trim() || 'default' + + return duplicated ? `${name}-${labelSlug(connectionLabel)}` : name +} + +/** + * Pool key for a backend serving (connection, profile). The local/primary + * connection keeps the BARE profile key so every legacy pool entry, reaper + * log line, and touch call stays byte-identical for single-source users; + * non-local connections get an unambiguous composite (`conn:::`) + * that cannot collide with a plain profile name (colons are invalid in + * profile names). The single home of the composite-key rule — Electron pool + * keying and any renderer socket registry must both derive keys here. + */ +export function backendScopeKey(connectionId: null | string | undefined, profile: null | string | undefined): string { + const profileKey = String(profile ?? '').trim() || 'default' + const connection = String(connectionId ?? '').trim() + + if (!connection || connection === LOCAL_CONNECTION_ID) { + return profileKey + } + + return `conn:${connection}::${profileKey}` +} + +/** All pool keys owned by a connection share this prefix (used to stop them on remove). */ +export function backendScopePrefix(connectionId: string): string { + return `conn:${String(connectionId).trim()}::` +} + +/** Mint a registry-unique id from a label (slug, then -2/-3… suffixes). */ +export function connectionIdForLabel(label: string, taken: Iterable): string { + const used = new Set([...taken]) + const base = labelSlug(label) + + if (!used.has(base) && base !== LOCAL_CONNECTION_ID) { + return base + } + + for (let n = 2; ; n += 1) { + const candidate = `${base}-${n}` + + if (!used.has(candidate) && candidate !== LOCAL_CONNECTION_ID) { + return candidate + } + } +} + +// ── Validation ────────────────────────────────────────────────────────────── + +export interface ConnectionInput { + id?: string + kind: ConnectionKind + label: string + url?: string + authMode?: string + token?: unknown + org?: string + host?: string + user?: string + port?: number | string + keyPath?: string + remoteHermesPath?: string + remoteProfile?: string +} + +/** + * Validate + normalize a save payload into a RegistryConnection. + * Throws with a user-facing message on any violation. `registry` supplies the + * uniqueness context; when `input.id` matches an existing entry this is an + * edit and that entry is excluded from the label-collision check. + */ +export function normalizeConnectionInput(input: ConnectionInput, registry: ConnectionRegistry): RegistryConnection { + const label = String(input.label || '').trim() + + if (!label) { + throw new Error('Every connection needs a name. Give this instance a device name (e.g. "Homelab", "Work laptop").') + } + + if (label.length > LABEL_MAX) { + throw new Error(`Connection name is too long (max ${LABEL_MAX} characters).`) + } + + const key = labelKey(label) + const collision = registry.connections.find(c => labelKey(c.label) === key && c.id !== input.id) + + if (collision) { + throw new Error(`A connection named "${collision.label}" already exists. Connection names must be unique.`) + } + + const kind = input.kind + + if (kind === 'local') { + // The local entry is managed by the app; only its label is editable. + return { id: LOCAL_CONNECTION_ID, kind: 'local', label } + } + + // The reserved local id can never be claimed by a non-local entry — a + // crafted IPC payload ({id:'local', kind:'remote', …}) would otherwise + // replace the local entry via upsert and break the exactly-one-local + // invariant. connectionIdForLabel never mints 'local'; reject it when + // supplied, too. + if (input.id === LOCAL_CONNECTION_ID) { + throw new Error('The id "local" is reserved for the local connection.') + } + + const id = + input.id || + connectionIdForLabel( + label, + registry.connections.map(c => c.id) + ) + + if (kind === 'ssh') { + const ssh = normalizeSshConfig({ + mode: 'ssh', + host: input.host, + user: input.user, + port: input.port, + keyPath: input.keyPath, + remoteHermesPath: input.remoteHermesPath, + remoteProfile: input.remoteProfile + }) + + if (!ssh) { + throw new Error('SSH connections need a host.') + } + + const { mode: _mode, ...sshFields } = ssh + + return { id, kind: 'ssh', label, ...sshFields } + } + + if (kind === 'remote' || kind === 'cloud') { + // normalizeRemoteBaseUrl throws its own user-facing message on bad input. + const url = normalizeRemoteBaseUrl(input.url) + const authMode = normAuthMode(input.authMode) + const entry: RegistryConnection = { id, kind, label, url, authMode } + + // A token is only meaningful for token-auth remotes. Dropping it here is + // what clears the stale envelope when an entry is switched token→oauth + // (or is a cloud entry, which authenticates via the portal session) — + // otherwise dead secret material rides along on the edited entry. + if (input.token !== undefined && kind === 'remote' && authMode === 'token') { + entry.token = input.token + } + + const org = String(input.org || '').trim() + + if (kind === 'cloud' && org) { + entry.org = org + } + + return entry + } + + throw new Error(`Unknown connection kind: ${String(kind)}`) +} + +/** + * Merge a (possibly partial) edit payload over the stored entry so fields the + * editor doesn't carry survive a save. Renaming a migrated cloud entry must + * not drop its `org` (downstream update-fanout uses it to skip + * platform-managed instances), and renaming an ssh entry must not drop + * `remoteHermesPath`/`remoteProfile`. Only fields the payload explicitly + * carries (non-undefined) override; `token` is deliberately NOT merged here — + * the caller owns secret handling. + */ +export function mergeConnectionInput(input: ConnectionInput, existing?: null | RegistryConnection): ConnectionInput { + if (!existing || existing.kind !== input.kind) { + return input + } + + const merged: ConnectionInput = { ...input } + + const inherit = (field: keyof ConnectionInput & keyof RegistryConnection) => { + if (merged[field] === undefined && existing[field] !== undefined) { + ;(merged as unknown as Record)[field] = existing[field] + } + } + + inherit('url') + inherit('authMode') + inherit('org') + inherit('host') + inherit('keyPath') + inherit('remoteHermesPath') + inherit('remoteProfile') + + // ssh user/port: the editor shows ONE composite host field (user@host:port), + // and normalizeSshConfig gives explicit user/port fields precedence over the + // parsed host string. Inheriting stored user/port alongside a NEW host string + // would resurrect the old values over what the user just typed — so when the + // payload carries a host, the host string is authoritative and stored + // user/port are NOT inherited. + if (input.host === undefined || !String(input.host).trim()) { + inherit('user') + inherit('port') + } + + return merged +} + +// ── Registry-level operations (all pure: return a new registry) ──────────── + +function localEntry(label = 'This device'): RegistryConnection { + return { id: LOCAL_CONNECTION_ID, kind: 'local', label } +} + +/** + * Coerce arbitrary parsed JSON into a valid registry: version stamped, a + * local entry guaranteed, labels de-duplicated defensively (suffix, never + * drop), primary always pointing at an existing entry. A hand-edited or + * corrupt file degrades to a minimal local-only registry rather than + * throwing at boot. + */ +export function normalizeRegistry(raw: unknown): ConnectionRegistry { + const parsed = raw && typeof raw === 'object' ? (raw as Record) : {} + const rawConnections = Array.isArray(parsed.connections) ? parsed.connections : [] + const seenLabels = new Set() + const seenIds = new Set() + const connections: RegistryConnection[] = [] + + for (const item of rawConnections) { + if (!item || typeof item !== 'object') { + continue + } + + const entry = item as Record + const kind = entry.kind + + if (kind !== 'local' && kind !== 'remote' && kind !== 'cloud' && kind !== 'ssh') { + continue + } + + let label = String(entry.label || '').trim() + + if (!label) { + // Defensive: registry entries are always written with labels, but a + // hand-edited file may drop one. Derive rather than discard. + label = + kind === 'ssh' ? String(entry.host || 'ssh') : hostLabelFromBaseUrl(String(entry.url || '')) || String(kind) + } + + label = uniqueLabel(label, seenLabels) + + let id = kind === 'local' ? LOCAL_CONNECTION_ID : String(entry.id || '').trim() + + if (!id || (seenIds.has(id) && kind !== 'local')) { + id = connectionIdForLabel(label, seenIds) + } + + if (seenIds.has(id)) { + continue // second 'local' entry — first one wins + } + + seenLabels.add(labelKey(label)) + seenIds.add(id) + + const clean: RegistryConnection = { id, kind, label } + + if (kind === 'remote' || kind === 'cloud') { + const url = String(entry.url || '').trim() + + if (!url) { + continue + } + + clean.url = url + clean.authMode = normAuthMode(entry.authMode) + + if (entry.token !== undefined) { + clean.token = entry.token + } + + const org = String(entry.org || '').trim() + + if (kind === 'cloud' && org) { + clean.org = org + } + } else if (kind === 'ssh') { + const ssh = normalizeSshConfig({ ...entry, mode: 'ssh' }) + + if (!ssh) { + continue + } + + const { mode: _mode, ...sshFields } = ssh + Object.assign(clean, sshFields) + } + + connections.push(clean) + } + + if (!connections.some(c => c.kind === 'local')) { + connections.unshift(localEntry()) + } + + const primary = String(parsed.primary || '').trim() + + return { + version: REGISTRY_VERSION, + primary: connections.some(c => c.id === primary) ? primary : LOCAL_CONNECTION_ID, + connections + } +} + +/** + * One-time import of the v1 connection.json shape (global `mode` + `remote` + * block + per-profile `profiles` map) into a v2 registry. v1 had no labels, + * so they are derived (URL host, SSH host, "This device") and uniqued by + * suffixing. The active v1 global connection becomes the primary. The v1 + * file is left untouched by the caller — old builds keep working. + * + * Per-profile override entries become registry connections too (deduped by + * URL/host against the global block), so a user who had `research` pinned to + * a second gateway sees both sources registered on first launch. + */ +export function migrateV1ToRegistry(v1: unknown): ConnectionRegistry { + const config = v1 && typeof v1 === 'object' ? (v1 as Record) : {} + const connections: RegistryConnection[] = [localEntry()] + const byFingerprint = new Map() + + const addRemoteLike = (block: Record, kind: 'cloud' | 'remote'): null | RegistryConnection => { + const url = String(block?.url || '').trim() + + if (!url) { + return null + } + + const fingerprint = `${kind}:${url}` + const existing = byFingerprint.get(fingerprint) + + if (existing) { + return existing + } + + const label = uniqueLabel( + hostLabelFromBaseUrl(url) || (kind === 'cloud' ? 'Hermes Cloud' : 'Remote gateway'), + connections.map(c => c.label) + ) + + const entry: RegistryConnection = { + id: connectionIdForLabel( + label, + connections.map(c => c.id) + ), + kind, + label, + url, + authMode: normAuthMode(block.authMode) + } + + if (block.token !== undefined) { + entry.token = block.token + } + + const org = String(block.org || '').trim() + + if (kind === 'cloud' && org) { + entry.org = org + } + + connections.push(entry) + byFingerprint.set(fingerprint, entry) + + return entry + } + + const addSsh = (block: Record): null | RegistryConnection => { + const ssh = normalizeSshConfig({ ...block, mode: 'ssh' }) + + if (!ssh) { + return null + } + + const fingerprint = `ssh:${ssh.user || ''}@${ssh.host}:${ssh.port || 22}` + const existing = byFingerprint.get(fingerprint) + + if (existing) { + return existing + } + + const label = uniqueLabel( + ssh.host, + connections.map(c => c.label) + ) + + const { mode: _mode, ...sshFields } = ssh + + const entry: RegistryConnection = { + id: connectionIdForLabel( + label, + connections.map(c => c.id) + ), + kind: 'ssh', + label, + ...sshFields + } + + connections.push(entry) + byFingerprint.set(fingerprint, entry) + + return entry + } + + // Global connection → an entry + the primary designation. + let primary = LOCAL_CONNECTION_ID + const globalMode = config.mode + + if (modeIsRemoteLike(globalMode)) { + const entry = addRemoteLike(config.remote || {}, globalMode === 'cloud' ? 'cloud' : 'remote') + + if (entry) { + primary = entry.id + } + } else if (globalMode === 'ssh') { + const entry = addSsh(config.remote || {}) + + if (entry) { + primary = entry.id + } + } + + // Per-profile overrides → additional registered sources (deduped). + const profiles = config.profiles && typeof config.profiles === 'object' ? config.profiles : {} + + for (const block of Object.values(profiles) as Record[]) { + if (!block || typeof block !== 'object') { + continue + } + + if (modeIsRemoteLike(block.mode)) { + addRemoteLike(block, block.mode === 'cloud' ? 'cloud' : 'remote') + } else if (block.mode === 'ssh') { + addSsh(block) + } else if (block.mode === 'local' && block.savedSsh) { + addSsh(block.savedSsh) + } + } + + return { version: REGISTRY_VERSION, primary, connections } +} + +/** Insert or replace by id. Input must already be normalized/validated. */ +export function upsertConnection(registry: ConnectionRegistry, entry: RegistryConnection): ConnectionRegistry { + const connections = registry.connections.some(c => c.id === entry.id) + ? registry.connections.map(c => (c.id === entry.id ? entry : c)) + : [...registry.connections, entry] + + return { ...registry, connections } +} + +/** + * Remove a connection. The local entry is not removable; removing the + * current primary retargets primary to local. + */ +export function removeConnection(registry: ConnectionRegistry, id: string): ConnectionRegistry { + const target = registry.connections.find(c => c.id === id) + + if (!target) { + return registry + } + + if (target.kind === 'local') { + throw new Error('The local connection cannot be removed.') + } + + return { + ...registry, + primary: registry.primary === id ? LOCAL_CONNECTION_ID : registry.primary, + connections: registry.connections.filter(c => c.id !== id) + } +} + +/** Point the window/primary backend at another registered connection. */ +export function setPrimaryConnection(registry: ConnectionRegistry, id: string): ConnectionRegistry { + if (!registry.connections.some(c => c.id === id)) { + throw new Error(`No connection with id "${id}".`) + } + + return { ...registry, primary: id } +} diff --git a/apps/desktop/electron/desktop-log-line.test.ts b/apps/desktop/electron/desktop-log-line.test.ts new file mode 100644 index 0000000000..4adca4c29d --- /dev/null +++ b/apps/desktop/electron/desktop-log-line.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest' + +import { formatDesktopLogLine } from './desktop-log-line' + +describe('formatDesktopLogLine', () => { + it('prefixes each line with an ISO-8601 timestamp and the hermes tag', () => { + const line = formatDesktopLogLine('[boot] Resolving Hermes backend') + + // Shape contract (not a snapshot): every desktop log line starts with + // an ISO timestamp so multi-surface logs are chronologically readable. + // See #84405. + expect(line).toMatch( + /^\[\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\] \[hermes\] \[boot\] Resolving Hermes backend$/ + ) + }) + + it('keeps the message verbatim after the prefix', () => { + const line = formatDesktopLogLine('Hermes backend exited (0)') + + expect(line).toMatch(/^\[\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\] \[hermes\] Hermes backend exited \(0\)$/) + }) +}) diff --git a/apps/desktop/electron/desktop-log-line.ts b/apps/desktop/electron/desktop-log-line.ts new file mode 100644 index 0000000000..86db90149e --- /dev/null +++ b/apps/desktop/electron/desktop-log-line.ts @@ -0,0 +1,19 @@ +/** + * Desktop log line formatting shared by every desktop log surface: + * `desktop.log`, the in-app "RECENT LOGS" view, and crash forensics. + * + * Historically each line was prefixed with just `[hermes] `, so lines from + * different moments were indistinguishable. Every surface now carries an + * ISO-8601 UTC timestamp, matching the Python-side `agent.log` / + * `gateway.log` convention (`2026-07-12 16:22:17,540 INFO ...`). See #84405. + */ + +/** + * Format one desktop log line with an ISO-8601 UTC timestamp. + * + * `stamp` defaults to now; callers that batch multiple lines (a single + * stdout chunk) pass one shared stamp so the group reads as one event. + */ +export function formatDesktopLogLine(text: string, stamp = new Date().toISOString()): string { + return `[${stamp}] [hermes] ${text}` +} diff --git a/apps/desktop/electron/external-terminal.test.ts b/apps/desktop/electron/external-terminal.test.ts new file mode 100644 index 0000000000..c5bac2c989 --- /dev/null +++ b/apps/desktop/electron/external-terminal.test.ts @@ -0,0 +1,133 @@ +import assert from 'node:assert/strict' + +import { test } from 'vitest' + +import { + buildTerminalScript, + posixQuote, + resolveTerminalLaunch, + terminalScriptEnv, + terminalScriptExtension, + tuiResumeArgs, + windowsQuote +} from './external-terminal' + +const never = () => null +const always = (command: string) => `/usr/bin/${command}` + +test('tuiResumeArgs resumes the session in the TUI', () => { + assert.deepEqual(tuiResumeArgs('20260814_101010_abc123'), ['--tui', '--resume', '20260814_101010_abc123']) +}) + +test('tuiResumeArgs pins the profile ahead of the mode flag', () => { + assert.deepEqual(tuiResumeArgs('sess', 'work'), ['--profile', 'work', '--tui', '--resume', 'sess']) +}) + +test('posixQuote survives embedded single quotes', () => { + assert.equal(posixQuote("/tmp/o'brien"), `'/tmp/o'\\''brien'`) +}) + +test('windowsQuote doubles embedded quotes', () => { + assert.equal(windowsQuote('C:\\a "b"'), '"C:\\a ""b"""') +}) + +test('terminalScriptEnv drops PATH in any casing and keeps the rest', () => { + const env = terminalScriptEnv( + { Path: 'C:\\junk', PATH: '/junk', PYTHONPATH: '/repo', PYTHONUTF8: '1' }, + '/home/b/.hermes' + ) + + assert.deepEqual(env, { PYTHONPATH: '/repo', PYTHONUTF8: '1', HERMES_HOME: '/home/b/.hermes' }) +}) + +test('terminalScriptEnv skips empty values and an absent home', () => { + assert.deepEqual(terminalScriptEnv({ PYTHONPATH: '' }), {}) +}) + +test('buildTerminalScript execs the resolved runtime with its env', () => { + const script = buildTerminalScript({ + args: ['-m', 'hermes_cli.main', '--tui', '--resume', 'sess'], + command: '/home/b/.hermes/hermes-agent/venv/bin/python', + cwd: "/home/b/o'brien", + env: { PYTHONPATH: '/home/b/.hermes/hermes-agent' }, + platform: 'darwin' + }) + + assert.equal( + script, + [ + '#!/bin/sh', + `cd '/home/b/o'\\''brien' || exit 1`, + `export PYTHONPATH='/home/b/.hermes/hermes-agent'`, + `exec '/home/b/.hermes/hermes-agent/venv/bin/python' '-m' 'hermes_cli.main' '--tui' '--resume' 'sess'`, + '' + ].join('\n') + ) +}) + +test('buildTerminalScript emits a cmd script on Windows', () => { + const script = buildTerminalScript({ + args: ['--tui', '--resume', 'sess'], + command: 'C:\\hermes\\venv\\Scripts\\hermes.exe', + cwd: 'C:\\Users\\b', + env: { PYTHONUTF8: '1' }, + platform: 'win32' + }) + + assert.deepEqual(script.split('\r\n'), [ + '@echo off', + 'cd /d "C:\\Users\\b"', + 'set "PYTHONUTF8=1"', + '"C:\\hermes\\venv\\Scripts\\hermes.exe" "--tui" "--resume" "sess"', + '' + ]) +}) + +test('terminalScriptExtension matches what the platform binds to a terminal', () => { + assert.equal(terminalScriptExtension('darwin'), '.command') + assert.equal(terminalScriptExtension('win32'), '.cmd') + assert.equal(terminalScriptExtension('linux'), '.sh') +}) + +test('macOS opens the script with no -a so LaunchServices picks the user handler', () => { + assert.deepEqual(resolveTerminalLaunch({ findOnPath: never, platform: 'darwin', scriptPath: '/tmp/x.command' }), { + command: 'open', + args: ['/tmp/x.command'] + }) +}) + +test('Windows prefers Windows Terminal and falls back to a cmd console', () => { + assert.deepEqual( + resolveTerminalLaunch({ + findOnPath: command => (command === 'wt.exe' ? 'C:\\wt.exe' : null), + platform: 'win32', + scriptPath: 'C:\\x.cmd' + }), + { command: 'C:\\wt.exe', args: ['cmd.exe', '/k', 'C:\\x.cmd'] } + ) + + assert.deepEqual(resolveTerminalLaunch({ findOnPath: never, platform: 'win32', scriptPath: 'C:\\x.cmd' }), { + command: 'cmd.exe', + args: ['/c', 'start', '', 'cmd.exe', '/k', 'C:\\x.cmd'] + }) +}) + +test("Linux leads with the user's x-terminal-emulator alternative", () => { + assert.deepEqual(resolveTerminalLaunch({ findOnPath: always, platform: 'linux', scriptPath: '/tmp/x.sh' }), { + command: '/usr/bin/x-terminal-emulator', + args: ['-e', '/bin/sh', '/tmp/x.sh'] + }) +}) + +test('Linux falls down the emulator ladder and omits a flagless terminal', () => { + const onlyKitty = (command: string) => (command === 'kitty' ? '/usr/bin/kitty' : null) + + assert.deepEqual(resolveTerminalLaunch({ findOnPath: onlyKitty, platform: 'linux', scriptPath: '/tmp/x.sh' }), { + command: '/usr/bin/kitty', + args: ['/bin/sh', '/tmp/x.sh'] + }) +}) + +test('Linux with no emulator installed reports no launch', () => { + assert.equal(resolveTerminalLaunch({ findOnPath: never, platform: 'linux', scriptPath: '/tmp/x.sh' }), null) +}) diff --git a/apps/desktop/electron/external-terminal.ts b/apps/desktop/electron/external-terminal.ts new file mode 100644 index 0000000000..07cfe53a17 --- /dev/null +++ b/apps/desktop/electron/external-terminal.ts @@ -0,0 +1,173 @@ +// Launching the Hermes TUI in the user's OWN terminal emulator. +// +// This is deliberately NOT the in-app terminal pane: the point of the verb is +// to hand a session to the terminal the user already lives in, running +// `hermes --tui --resume ` there. Two problems have to be solved for that +// to work anywhere: +// +// 1. WHAT to run. The desktop's Hermes runtime is often a venv Python invoked +// as `python -m hermes_cli.main`, not a `hermes` on PATH — so the command +// and its PYTHONPATH have to be carried over verbatim. We write them into a +// small launcher script instead of trying to quote a nested command through +// a terminal emulator's `-e` argument, which every emulator parses +// differently. +// 2. WHERE to run it. There is no portable "default terminal" API, so each +// platform gets its own resolution: +// - macOS: `open` the `.command` script with NO `-a`, letting +// LaunchServices route it to whichever app the user has bound to shell +// scripts (Terminal.app by default, iTerm2/Ghostty/WezTerm when they've +// claimed it). That is the closest thing macOS has to "their terminal". +// - Linux: an ordered ladder of emulators, led by Debian's +// `x-terminal-emulator` alternative — which IS the user's configured +// choice — before falling back to the common concrete emulators. +// - Windows: Windows Terminal when installed, else a `cmd.exe` console. +// +// Everything here is pure so it can be unit-tested without Electron; the side +// effects (writing the script, spawning) live in main.ts. + +/** Argv for resuming a session in the TUI, profile-pinned when we know it. */ +export function tuiResumeArgs(sessionId: string, profile?: string): string[] { + const head = profile ? ['--profile', profile] : [] + + return [...head, '--tui', '--resume', sessionId] +} + +/** Single-quote a value for /bin/sh (the POSIX launcher script). */ +export function posixQuote(value: string): string { + return `'${String(value ?? '').replaceAll("'", `'\\''`)}'` +} + +/** Quote a value for a cmd.exe script line. */ +export function windowsQuote(value: string): string { + return `"${String(value ?? '').replaceAll('"', '""')}"` +} + +/** + * The environment the launcher script exports. + * + * PATH is deliberately dropped: the script runs inside a login shell that + * already has the user's own PATH, and the desktop's PATH (assembled for a + * headless child) is the wrong answer for an interactive terminal. The Hermes + * command is invoked by absolute path, so nothing here depends on PATH. + */ +export function terminalScriptEnv( + backendEnv: Record = {}, + hermesHome?: string +): Record { + const out: Record = {} + + for (const [key, value] of Object.entries(backendEnv)) { + if (key.toUpperCase() === 'PATH' || value === undefined || value === '') { + continue + } + + out[key] = value + } + + if (hermesHome) { + out.HERMES_HOME = hermesHome + } + + return out +} + +export interface TerminalScriptSpec { + command: string + args: string[] + cwd: string + env?: Record + platform?: NodeJS.Platform +} + +/** + * The launcher script contents. `exec` on POSIX so the terminal window belongs + * to the TUI itself rather than an idle shell wrapping it. + */ +export function buildTerminalScript({ command, args, cwd, env = {}, platform = process.platform }: TerminalScriptSpec) { + const entries = Object.entries(env) + + if (platform === 'win32') { + return [ + '@echo off', + `cd /d ${windowsQuote(cwd)}`, + ...entries.map(([key, value]) => `set ${windowsQuote(`${key}=${value}`)}`), + [command, ...args].map(windowsQuote).join(' '), + '' + ].join('\r\n') + } + + return [ + '#!/bin/sh', + `cd ${posixQuote(cwd)} || exit 1`, + ...entries.map(([key, value]) => `export ${key}=${posixQuote(value)}`), + `exec ${[command, ...args].map(posixQuote).join(' ')}`, + '' + ].join('\n') +} + +export function terminalScriptExtension(platform: NodeJS.Platform = process.platform): string { + if (platform === 'win32') { + return '.cmd' + } + + // `.command` is the UTI macOS binds to a terminal app; on Linux the + // extension is cosmetic (we always name the interpreter explicitly). + return platform === 'darwin' ? '.command' : '.sh' +} + +// Linux emulators in resolution order, with the flag that precedes a program +// to run. `x-terminal-emulator` is Debian/Ubuntu's alternatives symlink to the +// user's chosen terminal, so it leads; the rest are the common concretes. +const LINUX_TERMINALS: Array<{ command: string; flag: string }> = [ + { command: 'x-terminal-emulator', flag: '-e' }, + { command: 'gnome-terminal', flag: '--' }, + { command: 'konsole', flag: '-e' }, + { command: 'xfce4-terminal', flag: '-x' }, + { command: 'tilix', flag: '-e' }, + { command: 'kitty', flag: '' }, + { command: 'alacritty', flag: '-e' }, + { command: 'wezterm', flag: '-e' }, + { command: 'foot', flag: '' }, + { command: 'xterm', flag: '-e' } +] + +export interface TerminalLaunchOptions { + scriptPath: string + findOnPath: (command: string) => null | string + platform?: NodeJS.Platform +} + +/** + * Resolve the argv that opens `scriptPath` in a terminal window, or null when + * no terminal emulator could be found (Linux boxes with none installed). + */ +export function resolveTerminalLaunch({ + scriptPath, + findOnPath, + platform = process.platform +}: TerminalLaunchOptions): { command: string; args: string[] } | null { + if (platform === 'darwin') { + // No `-a`: LaunchServices picks the user's handler for shell scripts. + return { command: 'open', args: [scriptPath] } + } + + if (platform === 'win32') { + const windowsTerminal = findOnPath('wt.exe') + + if (windowsTerminal) { + return { command: windowsTerminal, args: ['cmd.exe', '/k', scriptPath] } + } + + return { command: 'cmd.exe', args: ['/c', 'start', '', 'cmd.exe', '/k', scriptPath] } + } + + for (const { command, flag } of LINUX_TERMINALS) { + const resolved = findOnPath(command) + + if (resolved) { + return { command: resolved, args: [...(flag ? [flag] : []), '/bin/sh', scriptPath] } + } + } + + return null +} diff --git a/apps/desktop/electron/find-in-page-native-fixture/package.json b/apps/desktop/electron/find-in-page-native-fixture/package.json new file mode 100644 index 0000000000..733769cd92 --- /dev/null +++ b/apps/desktop/electron/find-in-page-native-fixture/package.json @@ -0,0 +1,6 @@ +{ + "name": "find-in-page-native-fixture", + "private": true, + "type": "module", + "main": "../find-in-page-native.test.mjs" +} diff --git a/apps/desktop/electron/find-in-page-native.test.mjs b/apps/desktop/electron/find-in-page-native.test.mjs new file mode 100644 index 0000000000..25d5f9c6df --- /dev/null +++ b/apps/desktop/electron/find-in-page-native.test.mjs @@ -0,0 +1,104 @@ +import assert from 'node:assert/strict' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { app, BrowserWindow } from 'electron' + +const runtimeDir = mkdtempSync(join(tmpdir(), 'hermes-find-in-page-')) +app.setPath('userData', runtimeDir) +app.setPath('sessionData', runtimeDir) + +async function findCount(window, query, afterFirstResult) { + return await new Promise((resolve, reject) => { + const timeout = setTimeout(() => reject(new Error(`findInPage timed out for ${query}`)), 5000) + let requestId + let firstResult = true + + const onResult = (_event, result) => { + if (result.requestId !== requestId) return + if (firstResult) { + firstResult = false + afterFirstResult?.() + } + if (!result.finalUpdate) return + clearTimeout(timeout) + window.webContents.off('found-in-page', onResult) + resolve(result.matches) + } + + window.webContents.on('found-in-page', onResult) + requestId = window.webContents.findInPage(query) + }) +} + +async function run() { + const window = new BrowserWindow({ + show: true, + x: 0, + y: 0, + width: 320, + height: 240, + skipTaskbar: true, + opacity: 0, + webPreferences: { backgroundThrottling: false } + }) + + try { + const html = ['', '

needle

'].join('') + const fixturePath = join(runtimeDir, 'fixture.html') + writeFileSync(fixturePath, html) + await window.loadFile(fixturePath) + await window.webContents.executeJavaScript('document.body.innerText') + + assert.equal(await findCount(window, 'needle'), 2, 'control: search input is indexed') + window.webContents.stopFindInPage('clearSelection') + + await window.webContents.executeJavaScript('query.focus(); query.setSelectionRange(6, 6); query.inert = true') + const count = await findCount(window, 'needle', () => { + void window.webContents.executeJavaScript('query.inert = false; query.focus(); query.setSelectionRange(6, 6)') + }) + + assert.equal(count, 1, 'transient inert excludes the visible query from Chromium indexing') + + const state = await window.webContents.executeJavaScript(`JSON.stringify({ + type: query.type, + explicitRole: query.getAttribute('role'), + inert: query.inert, + focused: document.activeElement === query, + selectionStart: query.selectionStart, + value: query.value + })`) + assert.deepEqual(JSON.parse(state), { + type: 'search', + explicitRole: null, + inert: false, + focused: true, + selectionStart: 6, + value: 'needle' + }) + + window.webContents.debugger.attach('1.3') + try { + await window.webContents.debugger.sendCommand('Accessibility.enable') + const { nodes } = await window.webContents.debugger.sendCommand('Accessibility.getFullAXTree') + assert.ok( + nodes.some(node => node.role?.value === 'searchbox' && node.name?.value === 'Find in page'), + 'Chromium accessibility tree exposes a truthful searchbox' + ) + } finally { + window.webContents.debugger.detach() + } + } finally { + window.destroy() + } +} + +app + .whenReady() + .then(run) + .then(() => app.exit(0)) + .catch(error => { + console.error(error) + app.exit(1) + }) diff --git a/apps/desktop/electron/find-in-page.test.ts b/apps/desktop/electron/find-in-page.test.ts index e902bb4296..c238847cdb 100644 --- a/apps/desktop/electron/find-in-page.test.ts +++ b/apps/desktop/electron/find-in-page.test.ts @@ -7,9 +7,17 @@ import assert from 'node:assert/strict' import { EventEmitter } from 'node:events' +import type { BrowserWindow } from 'electron' import { describe, test } from 'vitest' -import { formatFoundInPage, installFoundInPageForwarder, performFind, stopFind } from './find-in-page' +import { + formatFoundInPage, + installFindShortcut, + installFoundInPageForwarder, + performFind, + performFindAfterIndexingStarted, + stopFind +} from './find-in-page' // Minimal webContents stub. The Electron.WebContents type is huge, so we // model just the slice the helpers touch (`isDestroyed`, `findInPage`, @@ -23,10 +31,11 @@ interface FakeWebContents { } isDestroyed: () => boolean destroy: () => void - findInPage: (query: string, options: { forward: boolean; findNext: boolean }) => void + findInPage: (query: string, options: { forward: boolean; findNext: boolean }) => number stopFindInPage: (action: 'clearSelection' | 'keepSelection' | 'activateSelection') => void send: (channel: string, payload: unknown) => void on: typeof EventEmitter.prototype.on + once: typeof EventEmitter.prototype.once off: typeof EventEmitter.prototype.off emit: (event: string | symbol, ...args: unknown[]) => boolean } @@ -51,6 +60,8 @@ function makeFakeWebContents(): FakeWebContents { }, findInPage(query: string, options: { forward: boolean; findNext: boolean }) { calls.find.push({ query, options }) + + return 17 }, stopFindInPage(action: 'clearSelection' | 'keepSelection' | 'activateSelection') { calls.stop.push(action) @@ -59,6 +70,7 @@ function makeFakeWebContents(): FakeWebContents { calls.send.push({ channel, payload }) }, on: emitter.on.bind(emitter), + once: emitter.once.bind(emitter), off: emitter.off.bind(emitter), emit: emitter.emit.bind(emitter) } @@ -136,6 +148,39 @@ describe('performFind', () => { }) }) +describe('performFindAfterIndexingStarted', () => { + test('resolves only after the matching request emits its first result', async () => { + const wc = makeFakeWebContents() + let resolved = false + + const pending = performFindAfterIndexingStarted(asWC(wc), 'needle', { + forward: true, + findNext: false + }).then(() => { + resolved = true + }) + + await Promise.resolve() + assert.equal(resolved, false) + + wc.emit('found-in-page', {}, { requestId: 9, matches: 1 }) + await Promise.resolve() + assert.equal(resolved, false) + + wc.emit('found-in-page', {}, { requestId: 17, matches: 1 }) + await pending + assert.equal(resolved, true) + }) + + test('resolves safely if the webContents is destroyed before a result', async () => { + const wc = makeFakeWebContents() + const pending = performFindAfterIndexingStarted(asWC(wc), 'needle', null) + + wc.destroy() + await pending + }) +}) + describe('stopFind', () => { test('calls stopFindInPage with the default action (clearSelection)', () => { const wc = makeFakeWebContents() @@ -220,3 +265,184 @@ describe('installFoundInPageForwarder', () => { assert.equal(wcB.calls.send.length, 0, 'wcB must not receive wcA results') }) }) + + +describe('installFindShortcut', () => { + // Minimal BrowserWindow stub: only `webContents` is touched. + function makeFakeWindow(wc: FakeWebContents) { + return { webContents: asWC(wc) } as unknown as BrowserWindow + } + + test('sends hermes:open-find-bar on Ctrl+F (Linux/Windows) and prevents default', () => { + const wc = makeFakeWebContents() + const win = makeFakeWindow(wc) + const uninstall = installFindShortcut(win) + + // Ctrl+F on Linux/Windows (no meta, no alt, no shift). + const result = wc.emit('before-input-event', {}, { + key: 'f', + control: true, + meta: false, + alt: false, + shift: false, + }) + // The listener calls preventDefault on the event; the fake's emit returns + // truthy because the event fired — what matters is the side effects. + void result + + assert.deepEqual(wc.calls.send, [ + { channel: 'hermes:open-find-bar', payload: undefined } + ]) + + uninstall() + }) + + // macOS branch: inject `isMac: () => true` so we exercise the REAL + // `meta` (Cmd) path — previously untested, because `process.platform` is + // baked at import time and the old "Cmd+F" case actually sent Ctrl. + test('sends hermes:open-find-bar on Cmd+F (meta) on macOS and prevents default', () => { + const wc = makeFakeWebContents() + const win = makeFakeWindow(wc) + const uninstall = installFindShortcut(win, () => true) + + // Cmd+F on macOS: meta held, no control/alt/shift. + wc.emit('before-input-event', {}, { + key: 'f', + control: false, + meta: true, + alt: false, + shift: false, + }) + + assert.deepEqual(wc.calls.send, [ + { channel: 'hermes:open-find-bar', payload: undefined } + ]) + + uninstall() + }) + + // The design intentionally accepts literal Ctrl on macOS too (dual-channel) + // so a non-macOS layout still works. Pin that behavior so the width of the + // chord doesn't silently drift. + test('accepts literal Ctrl+F on macOS (dual-channel with Cmd)', () => { + const wc = makeFakeWebContents() + const win = makeFakeWindow(wc) + const uninstall = installFindShortcut(win, () => true) + + // Ctrl+F with no meta on macOS still opens the FindBar. + wc.emit('before-input-event', {}, { + key: 'F', + control: true, + meta: false, + alt: false, + shift: false, + }) + + assert.deepEqual(wc.calls.send, [ + { channel: 'hermes:open-find-bar', payload: undefined } + ]) + + uninstall() + }) + + // Cross-check: a bare Ctrl+F WITHOUT meta must NOT open on Linux/Windows, + // where only `control` counts (the macOS `meta || control` widening must not + // leak across the platform boundary). + test('does NOT fire for Ctrl+F with meta only on Linux/Windows', () => { + const wc = makeFakeWebContents() + const win = makeFakeWindow(wc) + const uninstall = installFindShortcut(win, () => false) + + wc.emit('before-input-event', {}, { + key: 'f', + control: false, + meta: true, + alt: false, + shift: false, + }) + + assert.equal(wc.calls.send.length, 0, 'meta (Cmd) is not a valid chord on non-macOS') + + uninstall() + }) + + test('does NOT fire for plain F without Ctrl/Cmd', () => { + const wc = makeFakeWebContents() + const win = makeFakeWindow(wc) + const uninstall = installFindShortcut(win) + + wc.emit('before-input-event', {}, { + key: 'f', + control: false, + meta: false, + alt: false, + shift: false, + }) + + assert.equal(wc.calls.send.length, 0, 'plain F must not open the FindBar') + + uninstall() + }) + + test('does NOT fire for Ctrl+Shift+F (different chord)', () => { + const wc = makeFakeWebContents() + const win = makeFakeWindow(wc) + const uninstall = installFindShortcut(win) + + wc.emit('before-input-event', {}, { + key: 'f', + control: true, + meta: false, + alt: false, + shift: true, + }) + + assert.equal(wc.calls.send.length, 0, 'Ctrl+Shift+F is reserved (session.focusSearch)') + + uninstall() + }) + + test('does NOT fire for Ctrl+F with Alt held (combo change)', () => { + const wc = makeFakeWebContents() + const win = makeFakeWindow(wc) + const uninstall = installFindShortcut(win) + + wc.emit('before-input-event', {}, { + key: 'f', + control: true, + meta: false, + alt: true, + shift: false, + }) + + assert.equal(wc.calls.send.length, 0) + + uninstall() + }) + + test('uninstall detaches the listener', () => { + const wc = makeFakeWebContents() + const win = makeFakeWindow(wc) + const uninstall = installFindShortcut(win) + uninstall() + + wc.emit('before-input-event', {}, { + key: 'f', + control: true, + meta: false, + alt: false, + shift: false, + }) + + assert.equal(wc.calls.send.length, 0, 'listener must be detached after uninstall()') + }) + + test('is a no-op on a destroyed webContents', () => { + const wc = makeFakeWebContents() + wc.destroy() + const win = makeFakeWindow(wc) + // Should not throw — uninstall is the no-op fn returned in this branch. + const uninstall = installFindShortcut(win) + assert.doesNotThrow(() => uninstall()) + }) +}) diff --git a/apps/desktop/electron/find-in-page.ts b/apps/desktop/electron/find-in-page.ts index 509fc5dccd..57f3f901bf 100644 --- a/apps/desktop/electron/find-in-page.ts +++ b/apps/desktop/electron/find-in-page.ts @@ -71,6 +71,46 @@ export function performFind( }) } +/** + * Start a find request and resolve after Chromium emits its first matching + * result. This acknowledgment lets the renderer remove a temporary `inert` + * boundary only after the query field has been excluded from the index. + */ +export function performFindAfterIndexingStarted( + webContents: Electron.WebContents | null | undefined, + query: string, + options: FindInPageOptions | null | undefined +): Promise { + if (!webContents || webContents.isDestroyed()) { + return Promise.resolve() + } + + return new Promise(resolve => { + let requestId: number | undefined + + const finish = () => { + webContents.off('found-in-page', onFound) + webContents.off('destroyed', finish) + resolve() + } + + const onFound = (_event: Electron.Event, result: { requestId?: number }) => { + if (requestId !== undefined && result?.requestId === requestId) { + finish() + } + } + + webContents.on('found-in-page', onFound) + webContents.once('destroyed', finish) + + const opts = options && typeof options === 'object' ? options : {} + requestId = webContents.findInPage(String(query ?? ''), { + forward: opts.forward !== false, + findNext: Boolean(opts.findNext) + }) + }) +} + /** * Stop the current find and clear highlights. The default `action` matches * what the renderer sends on Escape / close. @@ -118,3 +158,72 @@ export function installFoundInPageForwarder(webContents: Electron.WebContents | webContents.off('found-in-page', handler) } } + +/** + * Install a main-process before-input-event hook that claims Ctrl/Cmd+F and + * forwards an "open the find bar" intent to the renderer. + * + * Linux only (#81727): on Pop!_OS / GNOME-based distros the Ctrl+F keydown + * does not reach the renderer's `view.findInPage` binding, so the find bar + * stays closed. Routing the chord through `before-input-event` (which Chromium + * dispatches before the DOM keydown) lets us forward the intent directly. + * The exact interception layer varies by distro/desktop (COSMIC shortcut, + * webview focus split, etc.); this sidesteps it regardless of cause by acting + * at the earliest point the keystroke is observable. + * + * On macOS / Windows the renderer's own rebindable `view.findInPage` keybind + * (`mod+f`, clearable/rebindable via the keybind registry) owns Ctrl/Cmd+F, so + * the main-process hook is NOT installed there — installing it would make the + * chord un-rebindable and double-open on a rebound binding. + * + * The renderer's existing find-in-page pipeline still does the actual work + * (it owns the FindBar UI, the store, the `hermes:find-in-page` IPC to drive + * `webContents.findInPage`). This helper just guarantees that a Ctrl/Cmd+F + * press reaches that pipeline on Linux. + * + * `isMac` is injectable so the macOS-modifier branch can be exercised by + * unit tests without rebooting the process under a different platform. + * + * Returns an uninstall fn that detaches the listener. + */ +const IS_MAC = () => process.platform === 'darwin' + +export function installFindShortcut(window: Electron.BrowserWindow, isMac: () => boolean = IS_MAC): () => void { + const { webContents } = window + if (!webContents || webContents.isDestroyed()) { + return () => {} + } + + const handler = (event: Electron.Event, input: Electron.Input) => { + if (!webContents || webContents.isDestroyed()) { + return + } + const key = String(input.key || '').toLowerCase() + // Accept the platform's primary accelerator (Cmd on macOS, Ctrl elsewhere) + // AND literal Ctrl on macOS so the chord still reaches us when the user + // is on a non-macOS layout. On Pop!_OS / GNOME the GTK compositor owns + // Ctrl+F before the renderer's keydown fires — this main-process handler + // runs strictly before that (#81727). + const hasMod = isMac() ? input.meta || input.control : input.control + const isFindChord = + key === 'f' && + hasMod && + !input.alt && + !input.shift + if (!isFindChord) { + return + } + if (typeof event.preventDefault === 'function') { + event.preventDefault() + } + webContents.send('hermes:open-find-bar') + } + + webContents.on('before-input-event', handler) + + return () => { + if (!webContents.isDestroyed()) { + webContents.off('before-input-event', handler) + } + } +} diff --git a/apps/desktop/electron/gateway-file-download-transport.test.ts b/apps/desktop/electron/gateway-file-download-transport.test.ts new file mode 100644 index 0000000000..8ade7ec2a6 --- /dev/null +++ b/apps/desktop/electron/gateway-file-download-transport.test.ts @@ -0,0 +1,71 @@ +/** + * Wiring coverage for the main.ts gateway download transports. These functions + * pull in main-process singletons (https/http, electronNet, the OAuth session, + * the save dialog), so we assert on their source shape — the same approach as + * oauth-session-request.test.ts — while gateway-file-download.test.ts unit-tests + * the extracted streaming/decoding logic behaviorally. + */ + +import assert from 'node:assert/strict' +import fs from 'node:fs' +import path from 'node:path' +import { fileURLToPath } from 'node:url' + +import { test } from 'vitest' + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const source = fs.readFileSync(path.join(__dirname, 'main.ts'), 'utf8') + +function extract(startMarker: string, endMarker: string): string { + const start = source.indexOf(startMarker) + assert.notEqual(start, -1, `${startMarker} should exist`) + const end = source.indexOf(endMarker, start + startMarker.length) + assert.notEqual(end, -1, `boundary after ${startMarker} should exist`) + + return source.slice(start, end) +} + +test('token transport streams to disk instead of buffering the whole body', () => { + const fn = extract('function downloadViaTokenToFile', '\nfunction ') + + // Delegates byte-moving to the streaming finalizer... + assert.match(fn, /finalizeGatewayDownload\(/) + // ...and must NOT accumulate the full response before writing. + assert.doesNotMatch(fn, /Buffer\.concat/) + assert.doesNotMatch(fn, /chunks\.push/) + // Idle timeout is dropped once headers arrive so the dialog/stream isn't killed. + assert.match(fn, /setTimeout\(0\)/) +}) + +test('oauth transport streams to disk instead of buffering the whole body', () => { + const fn = extract('function downloadViaOauthSessionToFile', '\nasync function finalizeGatewayDownload') + + assert.match(fn, /electronNet\.request/) + assert.match(fn, /finalizeGatewayDownload\(/) + assert.doesNotMatch(fn, /Buffer\.concat/) + assert.doesNotMatch(fn, /chunks\.push/) +}) + +test('finalizeGatewayDownload prompts a save dialog then streams the response', () => { + const fn = extract('async function finalizeGatewayDownload', '\nfunction readGatewayErrorText') + + assert.match(fn, /dialog\.showSaveDialog/) + assert.match(fn, /pumpStreamToFile\(/) + // HTTP errors carry their status so a 404 can trigger the fallback. + assert.match(fn, /error\.statusCode = statusCode/) +}) + +test('saveGatewayFile falls back to the data-url route only on 404', () => { + const fn = extract('async function saveGatewayFile', '\nasync function saveGatewayFileViaDataUrl') + + assert.match(fn, /\/api\/fs\/download\?path=/) + assert.match(fn, /isNotFoundError\(error\)/) + assert.match(fn, /saveGatewayFileViaDataUrl\(/) +}) + +test('data-url fallback reads the capped route and decodes it', () => { + const fn = extract('async function saveGatewayFileViaDataUrl', '// Mint a single-use WS ticket') + + assert.match(fn, /\/api\/fs\/read-data-url\?path=/) + assert.match(fn, /parseDataUrlToBuffer\(/) +}) diff --git a/apps/desktop/electron/gateway-file-download.test.ts b/apps/desktop/electron/gateway-file-download.test.ts new file mode 100644 index 0000000000..a9d054b1fa --- /dev/null +++ b/apps/desktop/electron/gateway-file-download.test.ts @@ -0,0 +1,190 @@ +import assert from 'node:assert/strict' +import { EventEmitter } from 'node:events' + +import { test } from 'vitest' + +import { + filenameFromContentDisposition, + gatewayFilePath, + isNotFoundError, + parseDataUrlToBuffer, + pumpStreamToFile +} from './gateway-file-download' + +// A Readable-like response driven manually in tests. +class FakeResponse extends EventEmitter { + paused = false + resumed = false + destroyed = false + + pause() { + this.paused = true + } + + resume() { + this.resumed = true + } + + destroy() { + this.destroyed = true + } +} + +// A write stream that records writes and lets tests control backpressure. +class FakeWriteStream extends EventEmitter { + chunks: Buffer[] = [] + ended = false + destroyed = false + private writeReturns: boolean[] + + constructor(writeReturns: boolean[] = []) { + super() + this.writeReturns = writeReturns + } + + write(chunk: Buffer): boolean { + this.chunks.push(chunk) + + return this.writeReturns.length ? this.writeReturns.shift()! : true + } + + end(cb: () => void) { + this.ended = true + cb() + } + + destroy() { + this.destroyed = true + } +} + +test('pumpStreamToFile streams chunks to the destination without buffering the whole body', async () => { + const res = new FakeResponse() + const ws = new FakeWriteStream() + const unlinked: string[] = [] + + const promise = pumpStreamToFile(res as never, '/tmp/out.bin', { + createWriteStream: () => ws as never, + unlink: async p => { + unlinked.push(p) + } + }) + + res.emit('data', Buffer.from('abc')) + res.emit('data', Buffer.from('def')) + res.emit('end') + + await promise + + assert.equal(Buffer.concat(ws.chunks).toString('utf8'), 'abcdef') + assert.equal(ws.ended, true) + assert.deepEqual(unlinked, []) // success -> no cleanup +}) + +test('pumpStreamToFile applies backpressure: pauses on a full buffer and resumes on drain', async () => { + const res = new FakeResponse() + const ws = new FakeWriteStream([false]) // first write signals "buffer full" + + const promise = pumpStreamToFile(res as never, '/tmp/out.bin', { + createWriteStream: () => ws as never, + unlink: async () => {} + }) + + res.emit('data', Buffer.from('big-chunk')) + assert.equal(res.paused, true, 'source should be paused when write() returns false') + assert.equal(res.resumed, false) + + ws.emit('drain') + assert.equal(res.resumed, true, 'source should resume after the write stream drains') + + res.emit('end') + await promise +}) + +test('pumpStreamToFile unlinks the partial file and rejects on a write error', async () => { + const res = new FakeResponse() + const ws = new FakeWriteStream() + const unlinked: string[] = [] + + const promise = pumpStreamToFile(res as never, '/tmp/partial.bin', { + createWriteStream: () => ws as never, + unlink: async p => { + unlinked.push(p) + } + }) + + res.emit('data', Buffer.from('abc')) + ws.emit('error', new Error('ENOSPC: disk full')) + + await assert.rejects(promise, /disk full/) + assert.deepEqual(unlinked, ['/tmp/partial.bin']) + assert.equal(res.destroyed, true, 'source should be torn down on write failure') +}) + +test('pumpStreamToFile unlinks the partial file and rejects on a response error', async () => { + const res = new FakeResponse() + const ws = new FakeWriteStream() + const unlinked: string[] = [] + + const promise = pumpStreamToFile(res as never, '/tmp/partial.bin', { + createWriteStream: () => ws as never, + unlink: async p => { + unlinked.push(p) + } + }) + + res.emit('data', Buffer.from('abc')) + res.emit('error', new Error('socket hang up')) + + await assert.rejects(promise, /socket hang up/) + assert.deepEqual(unlinked, ['/tmp/partial.bin']) +}) + +test('parseDataUrlToBuffer decodes base64 payloads', () => { + const buffer = parseDataUrlToBuffer('data:text/markdown;base64,IyByZXBvcnQ=') + + assert.equal(buffer.toString('utf8'), '# report') +}) + +test('parseDataUrlToBuffer decodes percent-encoded (non-base64) payloads', () => { + const buffer = parseDataUrlToBuffer('data:text/plain,hello%20world') + + assert.equal(buffer.toString('utf8'), 'hello world') +}) + +test('parseDataUrlToBuffer throws on a malformed data URL', () => { + assert.throws(() => parseDataUrlToBuffer('not-a-data-url'), /Malformed data URL/) +}) + +test('filenameFromContentDisposition prefers filename* and reduces to a basename', () => { + assert.equal( + filenameFromContentDisposition("attachment; filename*=UTF-8''report%20with%20spaces.pdf"), + 'report with spaces.pdf' + ) + assert.equal(filenameFromContentDisposition('attachment; filename="report.md"'), 'report.md') + // A traversal attempt in the header cannot escape the chosen directory. + assert.equal(filenameFromContentDisposition('attachment; filename="../../etc/passwd"'), 'passwd') + assert.equal(filenameFromContentDisposition(''), '') + assert.equal(filenameFromContentDisposition(undefined), '') +}) + +test('gatewayFilePath normalizes bare paths and file:// URLs', () => { + assert.equal(gatewayFilePath('/Users/me/report.md'), '/Users/me/report.md') + assert.equal(gatewayFilePath('file:///Users/me/a%20b.md'), '/Users/me/a b.md') + assert.equal(gatewayFilePath(''), '') + assert.equal(gatewayFilePath(null), '') +}) + +test('isNotFoundError matches only HTTP 404', () => { + const notFound: any = new Error('404: missing') + + notFound.statusCode = 404 + assert.equal(isNotFoundError(notFound), true) + + const forbidden: any = new Error('403: nope') + + forbidden.statusCode = 403 + assert.equal(isNotFoundError(forbidden), false) + assert.equal(isNotFoundError(new Error('plain')), false) + assert.equal(isNotFoundError(null), false) +}) diff --git a/apps/desktop/electron/gateway-file-download.ts b/apps/desktop/electron/gateway-file-download.ts new file mode 100644 index 0000000000..acdca9723c --- /dev/null +++ b/apps/desktop/electron/gateway-file-download.ts @@ -0,0 +1,167 @@ +// Helpers for saving a gateway-hosted file to the local disk from the Electron +// main process. Extracted from main.ts so the streaming, data-URL decoding, and +// filename derivation are unit-testable without spinning up Electron. +// +// The transport wrappers (token / OAuth) live in main.ts because they need +// main-process singletons (https/http, electronNet, the OAuth session). They +// delegate the byte-moving to `pumpStreamToFile` here, which streams the +// response to a user-selected destination with backpressure and cleans up a +// partial file on error — so a large download never has to be buffered whole in +// the native process. + +import path from 'node:path' + +// Minimal shape of the response objects we consume. Both Node's +// http.IncomingMessage and Electron net's IncomingMessage satisfy it. +export interface ReadableLike { + on(event: 'data', listener: (chunk: Buffer | Uint8Array | string) => void): unknown + on(event: 'end', listener: () => void): unknown + on(event: 'error', listener: (err: Error) => void): unknown + pause?: () => void + resume?: () => void + destroy?: (err?: Error) => void +} + +export interface WriteStreamLike { + write(chunk: Buffer): boolean + end(cb: () => void): void + destroy(err?: Error): void + on(event: 'error', listener: (err: Error) => void): unknown + once(event: 'drain', listener: () => void): unknown +} + +export interface PumpDeps { + createWriteStream: (destPath: string) => WriteStreamLike + unlink: (destPath: string) => Promise +} + +// Stream `res` into `destPath`, honoring backpressure. On any read/write error +// the write stream is torn down and the (partial) destination file is removed +// before the returned promise rejects, so a failed download never leaves a +// truncated file behind. +export function pumpStreamToFile(res: ReadableLike, destPath: string, deps: PumpDeps): Promise { + return new Promise((resolve, reject) => { + const ws = deps.createWriteStream(destPath) + let failed = false + + const fail = (err: Error) => { + if (failed) { + return + } + + failed = true + + try { + res.destroy?.(err) + } catch { + // best effort — the socket may already be closed + } + + try { + ws.destroy() + } catch { + // best effort + } + + Promise.resolve(deps.unlink(destPath)) + .catch(() => {}) + .then(() => reject(err)) + } + + ws.on('error', fail) + res.on('error', fail) + + res.on('data', chunk => { + if (failed) { + return + } + + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array) + const ok = ws.write(buffer) + + // Backpressure: pause the source until the file stream drains so we never + // accumulate the whole payload in memory. + if (!ok && typeof res.pause === 'function') { + res.pause() + ws.once('drain', () => { + if (!failed) { + res.resume?.() + } + }) + } + }) + + res.on('end', () => { + if (failed) { + return + } + + ws.end(() => resolve()) + }) + }) +} + +// Decode a `data:[][;base64],` URL into a Buffer. Used by the +// compatibility fallback that reads through the capped `/api/fs/read-data-url` +// route when the gateway predates `/api/fs/download`. +export function parseDataUrlToBuffer(dataUrl: string): Buffer { + const match = /^data:([^,]*),([\s\S]*)$/.exec(String(dataUrl || '')) + + if (!match) { + throw new Error('Malformed data URL') + } + + const meta = match[1] || '' + const payload = match[2] || '' + + if (/;base64/i.test(meta)) { + return Buffer.from(payload, 'base64') + } + + return Buffer.from(decodeURIComponent(payload), 'utf8') +} + +// Extract a filename from a Content-Disposition header, preferring the RFC 5987 +// `filename*` form. Returns '' when none is present. Always reduced to a +// basename so a malicious header can't redirect the save outside the picked dir. +export function filenameFromContentDisposition(value: unknown): string { + const text = String(value || '') + const encoded = text.match(/filename\*=(?:UTF-8'')?([^;]+)/i)?.[1] + const plain = text.match(/filename="?([^";]+)"?/i)?.[1] + const raw = encoded || plain || '' + + if (!raw) { + return '' + } + + try { + return path.basename(decodeURIComponent(raw.trim())) + } catch { + return path.basename(raw.trim()) + } +} + +// Normalize a gateway file path that may arrive as a bare path or a file:// URL. +export function gatewayFilePath(rawPath: unknown): string { + const value = String(rawPath || '').trim() + + if (!value) { + return '' + } + + if (!/^file:/i.test(value)) { + return value + } + + try { + return decodeURIComponent(new URL(value).pathname) + } catch { + return value.replace(/^file:\/\//i, '') + } +} + +// True when an error thrown by a transport wrapper represents an HTTP 404, used +// to trigger the data-URL compatibility fallback (and nothing else). +export function isNotFoundError(error: unknown): boolean { + return Boolean(error) && (error as { statusCode?: number }).statusCode === 404 +} diff --git a/apps/desktop/electron/get-windows.d.ts b/apps/desktop/electron/get-windows.d.ts new file mode 100644 index 0000000000..444e7f9dd2 --- /dev/null +++ b/apps/desktop/electron/get-windows.d.ts @@ -0,0 +1,22 @@ +// Type declarations for the get-windows optionalDependency. +// +// get-windows ships no bundled types and is an optionalDependency. `npm ci` +// can skip it when its native install fails, including Linux and Windows ARM64 +// where 9.3.0 has no prebuilt, so it can legitimately be absent from +// node_modules. Declaring the module here keeps typecheck independent of +// whether the package installed. The runtime import in window-below.ts +// degrades to null when it is absent. + +declare module 'get-windows' { + export interface GetWindowsWindow { + bounds?: { height?: number; width?: number; x?: number; y?: number } + id?: number + owner?: { name?: string; processId?: number } + title?: string + } + + export function openWindows(options?: { + accessibilityPermission?: boolean + screenRecordingPermission?: boolean + }): Promise +} diff --git a/apps/desktop/electron/git-worktree-ops.test.ts b/apps/desktop/electron/git-worktree-ops.test.ts index 43af70e83d..d3cf6a19e9 100644 --- a/apps/desktop/electron/git-worktree-ops.test.ts +++ b/apps/desktop/electron/git-worktree-ops.test.ts @@ -435,3 +435,38 @@ test('addWorktree: a remote default branch gets its own worktree, not a home swi fs.rmSync(cloneDir, { recursive: true, force: true }) } }) + +test('switchBranch: non-repo dir short-circuits instead of throwing', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-sw-')) + + try { + // A plain folder pinned as a project (no .git): its lane label is the + // folder basename, not a branch — switching must no-op, not error, so + // callers like "+" new session can proceed with a plain session. + const result = await switchBranch(dir, '国创大赛', 'git') + + assert.deepEqual(result, { branch: null }) + } finally { + fs.rmSync(dir, { recursive: true, force: true }) + } +}) + +test('switchBranch: repo dir still validates the branch name and switches', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'hermes-sw-')) + + try { + execFileSync('git', ['init', '-b', 'main'], { cwd: dir }) + execFileSync('git', ['config', 'user.email', 't@example.com'], { cwd: dir }) + execFileSync('git', ['config', 'user.name', 'test'], { cwd: dir }) + execFileSync('git', ['commit', '--allow-empty', '-m', 'root'], { cwd: dir }) + + // Existing behaviour preserved: an illegal branch name still errors. + await assert.rejects(() => switchBranch(dir, '///', 'git'), /Branch name is required/) + + // And switching to a real branch still works. + const result = await switchBranch(dir, 'main', 'git') + assert.deepEqual(result, { branch: 'main' }) + } finally { + fs.rmSync(dir, { recursive: true, force: true }) + } +}) diff --git a/apps/desktop/electron/git-worktree-ops.ts b/apps/desktop/electron/git-worktree-ops.ts index 2ff43862c3..384e8c3fbe 100644 --- a/apps/desktop/electron/git-worktree-ops.ts +++ b/apps/desktop/electron/git-worktree-ops.ts @@ -437,6 +437,25 @@ async function listBranches(repoPath, gitBin) { async function switchBranch(repoPath, branch, gitBin) { const resolved = resolveRequestedPathForIpc(repoPath, { purpose: 'Branch switch' }) + + // Sidebar lanes exist for plain folders too (non-repo explicit projects), + // and their lane label is the folder basename — not a branch. `git switch` + // there is meaningless, and sanitizing that label would throw a misleading + // "Branch name is required." — so short-circuit for non-repo roots and let + // callers (e.g. "+" new session on the project lane) proceed with a plain + // session instead of aborting. + let inside = 'false' + + try { + inside = (await runGit(gitBin, ['rev-parse', '--is-inside-work-tree'], resolved)).trim() + } catch { + // Not a git repo (or git unavailable): fall through to the short-circuit. + } + + if (inside !== 'true') { + return { branch: null } + } + const target = sanitizeBranch(branch) if (!target) { diff --git a/apps/desktop/electron/image-context-menu.test.ts b/apps/desktop/electron/image-context-menu.test.ts new file mode 100644 index 0000000000..c36de87b07 --- /dev/null +++ b/apps/desktop/electron/image-context-menu.test.ts @@ -0,0 +1,81 @@ +import assert from 'node:assert/strict' + +import { test } from 'vitest' + +import { imageContextMenuItems } from './image-context-menu' + +function createActions() { + const calls = { + copyImageAt: [], + openImage: [], + copyImageAddress: [], + saveImage: [] + } + + return { + calls, + actions: { + copyImageAt: (x, y) => calls.copyImageAt.push([x, y]), + openImage: url => calls.openImage.push(url), + copyImageAddress: url => calls.copyImageAddress.push(url), + saveImage: url => calls.saveImage.push(url) + } + } +} + +test('keeps Copy Image available when Chromium omits a large image srcURL', () => { + const { actions, calls } = createActions() + + const items = imageContextMenuItems( + { mediaType: 'image', hasImageContents: true, srcURL: '', x: 100, y: 120 }, + actions + ) + + assert.deepEqual( + items.map(item => item.label), + ['Copy Image'] + ) + + items[0].click() + assert.deepEqual(calls.copyImageAt, [[100, 120]]) +}) + +test('keeps URL-dependent image actions when srcURL is available', () => { + const { actions, calls } = createActions() + const url = 'https://example.com/image.png' + + const items = imageContextMenuItems({ mediaType: 'image', hasImageContents: true, srcURL: url, x: 5, y: 8 }, actions) + + assert.deepEqual( + items.map(item => item.label), + ['Open Image', 'Copy Image', 'Copy Image Address', 'Save Image As...'] + ) + + items[0].click() + items[1].click() + items[2].click() + items[3].click() + + assert.deepEqual(calls.openImage, [url]) + assert.deepEqual(calls.copyImageAt, [[5, 8]]) + assert.deepEqual(calls.copyImageAddress, [url]) + assert.deepEqual(calls.saveImage, [url]) +}) + +test('does not add image actions for a non-image target', () => { + const { actions } = createActions() + + assert.deepEqual( + imageContextMenuItems({ mediaType: 'none', hasImageContents: false, srcURL: '', x: 0, y: 0 }, actions), + [] + ) +}) + +test('does not offer Copy Image when the target has no decoded image contents', () => { + const { actions } = createActions() + + assert.deepEqual( + imageContextMenuItems({ mediaType: 'image', hasImageContents: false, srcURL: '', x: 0, y: 0 }, actions), + [] + ) +}) diff --git a/apps/desktop/electron/image-context-menu.ts b/apps/desktop/electron/image-context-menu.ts new file mode 100644 index 0000000000..de817024c0 --- /dev/null +++ b/apps/desktop/electron/image-context-menu.ts @@ -0,0 +1,40 @@ +export function imageContextMenuItems(params, actions) { + if (params.mediaType !== 'image' || !params.hasImageContents) { + return [] + } + + const items = [] + const srcURL = params.srcURL || '' + + if (srcURL) { + items.push({ + label: 'Open Image', + click: () => { + if (!srcURL.startsWith('data:')) { + actions.openImage(srcURL) + } + }, + enabled: !srcURL.startsWith('data:') + }) + } + + items.push({ + label: 'Copy Image', + click: () => actions.copyImageAt(params.x, params.y) + }) + + if (srcURL) { + items.push( + { + label: 'Copy Image Address', + click: () => actions.copyImageAddress(srcURL) + }, + { + label: 'Save Image As...', + click: () => actions.saveImage(srcURL) + } + ) + } + + return items +} diff --git a/apps/desktop/electron/main.ts b/apps/desktop/electron/main.ts index 9cd3a13a17..2aeb23c6cc 100644 --- a/apps/desktop/electron/main.ts +++ b/apps/desktop/electron/main.ts @@ -17,7 +17,6 @@ import { globalShortcut, ipcMain, Menu, - nativeImage, nativeTheme, Notification, powerMonitor, @@ -37,6 +36,7 @@ import { dashboardFallbackArgs, sourceDeclaresServe } from './backend-command' import { createBackendConnectionState } from './backend-connection-state' import { buildDesktopBackendEnv, hermesManagedNodePathEntries, normalizeHermesHomeRoot } from './backend-env' import { isReauthRequiredError, waitForHermesReady } from './backend-health' +import { backendCommandMatches, createBackendOwnership, createBackendShutdownCoordinator } from './backend-ownership' import { canImportHermesCli, execProbeSync, @@ -61,6 +61,7 @@ import { buildGatewayWsUrlWithTicket, connectionScopeKey, cookiesHaveLiveSession, + cookiesHavePrivyAccessToken, cookiesHavePrivySession, cookiesHaveSession, gatewayTicketFailure, @@ -81,9 +82,21 @@ import { savedProfileSsh, tokenPreview } from './connection-config' +import { + backendScopeKey, + backendScopePrefix, + mergeConnectionInput, + migrateV1ToRegistry, + normalizeConnectionInput, + normalizeRegistry, + removeConnection, + setPrimaryConnection, + upsertConnection +} from './connection-registry' import { describeCrashReason, installCrashForensics } from './crash-forensics' import { adoptServedDashboardToken } from './dashboard-token' import { loadOrCreateInstallationId, sshOwnershipId } from './desktop-installation' +import { formatDesktopLogLine } from './desktop-log-line' import { buildPosixCleanupScript, buildWindowsCleanupScript, @@ -96,10 +109,24 @@ import { import { describeDevCdpDecision, resolveDevCdpPort } from './dev-cdp' import { installEmbedReferer } from './embed-referer' import { createEventDeduper } from './event-dedupe' +import { + buildTerminalScript, + resolveTerminalLaunch, + terminalScriptEnv, + terminalScriptExtension, + tuiResumeArgs +} from './external-terminal' import { findGitBash as _findGitBash } from './find-git-bash' -import { installFoundInPageForwarder, performFind, stopFind } from './find-in-page' +import { installFindShortcut, installFoundInPageForwarder, performFindAfterIndexingStarted, stopFind } from './find-in-page' import { createFirstRunSetupGate } from './first-run-setup-gate' import { readDirForIpc } from './fs-read-dir' +import { + filenameFromContentDisposition, + gatewayFilePath, + isNotFoundError, + parseDataUrlToBuffer, + pumpStreamToFile +} from './gateway-file-download' import { probeGatewayWebSocket } from './gateway-ws-probe' import { scanGitRepos } from './git-repo-scan' import { @@ -151,8 +178,10 @@ import { cursorPointInWindow } from './hud-cursor' import { snapHudBounds } from './hud-snap' import { createHudSnapShortcut } from './hud-snap-shortcut' import { buildHudWindowUrl } from './hud-url' +import { imageContextMenuItems } from './image-context-menu' import { createLinkTitleWindow, guardLinkTitleSession, readLinkTitleWindowTitle } from './link-title-window' import { ensureMainWindow } from './main-window-lifecycle' +import { createMediaProtocolHandler, MEDIA_PROTOCOL } from './media-protocol' import { oauthGuardMayHardFail, oauthSessionIsLive, @@ -174,7 +203,11 @@ import { createKeepAwake } from './power-save' import { FirstRunSetupResetError, runPrimaryBackendStartup } from './primary-backend-startup' import { rehomePrimaryConnection } from './primary-connection-rehome' import { decideProfileDeleteAction, profileNameFromDeleteRequest, resolveRouteProfile } from './profile-delete-routing' -import { fetchPrimaryProfileSessions } from './profile-session-routing' +import { + fetchPrimaryProfileSessions, + fetchRemoteProfileSessions, + mergeProfileSessionWindow +} from './profile-session-routing' import { createQuickEntryShortcut, quickEntryWindowBounds, sanitizeQuickEntrySettings } from './quick-entry' import { type ActiveWork, mergeActiveWork, normalizeActiveWork, quitPromptFor } from './quit-guard' import * as remoteLifecycle from './remote-lifecycle' @@ -184,6 +217,7 @@ import { revalidatePooledRemoteBackends, revalidateRemoteConnection } from './remote-liveness' +import { missingRendererAssets } from './renderer-bundle' import { attachRendererConsoleCapture, formatRendererBoundaryReport } from './renderer-log' import { buildSessionWindowUrl, @@ -205,12 +239,19 @@ import { } from './ssh-connection' import { createStreamThrottle } from './stream-throttle' import { nativeOverlayWidth as computeNativeOverlayWidth, macTitleBarOverlayHeight } from './titlebar-overlay-width' -import { resolveBehindCount, shouldCountCommits } from './update-count' +import { + compareApiUrl, + parseCompareBehindCount, + resolveBehindCount, + resolveCommitLogSelection, + shouldCountCommits +} from './update-count' import { waitForUpdateClearance } from './update-gate' import { readLiveUpdateMarker, updateHandoffConflict, writeUpdateMarker } from './update-marker' import { isOfficialSshRemote, OFFICIAL_REPO_HTTPS_URL } from './update-remote' import { collectRelaunchArgs, + observeUpdaterHandoff, resolvePosixScriptHandoff, resolveStagedUpdaterBinary, resolveUpdateScriptHandoff, @@ -284,6 +325,10 @@ const IS_WSL = isWslEnvironment() const DARWIN_MAJOR = IS_MAC ? Number.parseInt(os.release(), 10) || 0 : 0 const APP_ROOT = app.getAppPath() +// Device-local preference: block F12 from opening DevTools. +// Set dynamically via IPC from the renderer Settings → Advanced. +let f12Blocked = false + // Preload must be plain JS — Electron's sandbox can't run .ts, and tsx's // ESM loader is broken on Electron 40's Node (ERR_INVALID_RETURN_PROPERTY_VALUE). // Dev (`npm run dev`) and prod both load the esbuild output from dist/. @@ -453,7 +498,7 @@ if (IS_WINDOWS) { try { app.relaunch({ args: buildNoSandboxRelaunchArgs(process.argv.slice(1)) }) - app.exit(0) + void exitAfterBackendShutdown(0) } catch (error) { console.error(`[hermes] --no-sandbox relaunch failed: ${error?.message || error}`) } @@ -638,9 +683,15 @@ const BOOTSTRAP_COMPLETE_MARKER = path.join(ACTIVE_HERMES_ROOT, '.hermes-bootstr const BOOTSTRAP_MARKER_SCHEMA_VERSION = 1 const DESKTOP_CONNECTION_CONFIG_PATH = path.join(app.getPath('userData'), 'connection.json') +// v2 multi-connection registry (named agent sources). Lives BESIDE +// connection.json — v1 stays on disk untouched so older builds sharing the +// profile keep working; the registry imports from it once and then owns its +// own file. Same secret posture as connection.json (encrypted tokens, 0600). +const DESKTOP_CONNECTIONS_REGISTRY_PATH = path.join(app.getPath('userData'), 'connections.json') const DESKTOP_INSTALLATION_PATH = path.join(app.getPath('userData'), 'desktop-installation.json') const DESKTOP_UPDATE_CONFIG_PATH = path.join(app.getPath('userData'), 'updates.json') const DESKTOP_WINDOW_STATE_PATH = path.join(app.getPath('userData'), 'window-state.json') +const DESKTOP_BACKEND_OWNERSHIP_PATH = path.join(app.getPath('userData'), 'backend-ownership.json') // active-profile.json records which Hermes profile the desktop launches its // local backend as. When set, startHermes() passes `hermes --profile // dashboard …`, which deterministically pins HERMES_HOME (see @@ -1019,30 +1070,10 @@ app.setAboutPanelOptions({ copyright: 'Copyright © 2026 Nous Research' }) -// Custom scheme for streaming local media (video/audio) into the renderer. -// Reading large media through `readFileDataUrl` failed: it base64-loads the -// whole file into memory and is hard-capped (default 16 MB, Settings → Chat), -// so any non-trivial video silently refused to load. Streaming via a protocol -// handler removes the size cap and gives the