Merge origin/main (561b053f79) into simp/forwardport: forward-port 220 main commits into the simplified tree
This commit is contained in:
@@ -1588,7 +1588,11 @@ def create_openai_client(agent, client_kwargs: dict, *, reason: str, shared: boo
|
||||
# TCP keepalives so dead provider connections are detected (~60s) instead of hanging in
|
||||
# CLOSE-WAIT. Injected into the local copy only, so each client gets its own httpx.Client;
|
||||
# pinned by tests/run_agent/test_create_openai_client_reuse.py and
|
||||
# test_sequential_chats_live.py.
|
||||
# test_sequential_chats_live.py. What IS shared across those per-client wrappers is the
|
||||
# connection pool: ``build_keepalive_http_client`` mounts a process-shared ``HTTPTransport``
|
||||
# behind a per-client view whose ``close()`` is a no-op for the pool, so a closed wrapper
|
||||
# never takes a sibling's (or the successor's) connections with it
|
||||
# (tests/agent/test_shared_http_transport.py).
|
||||
if "http_client" not in client_kwargs:
|
||||
keepalive_http = agent._build_keepalive_http_client(client_kwargs.get("base_url", ""), verify=httpx_verify)
|
||||
if keepalive_http is not None:
|
||||
@@ -2678,10 +2682,15 @@ def reapply_reasoning_echo_for_provider(agent, api_messages: list) -> int:
|
||||
return reapply_reasoning_echo(api_messages, agent._needs_thinking_reasoning_pad())
|
||||
|
||||
|
||||
def _iter_httpx_pool_objects(http_client: Any):
|
||||
"""Yield httpcore pool objects reachable from an httpx client, including mounted transports:
|
||||
def _iter_httpx_pools_with_owner(http_client: Any):
|
||||
"""Yield ``(pool, owner)`` pairs reachable from an httpx client, including mounted transports:
|
||||
keepalive and proxy configs put live connections on ``client._mounts``, which a
|
||||
``_transport``-only walk misses."""
|
||||
``_transport``-only walk misses.
|
||||
|
||||
``owner`` is ``None`` for a pool this client owns outright, or the ``_SharedTransport`` view
|
||||
id when the pool is process-shared with other clients
|
||||
(``process_bootstrap.build_keepalive_http_client``). Callers must then touch only the
|
||||
in-flight requests stamped with that owner."""
|
||||
seen_pools: set[int] = set()
|
||||
try:
|
||||
transports = [getattr(http_client, "_transport", None)]
|
||||
@@ -2696,11 +2705,18 @@ def _iter_httpx_pool_objects(http_client: Any):
|
||||
pool = transport
|
||||
if pool is not None and id(pool) not in seen_pools:
|
||||
seen_pools.add(id(pool))
|
||||
yield pool
|
||||
owner = id(transport) if type(transport).__name__ == "_SharedTransport" else None
|
||||
yield pool, owner
|
||||
except Exception:
|
||||
return
|
||||
|
||||
|
||||
def _iter_httpx_pool_objects(http_client: Any):
|
||||
"""Yield httpcore pool objects reachable from an httpx client."""
|
||||
for pool, _owner in _iter_httpx_pools_with_owner(http_client):
|
||||
yield pool
|
||||
|
||||
|
||||
def _connection_candidates(conn: Any):
|
||||
"""Walk nested ``_connection`` wrappers (proxy tunnel → HTTP11/2)."""
|
||||
seen: set[int] = set()
|
||||
@@ -2734,20 +2750,30 @@ def _iter_pool_sockets(client: Any):
|
||||
try:
|
||||
# Some SDK wrappers *are* the httpx client; fall through so mount-aware discovery runs.
|
||||
http_client = getattr(client, "_client", None)
|
||||
pools = list(_iter_httpx_pool_objects(client if http_client is None else http_client))
|
||||
pools = list(_iter_httpx_pools_with_owner(client if http_client is None else http_client))
|
||||
except Exception:
|
||||
return
|
||||
if not pools:
|
||||
return
|
||||
from agent.process_bootstrap import HERMES_TRANSPORT_OWNER_EXT
|
||||
seen: set[int] = set()
|
||||
for pool in pools:
|
||||
for pool, owner in pools:
|
||||
# ``is None``, not falsiness: an empty ``_connections`` must still let us walk in-flight ``_requests``.
|
||||
raw_conns = getattr(pool, "_connections", None)
|
||||
if raw_conns is None:
|
||||
raw_conns = getattr(pool, "_pool", None)
|
||||
connections = list(raw_conns or [])
|
||||
connections += [
|
||||
c for c in (getattr(r, "connection", None) for r in list(getattr(pool, "_requests", None) or []))
|
||||
if c is not None
|
||||
]
|
||||
# A process-shared pool carries other clients' idle + in-flight connections: only this
|
||||
# client's own in-flight requests (stamped by ``_SharedTransport.handle_request``) may be
|
||||
# shut down.
|
||||
connections = [] if owner is not None else list(raw_conns or [])
|
||||
for pool_req in list(getattr(pool, "_requests", None) or []):
|
||||
if owner is not None:
|
||||
exts = getattr(getattr(pool_req, "request", None), "extensions", None) or {}
|
||||
if exts.get(HERMES_TRANSPORT_OWNER_EXT) != owner:
|
||||
continue
|
||||
conn = getattr(pool_req, "connection", None)
|
||||
if conn is not None:
|
||||
connections.append(conn)
|
||||
for conn in connections:
|
||||
for candidate in _connection_candidates(conn):
|
||||
stream = getattr(candidate, "_network_stream", None) or getattr(candidate, "_stream", None)
|
||||
|
||||
@@ -1322,6 +1322,10 @@ class _CodexCompletionsAdapter:
|
||||
timeout = kwargs.get("timeout")
|
||||
if timeout is not None:
|
||||
resp_kwargs["timeout"] = timeout
|
||||
# Per-request HTTP headers (OpenCode session affinity, Copilot x-initiator) map to real
|
||||
# headers via the SDK kwarg — forward them.
|
||||
if isinstance(kwargs.get("extra_headers"), dict) and kwargs["extra_headers"]:
|
||||
resp_kwargs["extra_headers"] = dict(kwargs["extra_headers"])
|
||||
# The Codex endpoint rejects max_output_tokens/temperature (400) — omit.
|
||||
extra_body = kwargs.get("extra_body") or {}
|
||||
if isinstance(extra_body, dict):
|
||||
@@ -1573,6 +1577,13 @@ class _AnthropicCompletionsAdapter:
|
||||
from agent.anthropic_adapter import _forbids_sampling_params
|
||||
if not _forbids_sampling_params(model):
|
||||
anthropic_kwargs["temperature"] = temperature
|
||||
# Per-request HTTP headers (OpenCode session affinity) — the Anthropic SDK accepts
|
||||
# ``extra_headers`` on messages.create/stream too.
|
||||
if isinstance(kwargs.get("extra_headers"), dict) and kwargs["extra_headers"]:
|
||||
anthropic_kwargs["extra_headers"] = {
|
||||
**(anthropic_kwargs.get("extra_headers") or {}),
|
||||
**kwargs["extra_headers"],
|
||||
}
|
||||
# response_format: top-level gets the same translation as the extra_body form; when both
|
||||
# are present the extra_body form wins. Passthrough excludes ``reasoning``/``response_format``
|
||||
# (already TRANSLATED to native fields — raw would 400 on strict gateways) and ``_`` Hermes plumbing.
|
||||
@@ -1839,15 +1850,21 @@ def _resolve_nous_pool_runtime_api(*, force_refresh: bool = False) -> Optional[t
|
||||
return api_key, base_url
|
||||
|
||||
|
||||
def _resolve_nous_runtime_api(*, force_refresh: bool = False) -> Optional[tuple[str, str]]:
|
||||
"""Fresh Nous runtime credentials (pool first, then auth store + JWT refresh) — mirrors the main agent's 401 recovery."""
|
||||
def _resolve_nous_runtime_api(
|
||||
*, force_refresh: bool = False, stale_access_token: Optional[str] = None
|
||||
) -> Optional[tuple[str, str]]:
|
||||
"""Fresh Nous runtime credentials (pool first, then auth store + JWT refresh) — mirrors the main
|
||||
agent's 401 recovery. ``stale_access_token`` is the bearer that just 401'd; with ``force_refresh``
|
||||
it lets the auth store adopt a sibling process's rotation instead of re-POSTing the shared grant."""
|
||||
pooled = _resolve_nous_pool_runtime_api(force_refresh=force_refresh)
|
||||
if pooled is not None:
|
||||
return pooled
|
||||
try:
|
||||
from hermes_cli.auth import resolve_nous_runtime_credentials
|
||||
creds = resolve_nous_runtime_credentials(
|
||||
timeout_seconds=env_float("HERMES_NOUS_TIMEOUT_SECONDS", 15), force_refresh=force_refresh,
|
||||
timeout_seconds=env_float("HERMES_NOUS_TIMEOUT_SECONDS", 15),
|
||||
force_refresh=force_refresh,
|
||||
stale_access_token=stale_access_token or None,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("Auxiliary Nous runtime credential resolution failed: %s", exc)
|
||||
@@ -5040,7 +5057,7 @@ def _refresh_nous_auxiliary_client(
|
||||
the exact entry the stale one is served from. Keying on the resolved model or an empty task
|
||||
would leave the expired client immortal and every auxiliary call 401ing forever.
|
||||
"""
|
||||
runtime = _resolve_nous_runtime_api(force_refresh=True)
|
||||
runtime = _resolve_nous_runtime_api(force_refresh=True, stale_access_token=api_key)
|
||||
if runtime is None:
|
||||
return None, model
|
||||
fresh_key, fresh_base_url = runtime
|
||||
@@ -5803,7 +5820,10 @@ def _build_call_kwargs(
|
||||
or _endpoint_speaks_anthropic_messages(raw_base) or _is_anthropic_compat_endpoint(provider_norm, raw_base)
|
||||
):
|
||||
kwargs["_reasoning_config"] = dict(reasoning_config)
|
||||
return kwargs
|
||||
# OpenCode relay session affinity — same key as the main turn so compression/title/vision
|
||||
# calls stay on the conversation's warm backend.
|
||||
from agent.opencode_affinity import merge_opencode_session_headers
|
||||
return merge_opencode_session_headers(kwargs, provider, base_url, _runtime_main_value("session_id") or None)
|
||||
|
||||
|
||||
def _validate_llm_response(
|
||||
|
||||
@@ -1499,7 +1499,25 @@ def _build_chat_completions_kwargs(agent, api_messages, tools_for_api, reasoning
|
||||
|
||||
|
||||
def build_api_kwargs(agent, api_messages: list, tools_for_api: list | None = None) -> dict:
|
||||
"""Build the keyword arguments dict for the active API mode."""
|
||||
"""Build the keyword arguments dict for the active API mode.
|
||||
|
||||
Wraps the per-api_mode builder so the OpenCode ``x-opencode-session``
|
||||
affinity header rides on every OpenCode request regardless of transport
|
||||
(chat_completions / codex_responses / anthropic_messages all route
|
||||
OpenCode models). No-op for every other provider.
|
||||
"""
|
||||
from agent.opencode_affinity import merge_opencode_session_headers
|
||||
|
||||
kwargs = _build_api_kwargs_for_mode(agent, api_messages, tools_for_api)
|
||||
return merge_opencode_session_headers(
|
||||
kwargs,
|
||||
getattr(agent, "provider", None),
|
||||
getattr(agent, "base_url", None),
|
||||
getattr(agent, "session_id", None),
|
||||
)
|
||||
|
||||
|
||||
def _build_api_kwargs_for_mode(agent, api_messages: list, tools_for_api: list | None = None) -> dict:
|
||||
# One-shot continuation override — consumed exactly once, on the FIRST
|
||||
# request this call builds (only one api_mode branch runs per invocation).
|
||||
reasoning_config = _reasoning_config_for_wire(agent)
|
||||
@@ -2079,6 +2097,9 @@ def _iteration_summary_api_messages(agent, messages: list) -> list:
|
||||
|
||||
# Compression/resume can orphan a tool result whose parent tool_call was summarized away.
|
||||
api_messages = agent._sanitize_api_messages(api_messages)
|
||||
# Same send-path vision eviction as the main loop (#89296).
|
||||
from agent.context_compressor import evict_stale_outbound_tool_images
|
||||
evict_stale_outbound_tool_images(api_messages)
|
||||
# Thinking-only assistant turns 400 on Anthropic-family providers; _thinking_prefill must
|
||||
# survive until here so the drop pass recognizes stubs after reasoning is stripped.
|
||||
api_messages = agent._drop_thinking_only_and_merge_users(api_messages)
|
||||
@@ -2541,6 +2562,15 @@ class _ToolCallAccumulator:
|
||||
self._notified: set = set()
|
||||
self._last_id_at_idx: dict = {} # raw_index -> last seen non-empty id
|
||||
self._active_slot_by_idx: dict = {} # raw_index -> current slot in acc
|
||||
# Argument deltas are collected per slot and joined once in ``materialize`` —
|
||||
# ``+=`` per chunk rebuilds the whole string every delta (quadratic on big args).
|
||||
self._argument_parts: dict[int, list[str]] = {}
|
||||
|
||||
def materialize(self) -> dict:
|
||||
"""Join buffered argument deltas into each entry's ``arguments``; idempotent. Returns ``acc``."""
|
||||
for idx, parts in self._argument_parts.items():
|
||||
self.acc[idx]["function"]["arguments"] = "".join(parts)
|
||||
return self.acc
|
||||
|
||||
def feed(self, tc_delta) -> Optional[str]:
|
||||
"""Merge one delta; return the tool name the first time it is complete."""
|
||||
@@ -2562,6 +2592,7 @@ class _ToolCallAccumulator:
|
||||
entry = self.acc.setdefault(
|
||||
idx, {"id": tc_id or "", "type": "function", "function": {"name": "", "arguments": ""}, "extra_content": None},
|
||||
)
|
||||
parts = self._argument_parts.setdefault(idx, [])
|
||||
if tc_id:
|
||||
entry["id"] = tc_id
|
||||
tc_function = getattr(tc_delta, "function", None)
|
||||
@@ -2571,7 +2602,7 @@ class _ToolCallAccumulator:
|
||||
# NVIDIA NIM) resend the full name every chunk — += gives "read_fileread_file".
|
||||
entry["function"]["name"] = tc_function.name
|
||||
if getattr(tc_function, "arguments", None):
|
||||
entry["function"]["arguments"] += tc_function.arguments
|
||||
parts.append(tc_function.arguments)
|
||||
extra = getattr(tc_delta, "extra_content", None)
|
||||
if extra is None and hasattr(tc_delta, "model_extra"):
|
||||
extra = (tc_delta.model_extra if isinstance(tc_delta.model_extra, dict) else {}).get("extra_content")
|
||||
@@ -2832,6 +2863,7 @@ class _StreamingCall:
|
||||
return self._open_chat_stream({**next_api_kwargs, "stream": True, "timeout": timeout})
|
||||
|
||||
def _relay_final_response() -> dict[str, Any]:
|
||||
tool_calls.materialize()
|
||||
message = {"role": role, "content": "".join(content_parts) or None,
|
||||
"reasoning_content": "".join(reasoning_parts) or None,
|
||||
"tool_calls": [tool_calls_acc[i] for i in sorted(tool_calls_acc)] or None}
|
||||
@@ -2920,6 +2952,7 @@ class _StreamingCall:
|
||||
# complete instead of silently discarding the action.
|
||||
self.result["partial_tool_names"].append(name)
|
||||
|
||||
tool_calls.materialize()
|
||||
self._close_managed_stream()
|
||||
if self._stream_attempt_was_cancelled(stream_attempt_id):
|
||||
raise _httpx.RemoteProtocolError(f"stream attempt {stream_attempt_id} was superseded")
|
||||
|
||||
@@ -490,7 +490,11 @@ class ClientLifecycleMixin:
|
||||
try:
|
||||
from hermes_cli.auth import resolve_nous_runtime_credentials
|
||||
timeout = env_float("HERMES_NOUS_TIMEOUT_SECONDS", 15)
|
||||
creds = resolve_nous_runtime_credentials(timeout_seconds=timeout, force_refresh=force)
|
||||
# Pass the bearer that just 401'd so a refresh already done by a sibling process is
|
||||
# adopted instead of rotating the grant again.
|
||||
creds = resolve_nous_runtime_credentials(
|
||||
timeout_seconds=timeout, force_refresh=force, stale_access_token=self.api_key or None,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("Nous credential refresh failed: %s", exc)
|
||||
return False
|
||||
|
||||
+312
-14
@@ -233,6 +233,21 @@ def _template_visible_role(message: Any) -> Optional[str]:
|
||||
return None if role == "tool" or (role == "assistant" and message.get("tool_calls")) else role
|
||||
|
||||
|
||||
def _last_template_visible_role(messages: List[Dict[str, Any]]) -> Optional[str]:
|
||||
"""Last role a strict alternation template would count in *messages*.
|
||||
|
||||
``None`` when every row is template-exempt (tool flow only).
|
||||
"""
|
||||
return next(
|
||||
(
|
||||
role
|
||||
for role in (_template_visible_role(m) for m in reversed(messages))
|
||||
if role is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _strip_persistence_markers(messages: List[Dict[str, Any]]) -> None:
|
||||
"""Enforce the invariant: no assembled message carries a persistence marker.
|
||||
A leaked ``_db_persisted`` makes the child-session rotation flush skip the row, losing it from state.db.
|
||||
@@ -296,6 +311,22 @@ _SUMMARY_END_MARKER = "--- END OF CONTEXT SUMMARY — respond to the message bel
|
||||
_MERGED_PRIOR_CONTEXT_HEADER = "[PRIOR CONTEXT — for reference only; not a new message]"
|
||||
_MERGED_SUMMARY_DELIMITER = "[END OF PRIOR CONTEXT — COMPACTION SUMMARY BELOW]"
|
||||
|
||||
# Prefixes the copy of a still-running user task that compaction re-states after
|
||||
# the handoff boundary (#100818). A cron run's only user turn is the job prompt
|
||||
# in the protected head, so compaction leaves it BEFORE the summary — and
|
||||
# SUMMARY_PREFIX tells the model to do nothing when no user message follows.
|
||||
# Set on a compaction carrier when the in-flight task was merged onto it (the
|
||||
# carrier ends the list, so a standalone user row would break alternation).
|
||||
# conversation_compression._ensure_compressed_has_user_turn treats it as
|
||||
# "intent present" so it does not insert a second copy of the same request.
|
||||
_INFLIGHT_REPLAY_MERGED_KEY = "_inflight_replay_merged"
|
||||
|
||||
_INFLIGHT_TASK_REPLAY_HEADER = (
|
||||
"[STILL IN PROGRESS — this is the active request, restated after the "
|
||||
"compaction boundary because it was not finished yet. Continue it; do not "
|
||||
"start over.]"
|
||||
)
|
||||
|
||||
_SALVAGE_SUMMARY_MAX_CHARS = 8_000
|
||||
_SALVAGE_KEEP_RECENT_TOOLS = 2
|
||||
|
||||
@@ -1140,6 +1171,23 @@ def _retire_stale_tool_result_images(result: List[Dict[str, Any]], keep_newest:
|
||||
return pruned
|
||||
|
||||
|
||||
def evict_stale_outbound_tool_images(
|
||||
api_messages: List[Dict[str, Any]],
|
||||
keep_newest: int = _MAX_KEEP_TOOL_IMAGES,
|
||||
) -> int:
|
||||
"""Drop stale screenshot/vision payloads from the per-call API copy.
|
||||
|
||||
Compression's keep-newest pass only runs when prune/compress fires, and
|
||||
the Anthropic adapter's screenshot eviction only sees nested
|
||||
``tool_result`` blocks. OpenAI-style ``image_url`` tool results
|
||||
otherwise ride every subsequent request until a 413 forces the reactive
|
||||
strip (#89286). Call this on the cloned ``api_messages`` list after
|
||||
sanitization so older frames never leave the box (#89296). Do not pass
|
||||
persisted history — the rewrite is send-path only.
|
||||
"""
|
||||
return _retire_stale_tool_result_images(api_messages, keep_newest=keep_newest)
|
||||
|
||||
|
||||
def _truncate_tool_call_args_json(args: str, head_chars: int = 200) -> str:
|
||||
"""Shrink long string leaves in a tool-call arguments JSON blob, keeping it valid (providers 400 on malformed args)."""
|
||||
try:
|
||||
@@ -1716,7 +1764,7 @@ class ContextCompressor(MicroCompactionMixin, ContextEngine):
|
||||
self._reset_real_usage_pairing()
|
||||
self._last_compression_telemetry = self._active_compression_telemetry = None
|
||||
self._compression_telemetry_seed = None
|
||||
self._proactive_prune_rearm_tokens = 0
|
||||
self._reset_proactive_prune_rearm()
|
||||
|
||||
def bind_session_state(self, session_db: Any = None, session_id: str = "") -> None:
|
||||
"""Bind the current session row so durable cooldowns can round-trip."""
|
||||
@@ -1726,8 +1774,9 @@ class ContextCompressor(MicroCompactionMixin, ContextEngine):
|
||||
self._cooldown_persist_failed = False
|
||||
self._last_summary_error = None
|
||||
self._consecutive_timeout_failures = self._fallback_compression_streak = 0
|
||||
self._ineffective_compression_count = self._prellm_skip_count = self._proactive_prune_rearm_tokens = 0
|
||||
self._ineffective_compression_count = self._prellm_skip_count = 0
|
||||
self._anti_thrash_recovery_deadline = self._structural_no_op_backoff_until = 0.0
|
||||
self._reset_proactive_prune_rearm()
|
||||
self.get_active_compression_failure_cooldown()
|
||||
self._load_fallback_compression_streak()
|
||||
self._load_ineffective_compression_count()
|
||||
@@ -2016,7 +2065,7 @@ class ContextCompressor(MicroCompactionMixin, ContextEngine):
|
||||
self._clear_compression_failure_cooldown()
|
||||
self._verify_compaction_cleared_threshold = self._last_compression_made_progress = False
|
||||
# Runway was computed against the previous model's trigger; clear the durable copy too.
|
||||
self._proactive_prune_rearm_tokens = 0
|
||||
self._reset_proactive_prune_rearm()
|
||||
self._clear_durable_proactive_prune_rearm()
|
||||
|
||||
# When the MINIMUM_CONTEXT_LENGTH floor binds on a small window, trigger near the top instead.
|
||||
@@ -2107,6 +2156,10 @@ class ContextCompressor(MicroCompactionMixin, ContextEngine):
|
||||
self.proactive_prune_min_reclaim_tokens = max(0, int(proactive_prune_min_reclaim_tokens or 0))
|
||||
# A committed prune is a cache boundary: rearm only after the prompt regrows the reclaimed tokens.
|
||||
self._proactive_prune_rearm_tokens: int = 0
|
||||
# Dedup key for the over-threshold "reclamation no-oped" warning
|
||||
# (#101889) so a tool loop riding above the threshold warns once per
|
||||
# distinct reason + rearm snapshot instead of every iteration.
|
||||
self._last_reclaim_block_warn: "tuple[str, int] | None" = None
|
||||
self.min_tail_user_messages = min_tail_user_messages
|
||||
self.summary_target_ratio = max(0.10, min(summary_target_ratio, 0.80))
|
||||
self.quiet_mode = quiet_mode
|
||||
@@ -2513,35 +2566,115 @@ class ContextCompressor(MicroCompactionMixin, ContextEngine):
|
||||
)
|
||||
return result, pruned
|
||||
|
||||
def _reset_proactive_prune_rearm(self) -> None:
|
||||
"""Fully rearm the proactive prune and let a future lockout warn again.
|
||||
|
||||
Every path that zeroes the rearm mark (compaction, session
|
||||
reset/end/rebind, model recalibration) is a reclamation or a fresh
|
||||
start, so the over-threshold no-op dedup key must not survive it —
|
||||
otherwise an identical lockout after a full compaction (rearm back
|
||||
at 0) would be silent (#101889).
|
||||
"""
|
||||
self._proactive_prune_rearm_tokens = 0
|
||||
self._last_reclaim_block_warn = None
|
||||
|
||||
def _billed_basis_over_threshold(self, current_tokens: "int | None") -> bool:
|
||||
"""Whether a provider-billed reading says the session is over threshold.
|
||||
|
||||
``current_tokens`` is the provider's ``prompt_tokens`` (or the
|
||||
overhead-aware fallback estimate): it counts the system prompt and tool
|
||||
schemas, which the message-only estimate behind
|
||||
``_proactive_prune_rearm_tokens`` does not. Used to stop schema
|
||||
overhead from parking the prune rearm gate above a real request that is
|
||||
already over ``threshold_tokens`` (#101889).
|
||||
"""
|
||||
return (
|
||||
current_tokens is not None
|
||||
and self.threshold_tokens > 0
|
||||
and current_tokens >= self.threshold_tokens
|
||||
)
|
||||
|
||||
def _warn_reclamation_no_op(
|
||||
self,
|
||||
reason: str,
|
||||
current_tokens: "int | None",
|
||||
before: "int | None" = None,
|
||||
) -> None:
|
||||
"""Warn when an over-threshold session's reclamation path no-ops.
|
||||
|
||||
A session sitting above ``threshold_tokens`` with every reclamation
|
||||
path declining is the failure mode from #101889: context keeps growing
|
||||
until the provider's hard limit rejects the request, with nothing in
|
||||
the log to explain it. Silent below the threshold (a declined prune
|
||||
there is ordinary hysteresis, not a lockout). Deduped on
|
||||
``reason`` + the rearm snapshot so a busy tool loop logs once per
|
||||
distinct state, not once per iteration; the key is cleared whenever
|
||||
the session drops back under threshold or any reclamation resets the
|
||||
rearm mark (prune commit, compaction, session reset/rebind, model
|
||||
recalibration) so a later lockout warns again.
|
||||
"""
|
||||
# The explicit None check is redundant with the predicate; it narrows
|
||||
# ``current_tokens`` for the type checker on the format below.
|
||||
if current_tokens is None or not self._billed_basis_over_threshold(
|
||||
current_tokens
|
||||
):
|
||||
self._last_reclaim_block_warn = None
|
||||
return
|
||||
key = (reason, int(self._proactive_prune_rearm_tokens))
|
||||
if self._last_reclaim_block_warn == key:
|
||||
return
|
||||
self._last_reclaim_block_warn = key
|
||||
logger.warning(
|
||||
"Context is over the compression threshold (~%s of %s tokens) but "
|
||||
"reclamation did not run: %s (message-token estimate %s, prune "
|
||||
"rearm mark %s). The session may keep growing until the provider "
|
||||
"rejects the request — /compact to compress history now.",
|
||||
f"{int(current_tokens):,}",
|
||||
f"{int(self.threshold_tokens):,}",
|
||||
reason,
|
||||
"n/a" if before is None else f"{int(before):,}",
|
||||
f"{int(self._proactive_prune_rearm_tokens):,}",
|
||||
)
|
||||
|
||||
def prune_tool_results_only(
|
||||
self, messages: List[Dict[str, Any]], current_tokens: int | None = None,
|
||||
) -> tuple[List[Dict[str, Any]], int]:
|
||||
"""Deterministic, no-LLM tool-result prune gated on ``proactive_prune_tokens``.
|
||||
Protects the tail by message COUNT only. A commit breaks the prompt cache, so it requires
|
||||
``proactive_prune_min_reclaim_tokens`` and a full regrowth runway; otherwise returns the INPUT
|
||||
object as ``(messages, 0)``."""
|
||||
if (
|
||||
self.proactive_prune_tokens <= 0
|
||||
or (current_tokens is not None and current_tokens < self.proactive_prune_tokens)
|
||||
or len(messages) <= self.protect_last_n + self._protect_head_size(messages) + 1
|
||||
object as ``(messages, 0)``. The rearm gate is measured on message bodies only, so it is
|
||||
bypassed (never the reclaim gate) when a provider-billed ``current_tokens`` reading already
|
||||
puts the request over ``threshold_tokens`` (#101889); every no-op taken while over threshold
|
||||
is logged once per distinct reason."""
|
||||
if self.proactive_prune_tokens <= 0 or (
|
||||
current_tokens is not None and current_tokens < self.proactive_prune_tokens
|
||||
):
|
||||
return messages, 0
|
||||
if len(messages) <= self.protect_last_n + self._protect_head_size(messages) + 1:
|
||||
self._warn_reclamation_no_op("prune:tail_only", current_tokens)
|
||||
return messages, 0
|
||||
before = sum(_estimate_msg_budget_tokens(m) for m in messages)
|
||||
if before < self._proactive_prune_rearm_tokens:
|
||||
# Under-threshold runway skip is ordinary hysteresis (silent); above it the lockout is the bug.
|
||||
if before < self._proactive_prune_rearm_tokens and not self._billed_basis_over_threshold(current_tokens):
|
||||
return messages, 0
|
||||
# Capability gate first: a store without archive_and_compact makes every prune a no-op.
|
||||
session_db = getattr(self, "_session_db", None)
|
||||
session_id = getattr(self, "_session_id", "")
|
||||
if session_db and session_id and not callable(getattr(session_db, "archive_and_compact", None)):
|
||||
self._warn_reclamation_no_op("prune:store_cannot_persist", current_tokens)
|
||||
return messages, 0
|
||||
pruned_msgs, pruned_count = self._prune_old_tool_results(
|
||||
messages, protect_tail_count=self.protect_last_n, protect_tail_tokens=None, min_prune_chars=self.proactive_prune_min_result_chars,
|
||||
)
|
||||
# No-op contract: return the INPUT object so callers can gate on `result is not input`.
|
||||
if not pruned_count:
|
||||
# No-op contract: return the INPUT object so callers can gate on `result is not input`.
|
||||
self._warn_reclamation_no_op("prune:nothing_eligible", current_tokens)
|
||||
return messages, 0
|
||||
# Prompt-cache hysteresis: commit only when the reclaim is meaningful.
|
||||
after = sum(_estimate_msg_budget_tokens(m) for m in pruned_msgs)
|
||||
reclaimed = max(0, before - after)
|
||||
if not pruned_count or reclaimed < self.proactive_prune_min_reclaim_tokens:
|
||||
if reclaimed < self.proactive_prune_min_reclaim_tokens:
|
||||
self._warn_reclamation_no_op("prune:reclaim_below_minimum", current_tokens, before=before)
|
||||
return messages, 0
|
||||
# Require a full trigger-sized regrowth before the next cache-breaking rewrite.
|
||||
runway = max(reclaimed, self.proactive_prune_tokens, self.proactive_prune_min_reclaim_tokens)
|
||||
@@ -2558,6 +2691,8 @@ class ContextCompressor(MicroCompactionMixin, ContextEngine):
|
||||
# Shared post-commit stamp site with the in-place commit and micro-compaction sync.
|
||||
stamp_db_persisted_markers(pruned_msgs)
|
||||
self._proactive_prune_rearm_tokens = next_rearm_tokens
|
||||
# Reclamation just ran: let a future lockout warn again.
|
||||
self._last_reclaim_block_warn = None
|
||||
return pruned_msgs, pruned_count
|
||||
|
||||
def _compute_summary_budget(self, turns_to_summarize: List[Dict[str, Any]]) -> int:
|
||||
@@ -3602,6 +3737,164 @@ Write only the summary body. Do not include any preamble or prefix."""
|
||||
return max(pair_end, head_end + 1)
|
||||
return adjusted
|
||||
|
||||
@classmethod
|
||||
def _find_inflight_user_task(
|
||||
cls, messages: List[Dict[str, Any]]
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
"""Return the user turn that is still awaiting completion, or ``None``.
|
||||
|
||||
Scans the WHOLE transcript, not just the compressible region: a cron
|
||||
run's only user turn is the job prompt sitting in the protected head
|
||||
(``protect_first_n`` keeps system + first user), which is exactly the
|
||||
turn ``_find_last_user_message_idx`` cannot see (#100818).
|
||||
|
||||
A turn is in-flight when the transcript does not already end with a
|
||||
completed assistant reply — i.e. a text-bearing assistant message with
|
||||
no pending ``tool_calls``. A trailing ``tool`` result or an assistant
|
||||
message that still has ``tool_calls`` outstanding means the run was
|
||||
interrupted mid-task and the instruction is still owed an answer.
|
||||
|
||||
Handoff carriers and synthetic scaffolding rows are excluded via the
|
||||
same filter pair as ``_find_last_user_message_idx``, so an idle session
|
||||
whose only user-role row is an inherited summary yields ``None`` and is
|
||||
never re-animated (#80622).
|
||||
"""
|
||||
from agent.conversation_compression import _is_real_user_message
|
||||
|
||||
last_user_idx = -1
|
||||
for i in range(len(messages) - 1, -1, -1):
|
||||
msg = messages[i]
|
||||
# _is_real_user_message also rejects metadata-flagged scaffolding
|
||||
# (_todo_snapshot_synthetic, recovery nudges, ...) that
|
||||
# _is_actionable_user_turn cannot see.
|
||||
if cls._is_actionable_user_turn(msg) and _is_real_user_message(msg):
|
||||
last_user_idx = i
|
||||
break
|
||||
if isinstance(msg, dict) and msg.get(_INFLIGHT_REPLAY_MERGED_KEY):
|
||||
# A previous cycle merged the live request onto this summary
|
||||
# carrier; it is the only copy left, so it is still the task.
|
||||
last_user_idx = i
|
||||
break
|
||||
if last_user_idx < 0:
|
||||
return None
|
||||
|
||||
for msg in reversed(messages[last_user_idx + 1:]):
|
||||
if not isinstance(msg, dict) or msg.get("role") != "assistant":
|
||||
# Trailing tool result (or anything else): still mid-task.
|
||||
break
|
||||
if msg.get("tool_calls"):
|
||||
break
|
||||
if _content_text_for_contains(msg.get("content")).strip():
|
||||
# Final answer already delivered — replaying the ask would
|
||||
# hand the model finished work as a fresh instruction.
|
||||
return None
|
||||
# Empty assistant row (a bare reasoning/stub turn): keep looking.
|
||||
return messages[last_user_idx]
|
||||
|
||||
def _reappend_inflight_user_task(
|
||||
self,
|
||||
compressed: List[Dict[str, Any]],
|
||||
inflight: Optional[Dict[str, Any]],
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Restate an unfinished user task after the compaction handoff.
|
||||
|
||||
``SUMMARY_PREFIX`` instructs the model to act only on a user message
|
||||
that appears AFTER the summary, and to do nothing when none does. When
|
||||
the single in-flight instruction lived in the protected head, the
|
||||
assembled transcript orders it before the handoff and the run ends in a
|
||||
``[SILENT]`` no-op that the scheduler records as success (#100818).
|
||||
|
||||
Re-append a copy of that turn after the surviving tail so the prefix's
|
||||
"latest user message" pointer resolves to it again. If the transcript
|
||||
already ends on a template-visible user row, appending a second one
|
||||
would break user/assistant alternation, so the restatement is merged
|
||||
onto the handoff carrier instead — after ``_SUMMARY_END_MARKER``, which
|
||||
is the boundary the prefix's rule is written against.
|
||||
"""
|
||||
if inflight is None or not compressed:
|
||||
return compressed
|
||||
|
||||
carrier_idx = -1
|
||||
for idx in range(len(compressed) - 1, -1, -1):
|
||||
if self._is_context_summary_message(compressed[idx]):
|
||||
carrier_idx = idx
|
||||
break
|
||||
if carrier_idx < 0:
|
||||
# No handoff was emitted — nothing reordered the instruction.
|
||||
return compressed
|
||||
|
||||
for msg in compressed[carrier_idx + 1:]:
|
||||
if self._is_actionable_user_turn(
|
||||
msg
|
||||
) and not self._is_synthetic_compression_user_turn(msg):
|
||||
# A real request already follows the summary.
|
||||
return compressed
|
||||
|
||||
carrier = compressed[carrier_idx]
|
||||
carrier_text = _content_text_for_contains(carrier.get("content"))
|
||||
if _SUMMARY_END_MARKER not in carrier_text:
|
||||
return compressed
|
||||
if carrier_text.split(_SUMMARY_END_MARKER, 1)[1].strip():
|
||||
# The _force_user_leading layout keeps the live request on the
|
||||
# carrier itself, after the marker. Already actionable.
|
||||
return compressed
|
||||
|
||||
task_text = _content_text_for_contains(inflight.get("content")).strip()
|
||||
if _INFLIGHT_TASK_REPLAY_HEADER in task_text:
|
||||
# Already a restatement from an earlier compaction (standalone row
|
||||
# or merged onto a carrier): take the text after the header so a
|
||||
# task that survives >1 cycle never stacks headers or drags the
|
||||
# old summary along.
|
||||
task_text = task_text.rsplit(_INFLIGHT_TASK_REPLAY_HEADER, 1)[1].strip()
|
||||
if not task_text:
|
||||
return compressed
|
||||
|
||||
if not self.quiet_mode:
|
||||
logger.info(
|
||||
"Re-appending the in-flight user task after the compaction "
|
||||
"handoff so it stays actionable (#100818)"
|
||||
)
|
||||
|
||||
last_visible_role = _last_template_visible_role(compressed)
|
||||
if inflight.get(_INFLIGHT_REPLAY_MERGED_KEY):
|
||||
# Never copy a summary carrier (metadata would mark the replay
|
||||
# synthetic): restate as a plain user row.
|
||||
replay = {"role": "user", "content": task_text}
|
||||
else:
|
||||
replay = _fresh_compaction_message_copy(inflight)
|
||||
replay.pop(_COMPACTION_TAIL_MARKER, None)
|
||||
if isinstance(replay.get("content"), str):
|
||||
# Plain text: rebuild from the header-stripped task text so a
|
||||
# task surviving several compactions never stacks headers.
|
||||
replay["content"] = _INFLIGHT_TASK_REPLAY_HEADER + "\n" + task_text
|
||||
else:
|
||||
# Multimodal parts: keep them, prepend the header text part.
|
||||
replay["content"] = _append_text_to_content(
|
||||
replay.get("content"),
|
||||
_INFLIGHT_TASK_REPLAY_HEADER + "\n",
|
||||
prepend=True,
|
||||
)
|
||||
drop_stale_api_content(replay)
|
||||
|
||||
if last_visible_role == "user":
|
||||
# Alternation is judged on template-visible rows only (tool_calls /
|
||||
# tool rows are exempt), so a user-pinned summary followed by a
|
||||
# tool tail still "ends on user": a standalone user row would break
|
||||
# the Mistral-style pre-flight check (#58753). Merge onto the
|
||||
# carrier instead and flag it — the carrier's own metadata marks it
|
||||
# synthetic, and without the flag _ensure_compressed_has_user_turn
|
||||
# would insert a second copy of the same request.
|
||||
carrier["content"] = _append_text_to_content(
|
||||
carrier.get("content"),
|
||||
"\n\n" + _INFLIGHT_TASK_REPLAY_HEADER + "\n" + task_text,
|
||||
)
|
||||
carrier[_INFLIGHT_REPLAY_MERGED_KEY] = True
|
||||
drop_stale_api_content(carrier)
|
||||
return compressed
|
||||
|
||||
compressed.append(replay)
|
||||
return compressed
|
||||
|
||||
def _ensure_last_n_user_messages_in_tail(
|
||||
self, messages: List[Dict[str, Any]], cut_idx: int, head_end: int, n: int,
|
||||
) -> int:
|
||||
@@ -3923,7 +4216,7 @@ Write only the summary body. Do not include any preamble or prefix."""
|
||||
last_head_role: Optional[str] = "user"
|
||||
if compressed:
|
||||
# None = all-exempt head: the summary opens the visible sequence and must be "user".
|
||||
last_head_role = next((r for r in map(_template_visible_role, reversed(compressed)) if r is not None), None)
|
||||
last_head_role = _last_template_visible_role(compressed)
|
||||
first_tail_visible_idx, first_tail_role = next(
|
||||
((idx, role) for idx, role in enumerate(map(_template_visible_role, tail_messages)) if role is not None),
|
||||
(None, None),
|
||||
@@ -3973,8 +4266,13 @@ Write only the summary body. Do not include any preamble or prefix."""
|
||||
self, compressed: List[Dict[str, Any]], messages: List[Dict[str, Any]], n_messages: int,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Post-assembly cleanup: orphan pairs, media, savings, markers, replay prune, mem trim."""
|
||||
self.compression_count += 1
|
||||
# Single-prompt cron shape: the only live instruction sits in the protected head, BEFORE the
|
||||
# handoff, and SUMMARY_PREFIX reads that as "nothing to do" — restate it past the boundary
|
||||
# (#100818). Sanitize FIRST: the trailing-in-flight exemption (#79278) walks back from the list
|
||||
# end, and a replay user row there would strip a genuinely pending assistant(tool_calls).
|
||||
compressed = self._sanitize_tool_pairs(compressed)
|
||||
compressed = self._reappend_inflight_user_task(compressed, self._find_inflight_user_task(messages))
|
||||
self.compression_count += 1
|
||||
# Replace historical image payloads with placeholders; multi-MB base64 blobs otherwise
|
||||
# exceed body limits.
|
||||
compressed = _strip_historical_media(compressed)
|
||||
@@ -4010,7 +4308,7 @@ Write only the summary body. Do not include any preamble or prefix."""
|
||||
# Batch marker holds MORE history than the rolling summary: reset micro state so it can't
|
||||
# supersede/defrag content it lacks; the next micro pass rehydrates from the batch marker.
|
||||
self._reset_micro_compact_cursor_state()
|
||||
self._proactive_prune_rearm_tokens = 0
|
||||
self._reset_proactive_prune_rearm()
|
||||
return compressed
|
||||
|
||||
def compress(
|
||||
|
||||
@@ -59,6 +59,9 @@ _SPLIT_FAILURE_COOLDOWN_SECONDS = 60
|
||||
# the phrase intact when rewording. Idle/preflight/retry lines lack it; is_compaction_progress_status covers those.
|
||||
COMPACTION_STATUS_MARKER = "Compacting context"
|
||||
COMPACTION_STATUS = f"🗜️ {COMPACTION_STATUS_MARKER} — summarizing earlier conversation so I can continue..."
|
||||
# Periodic heartbeat re-emitted while a long compression is still running so remote transports with
|
||||
# idle-turn watchdogs (#98371) see progress. Same marker as COMPACTION_STATUS so consumers classify it alike.
|
||||
COMPACTION_HEARTBEAT_STATUS = f"🗜️ {COMPACTION_STATUS_MARKER} — still summarizing earlier conversation so I can continue..."
|
||||
|
||||
COMPACTION_DONE_STATUS = "✓ Context compaction complete — continuing turn..."
|
||||
|
||||
@@ -113,7 +116,8 @@ CONTEXT_OVERFLOW_BLOCKED_WARNING_TEMPLATE = (
|
||||
|
||||
# Formatted from the same constants the emission sites use, so noise-filter tests exercise the ACTUAL wording.
|
||||
ROUTINE_COMPRESSION_STATUS_SAMPLES = (
|
||||
COMPACTION_STATUS, COMPACTION_DONE_STATUS, PRE_API_COMPRESSION_STATUS_TEMPLATE.format(tokens=123456),
|
||||
COMPACTION_STATUS, COMPACTION_HEARTBEAT_STATUS, COMPACTION_DONE_STATUS,
|
||||
PRE_API_COMPRESSION_STATUS_TEMPLATE.format(tokens=123456),
|
||||
PREFLIGHT_COMPRESSION_STATUS_TEMPLATE.format(tokens=120000, threshold=100000),
|
||||
IDLE_COMPACTION_STATUS_TEMPLATE.format(idle_seconds=3600, tokens=120000),
|
||||
COMPRESSION_RETRY_TOO_LARGE_STATUS_TEMPLATE.format(tokens=250000, attempt=1, cap=3),
|
||||
@@ -1388,7 +1392,8 @@ class _CompressionActivityHeartbeat:
|
||||
"""Refresh the agent inactivity tracker while compression blocks in an aux call."""
|
||||
|
||||
def __init__(
|
||||
self, agent: Any, interval_seconds: float | None = None, commit_fence: Optional[CompressionCommitFence] = None
|
||||
self, agent: Any, interval_seconds: float | None = None, *, emit_client_status: bool = False,
|
||||
commit_fence: Optional[CompressionCommitFence] = None,
|
||||
) -> None:
|
||||
self._agent = agent
|
||||
self._commit_fence = commit_fence
|
||||
@@ -1404,6 +1409,10 @@ class _CompressionActivityHeartbeat:
|
||||
except (TypeError, ValueError):
|
||||
interval_seconds = 60.0
|
||||
self._interval_seconds = max(0.1, interval_seconds)
|
||||
# Only a compression that opened a VISIBLE compaction phase (the
|
||||
# routine start status was emitted) keeps it alive with heartbeats;
|
||||
# quiet context engines emit neither (#98371 follow-up).
|
||||
self._emit_client_status = emit_client_status
|
||||
self._stop = threading.Event()
|
||||
self._thread = threading.Thread(target=self._run, name="compression-activity-heartbeat", daemon=True)
|
||||
|
||||
@@ -1449,11 +1458,40 @@ class _CompressionActivityHeartbeat:
|
||||
return
|
||||
touch(desc, provenance=ActivityProvenance.AGENT_COMPRESSION, force_persist=force_persist)
|
||||
|
||||
def _emit_progress_status(self) -> None:
|
||||
"""Re-publish the compacting status so remote transports see progress.
|
||||
|
||||
Compression can stream for minutes with no deltas, tool events, or
|
||||
status lines reaching remote transports. Idle-progress watchdogs on
|
||||
those clients (e.g. the Android relay app's 180s turn watchdog)
|
||||
treat the silence as a dead turn and fire ``session.interrupt`` —
|
||||
killing a healthy compression mid-flight and rolling back its work,
|
||||
which retriggers on the next prompt and loops forever on sessions
|
||||
near the context ceiling (#98371).
|
||||
|
||||
Routed through ``agent._emit_status`` like every other compaction
|
||||
status: same "lifecycle" key (the TUI gateway re-tags it to
|
||||
``compacting``; Telegram edits one bubble per key), same chat-platform
|
||||
filter, same CLI print path.
|
||||
"""
|
||||
if not self._emit_client_status:
|
||||
return
|
||||
emit = getattr(self._agent, "_emit_status", None)
|
||||
if not callable(emit):
|
||||
return
|
||||
try:
|
||||
emit(COMPACTION_HEARTBEAT_STATUS)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"status emit error in compression heartbeat", exc_info=True
|
||||
)
|
||||
|
||||
def _run(self) -> None:
|
||||
while not self._stop.wait(self._interval_seconds):
|
||||
if self._should_suppress():
|
||||
return
|
||||
self._touch("context compression in progress")
|
||||
self._emit_progress_status()
|
||||
|
||||
|
||||
def _direct_messages_for_pre_compress_memory(messages: Any) -> list[dict[str, Any]]:
|
||||
@@ -1941,7 +1979,12 @@ def _ensure_compressed_has_user_turn(original_messages: list, compressed: list)
|
||||
"""Preserve human intent, not merely a synthetic user-role placeholder."""
|
||||
if any(_is_real_user_message(message) for message in compressed) or _compressed_has_busy_steer(compressed):
|
||||
return "already_present"
|
||||
from agent.context_compressor import COMPRESSION_CONTINUATION_USER_CONTENT, _fresh_compaction_message_copy
|
||||
from agent.context_compressor import (
|
||||
_INFLIGHT_REPLAY_MERGED_KEY, COMPRESSION_CONTINUATION_USER_CONTENT, _fresh_compaction_message_copy,
|
||||
)
|
||||
if any(isinstance(message, dict) and message.get(_INFLIGHT_REPLAY_MERGED_KEY) for message in compressed):
|
||||
# The in-flight request was restated onto the summary carrier (#100818); an anchor would duplicate it.
|
||||
return "already_present"
|
||||
# One reversed scan over BOTH kinds: scanning steer then user would let an older
|
||||
# consumed steer outrank a newer real user request and replay it.
|
||||
for message in reversed(original_messages):
|
||||
@@ -2063,7 +2106,7 @@ class _CompactionLifecycle:
|
||||
|
||||
def __init__(self, agent: Any, status_emitted: bool) -> None:
|
||||
self._agent = agent
|
||||
self._status_emitted = status_emitted
|
||||
self.status_emitted = status_emitted
|
||||
self._done_emitted = False
|
||||
self.commit_status = "aborted"
|
||||
|
||||
@@ -2074,7 +2117,7 @@ class _CompactionLifecycle:
|
||||
# Suppressed start → no terminal edge. Non-compacting aborts (lock contender,
|
||||
# cancelled fence) opt in via force_terminal so clients can retire their phase.
|
||||
# Failure warnings go through _emit_warning and are never suppressed here.
|
||||
if self._status_emitted and (self.commit_status == "committed" or force_terminal):
|
||||
if self.status_emitted and (self.commit_status == "committed" or force_terminal):
|
||||
_emit_compaction_done(self._agent)
|
||||
|
||||
|
||||
@@ -2103,6 +2146,11 @@ class _CompressionLease:
|
||||
# cannot win between acquiring the lock and having a way to release it.
|
||||
self._lock_setup_entered = False
|
||||
|
||||
@property
|
||||
def status_emitted(self) -> bool:
|
||||
"""True when the routine compaction start status was shown (heartbeats may follow it)."""
|
||||
return self._lifecycle.status_emitted
|
||||
|
||||
def begin_lock_setup(self) -> bool:
|
||||
if self._commit_fence is None:
|
||||
return True
|
||||
@@ -2796,8 +2844,9 @@ def _warn_summary_or_aux_fallback(agent: Any) -> None:
|
||||
|
||||
|
||||
def _reset_read_dedup_caches(task_id: str, *, skills: bool = True) -> None:
|
||||
"""Clear the file-read (and skill_view) repeat-read dedup caches after a boundary.
|
||||
Original read content was summarized away, so a re-read must return full content, not a "file unchanged" stub.
|
||||
"""Advance the file-read (and skill_view) repeat-read dedup to a fresh generation after a boundary.
|
||||
The mtime map is kept: the first read of each unchanged key returns full content compaction may have
|
||||
omitted; later reads return stubs, and stub-hit counters restart at the same boundary (#84857).
|
||||
"""
|
||||
with contextlib.suppress(Exception):
|
||||
from tools.file_tools import reset_file_dedup
|
||||
@@ -3144,7 +3193,9 @@ def _run_summary_phase(
|
||||
bypass_cooldown=bypass_cooldown,
|
||||
)
|
||||
messages_before_compression = copy.deepcopy(messages)
|
||||
_activity_heartbeat = _CompressionActivityHeartbeat(agent, commit_fence=commit_fence).start()
|
||||
_activity_heartbeat = _CompressionActivityHeartbeat(
|
||||
agent, commit_fence=commit_fence, emit_client_status=lease.status_emitted,
|
||||
).start()
|
||||
compressed = _run_summary_dispatch(
|
||||
agent, messages, compress_fn, compress_kwargs, commit_fence=commit_fence,
|
||||
attempt_generation=attempt.generation, hard_cancel_event=hard_cancel_event,
|
||||
@@ -3495,7 +3546,7 @@ def _compress_context_via_codex_app_server(
|
||||
logger.info("codex app-server compaction started: session=%s messages=%d tokens=~%s", _sid, len(messages), _tokens)
|
||||
with contextlib.suppress(Exception):
|
||||
agent._emit_status(COMPACTION_STATUS)
|
||||
_activity_heartbeat = _CompressionActivityHeartbeat(agent).start()
|
||||
_activity_heartbeat = _CompressionActivityHeartbeat(agent, emit_client_status=True).start()
|
||||
try:
|
||||
result = codex_session.compact_thread()
|
||||
except BaseException:
|
||||
@@ -3717,7 +3768,7 @@ def try_shrink_image_parts_in_messages(api_messages: list, *, max_dimension: int
|
||||
|
||||
|
||||
__all__ = [
|
||||
"COMPACTION_STATUS", "COMPACTION_DONE_STATUS", "COMPACTION_STATUS_MARKER", "is_compaction_progress_status",
|
||||
"COMPACTION_STATUS", "COMPACTION_DONE_STATUS", "COMPACTION_HEARTBEAT_STATUS", "COMPACTION_STATUS_MARKER", "is_compaction_progress_status",
|
||||
"check_compression_model_feasibility", "replay_compression_warning", "compress_context",
|
||||
"try_shrink_image_parts_in_messages",
|
||||
]
|
||||
|
||||
@@ -328,6 +328,36 @@ def _is_stale_copilot_credential_error(status_code: Optional[int], error_message
|
||||
))
|
||||
|
||||
|
||||
def _pressure_with_real_floor(compressor: Any, rough_tokens: int) -> int:
|
||||
"""Floor the ROUGH pre-API pressure estimate at the last REAL prompt size.
|
||||
|
||||
Applied only on the fallback path -- when ``anchored_context_tokens`` has
|
||||
no valid anchor (first request, transcript rewritten under the anchor,
|
||||
provider never reported usage). A valid anchor is provider-exact and is
|
||||
used as-is; in particular on MoA turns the anchor deliberately uses the
|
||||
pre-fold aggregator usage while ``last_real_prompt_tokens`` holds the
|
||||
folded figure, so flooring an anchored value would re-add fan-out tokens
|
||||
the anchor exists to exclude.
|
||||
|
||||
On the rough path, non-ASCII text (Cyrillic, Greek, Polish, ...)
|
||||
under-counts by up to ~2x, so a session can sit at the provider's real
|
||||
context ceiling while the rough figure stays under the compaction
|
||||
threshold -- on silent-clip providers (ollama /v1) that is a truncation
|
||||
death spiral the reactive overflow handler never sees (observed live:
|
||||
real prompts 64,842->64,995 against a 55,705 threshold). The provider's
|
||||
last reported prompt_tokens is authoritative; never let the rough figure
|
||||
fall below it. Skipped for exactly one turn after a compaction, when
|
||||
last_real_prompt_tokens still holds the stale pre-compression value
|
||||
(#36718's awaiting_real_usage_after_compression window).
|
||||
"""
|
||||
last_real = int(getattr(compressor, "last_real_prompt_tokens", 0) or 0)
|
||||
if last_real > rough_tokens and not getattr(
|
||||
compressor, "awaiting_real_usage_after_compression", False
|
||||
):
|
||||
return last_real
|
||||
return rough_tokens
|
||||
|
||||
|
||||
def _ollama_context_limit_error(agent: Any, request_tokens: int) -> Optional[str]:
|
||||
"""Return a user-facing error when Ollama is loaded with too little context."""
|
||||
runtime_ctx = getattr(agent, "_ollama_num_ctx", None)
|
||||
|
||||
+19
-1
@@ -70,6 +70,11 @@ def file_uri(path: str) -> str:
|
||||
return "file://" + quote(abs_path, safe="/:")
|
||||
|
||||
|
||||
def _folder(root: str) -> Dict[str, str]:
|
||||
"""Build an LSP ``WorkspaceFolder`` for ``root``."""
|
||||
return {"name": os.path.basename(root.rstrip(os.sep)) or root, "uri": file_uri(root)}
|
||||
|
||||
|
||||
def uri_to_path(uri: str) -> str:
|
||||
"""Inverse of :func:`file_uri`."""
|
||||
if not uri.startswith("file://"):
|
||||
@@ -125,6 +130,9 @@ class LSPClient:
|
||||
seed_diagnostics_on_first_push: bool = False) -> None:
|
||||
self.server_id = server_id
|
||||
self.workspace_root = workspace_root
|
||||
# Roots this server serves. Single-root servers only ever hold ``workspace_root``;
|
||||
# multi-root servers (pyright) grow this via ``add_workspace_folder`` instead of a second process.
|
||||
self.workspace_folders: List[str] = [workspace_root]
|
||||
self._command = list(command)
|
||||
self._env = env
|
||||
self._cwd = cwd or workspace_root
|
||||
@@ -262,7 +270,17 @@ class LSPClient:
|
||||
await self._cleanup_process()
|
||||
|
||||
def _workspace_folders(self) -> List[Dict[str, str]]:
|
||||
return [{"name": "workspace", "uri": file_uri(self.workspace_root)}]
|
||||
return [_folder(r) for r in self.workspace_folders]
|
||||
|
||||
async def add_workspace_folder(self, root: str) -> None:
|
||||
"""Attach another root to a running multi-root server. Idempotent; the folder is recorded
|
||||
before the notification is sent so concurrent callers for the same root only announce once."""
|
||||
if root in self.workspace_folders:
|
||||
return
|
||||
self.workspace_folders.append(root)
|
||||
await self._send_notification(
|
||||
"workspace/didChangeWorkspaceFolders", {"event": {"added": [_folder(root)], "removed": []}},
|
||||
)
|
||||
|
||||
async def _initialize(self) -> None:
|
||||
params = {
|
||||
|
||||
+34
-16
@@ -2,7 +2,9 @@
|
||||
|
||||
:class:`LSPService` bridges the synchronous file_operations layer and the async
|
||||
:class:`agent.lsp.client.LSPClient`: one asyncio loop in a background thread, one lazily
|
||||
spawned client per ``(server_id, workspace_root)``, a **broken-set** of pairs that failed
|
||||
spawned client per ``(server_id, workspace_root)`` — servers flagged ``multi_root`` (pyright) get ONE
|
||||
client per ``server_id`` and further roots (typically sibling git worktrees) are attached to the running
|
||||
process via ``workspace/didChangeWorkspaceFolders`` — a **broken-set** of pairs that failed
|
||||
to spawn/initialize (never retried for the life of the service), and a **delta baseline**
|
||||
per file (``snapshot_baseline()`` runs BEFORE a write; the next ``get_diagnostics_sync()``
|
||||
returns only diagnostics not in it). Off unless config enables it.
|
||||
@@ -30,6 +32,12 @@ _Key = Tuple[str, str]
|
||||
_Diags = List[Dict[str, Any]]
|
||||
|
||||
|
||||
def _client_key(srv: ServerDef, root: str) -> _Key:
|
||||
"""Cache key for the client serving ``root``: multi-root servers share one process per
|
||||
``server_id``; everything else is keyed per resolved project root."""
|
||||
return (srv.server_id, "" if srv.multi_root else root)
|
||||
|
||||
|
||||
class _BackgroundLoop:
|
||||
"""A daemon thread owning one asyncio loop; :meth:`run` blocks on a coroutine."""
|
||||
|
||||
@@ -274,9 +282,10 @@ class LSPService:
|
||||
return
|
||||
already_broken = key in self._broken
|
||||
self._broken.add(key)
|
||||
ckey = _client_key(srv, key[1])
|
||||
with self._state_lock:
|
||||
client = self._clients.pop(key, None)
|
||||
self._last_used.pop(key, None)
|
||||
client = self._clients.pop(ckey, None)
|
||||
self._last_used.pop(ckey, None)
|
||||
if client is not None:
|
||||
try:
|
||||
# Fire-and-forget shutdown — we're already on a slow path.
|
||||
@@ -301,8 +310,9 @@ class LSPService:
|
||||
"""Return a snapshot of the service for ``hermes lsp status``."""
|
||||
with self._state_lock:
|
||||
clients = [
|
||||
{"server_id": k[0], "workspace_root": k[1], "state": c.state, "running": c.is_running}
|
||||
for k, c in self._clients.items()
|
||||
{"server_id": c.server_id, "workspace_root": c.workspace_root,
|
||||
"workspace_folders": list(c.workspace_folders), "state": c.state, "running": c.is_running}
|
||||
for c in self._clients.values()
|
||||
]
|
||||
broken = list(self._broken)
|
||||
return {
|
||||
@@ -349,7 +359,7 @@ class LSPService:
|
||||
if not (ws and gated and srv):
|
||||
return []
|
||||
with self._state_lock:
|
||||
client = self._clients.get((srv.server_id, ws))
|
||||
client = self._clients.get(_client_key(srv, ws))
|
||||
return list(client.diagnostics_for(file_path, fresh_only=True)) if client else []
|
||||
|
||||
async def _get_or_spawn(self, file_path: str) -> Optional[LSPClient]:
|
||||
@@ -367,39 +377,47 @@ class LSPService:
|
||||
if root is None:
|
||||
eventlog.log_disabled(srv.server_id, file_path, "exclude marker hit (server gated off)")
|
||||
return None
|
||||
key = (srv.server_id, root)
|
||||
if key in self._broken:
|
||||
if (srv.server_id, root) in self._broken:
|
||||
return None
|
||||
key = _client_key(srv, root)
|
||||
with self._state_lock:
|
||||
client = self._clients.get(key)
|
||||
if client is not None and client.is_running:
|
||||
self._last_used[key] = time.time()
|
||||
eventlog.log_active(*key)
|
||||
return client
|
||||
eventlog.log_active(srv.server_id, root)
|
||||
return await self._attach_root(srv, client, root)
|
||||
spawning = self._spawning.get(key)
|
||||
owner = spawning is None
|
||||
if owner:
|
||||
spawning = self._spawning[key] = asyncio.get_running_loop().create_future()
|
||||
if not owner:
|
||||
try:
|
||||
return await spawning
|
||||
client = await spawning
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
return await self._attach_root(srv, client, root) if client is not None else None
|
||||
try:
|
||||
client = await self._spawn_client(srv, root)
|
||||
if client is None:
|
||||
self._broken.add(key)
|
||||
self._broken.add((srv.server_id, root))
|
||||
else:
|
||||
with self._state_lock:
|
||||
self._clients[key] = client
|
||||
self._last_used[key] = time.time()
|
||||
eventlog.log_active(*key)
|
||||
eventlog.log_active(srv.server_id, root)
|
||||
spawning.set_result(client)
|
||||
return client
|
||||
finally:
|
||||
with self._state_lock:
|
||||
self._spawning.pop(key, None)
|
||||
|
||||
@staticmethod
|
||||
async def _attach_root(srv: ServerDef, client: LSPClient, root: str) -> LSPClient:
|
||||
"""Multi-root servers: announce ``root`` to the shared process instead of spawning another."""
|
||||
if srv.multi_root:
|
||||
await client.add_workspace_folder(root)
|
||||
return client
|
||||
|
||||
async def _spawn_client(self, srv: ServerDef, root: str) -> Optional[LSPClient]:
|
||||
"""Resolve the binary and start a client; ``None`` (after logging) when either fails."""
|
||||
ctx = ServerContext(
|
||||
@@ -425,10 +443,10 @@ class LSPService:
|
||||
|
||||
def _touch(self, client: LSPClient) -> None:
|
||||
"""Refresh last-used; guarded on membership so a client reaped mid-operation can't resurrect its entry."""
|
||||
key = (client.server_id, client.workspace_root)
|
||||
with self._state_lock:
|
||||
if key in self._clients:
|
||||
self._last_used[key] = time.time()
|
||||
for key, c in self._clients.items():
|
||||
if c is client:
|
||||
self._last_used[key] = time.time()
|
||||
|
||||
async def _start_idle_reaper(self) -> None:
|
||||
self._idle_reaper_task = asyncio.create_task(self._idle_reaper_loop())
|
||||
|
||||
@@ -72,6 +72,9 @@ class ServerDef:
|
||||
build_spawn: _SpawnFn
|
||||
seed_first_push: bool = False
|
||||
description: str = ""
|
||||
# Server handles ``workspace/didChangeWorkspaceFolders``: one process serves every project root
|
||||
# (git worktrees included) as extra workspaceFolders instead of one process per root.
|
||||
multi_root: bool = False
|
||||
|
||||
def matches(self, file_path: str) -> bool:
|
||||
return _file_ext_or_basename(file_path) in self.extensions
|
||||
@@ -263,20 +266,21 @@ def _server(server_id: str, extensions: Tuple[str, ...], description: str, *,
|
||||
markers: Optional[Sequence[str]] = None, excludes: Sequence[str] = (),
|
||||
resolve_root: Optional[_RootFn] = None, build_spawn: Optional[_SpawnFn] = None,
|
||||
which: Sequence[str] = (), args: Sequence[str] = (), install_pkg: Optional[str] = None,
|
||||
base_init: Optional[Dict[str, Any]] = None, seed: bool = False) -> ServerDef:
|
||||
base_init: Optional[Dict[str, Any]] = None, seed: bool = False,
|
||||
multi_root: bool = False) -> ServerDef:
|
||||
"""Registry entry factory: defaults to marker-based root + single-binary spawn."""
|
||||
return ServerDef(
|
||||
server_id, extensions,
|
||||
resolve_root or _markers_root(markers, excludes),
|
||||
build_spawn or _simple_spawn(server_id, which or (server_id,), args, install_pkg, base_init, seed),
|
||||
seed_first_push=seed, description=description,
|
||||
seed_first_push=seed, description=description, multi_root=multi_root,
|
||||
)
|
||||
|
||||
|
||||
SERVERS: List[ServerDef] = [
|
||||
_server("pyright", (".py", ".pyi"), "Python — Microsoft pyright",
|
||||
markers=["pyproject.toml", "setup.py", "setup.cfg", "requirements.txt", "Pipfile", "pyrightconfig.json"],
|
||||
build_spawn=_spawn_pyright),
|
||||
build_spawn=_spawn_pyright, multi_root=True),
|
||||
_server("typescript", (".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".mts", ".cts"),
|
||||
"JavaScript/TypeScript — typescript-language-server", resolve_root=_root_typescript,
|
||||
which=("typescript-language-server",), args=("--stdio",), install_pkg="typescript-language-server", seed=True),
|
||||
|
||||
@@ -120,7 +120,9 @@ def nearest_root(
|
||||
# Excludes are checked before markers at each level.
|
||||
if present(cur, excludes_list):
|
||||
return None
|
||||
if present(cur, markers_list):
|
||||
# A directory holding __init__.py is a Python package, never a project root (hermes_cli/setup.py
|
||||
# matched the python marker list and gave every package dir its own pyright).
|
||||
if not present(cur, ["__init__.py"]) and present(cur, markers_list):
|
||||
return str(cur)
|
||||
if ceiling_path is not None and cur == ceiling_path:
|
||||
return None
|
||||
|
||||
+22
-9
@@ -337,8 +337,10 @@ DEFAULT_CONTEXT_LENGTHS = {
|
||||
# https://api-docs.deepseek.com/zh-cn/quick_start/pricing
|
||||
"deepseek-v4-pro": 1_000_000, "deepseek-v4-flash": 1_000_000, "deepseek-chat": 1_000_000,
|
||||
"deepseek-reasoner": 1_000_000, "deepseek": 128000,
|
||||
# Meta; Thinking Machines inkling (covers inkling-small and :free/:batch variants)
|
||||
"llama": 131072, "inkling": 1_048_576,
|
||||
# Meta; Muse Spark family (1.1/1.2/1.3, -contributor(-free), meta/ prefixed) is 1M per OpenRouter,
|
||||
# models.dev and api.commandcode.ai /models — keep the "muse-spark" prefix (bare "muse" would match
|
||||
# muse-image/muse-voice). Thinking Machines inkling (covers inkling-small and :free/:batch variants)
|
||||
"llama": 131072, "muse-spark-1.3": 1_048_576, "muse-spark": 1_048_576, "inkling": 1_048_576,
|
||||
# Qwen — https://help.aliyun.com/zh/model-studio/developer-reference/ (3.8-max/flash
|
||||
# 1M verified on OpenRouter & Nous portal 2026-08; qwen3-max = 256K Coding Plan snapshot)
|
||||
"qwen3.8-max": 1_000_000, "qwen3.8-flash": 1_000_000, "qwen3.6-plus": 1048576, "qwen3.7-plus": 1048576,
|
||||
@@ -1269,6 +1271,7 @@ def _model_name_suggests_minimax_m3(model: str) -> bool:
|
||||
# shorter matching key and the 256K fallback — the threshold is inferred from them.
|
||||
_PRE_CATALOG_STALE_KEYS = frozenset({
|
||||
"minimax-m3", # 1M; "minimax" catch-all persisted 204,800
|
||||
"muse-spark-1.3", "muse-spark", # 1M; pre-entry builds fell through to the 256K fallback
|
||||
"grok-4.3", "grok-4.6", # 1M / 500K; "grok-4" catch-all persisted 256,000
|
||||
"grok-4-fast", "grok-4.20", # 2M; fell through to the 256K fallback
|
||||
"qwen3.6-plus", # 1M; "qwen" catch-all persisted 131,072
|
||||
@@ -1775,8 +1778,9 @@ def _resolve_provider_aware_context_length(model: str, base_url: str, api_key: s
|
||||
if base_url and source == persist_on:
|
||||
save_context_length(model, base_url, ctx)
|
||||
return ctx
|
||||
if effective_provider == "gmi" and base_url:
|
||||
# GMI exposes authoritative context_length via /models, but it is not in models.dev yet.
|
||||
if effective_provider in {"gmi", "commandcode", "commandcode-anthropic"} and base_url:
|
||||
# GMI and CommandCode expose authoritative context_length via /models (e.g. muse-spark 1M) but are
|
||||
# not in models.dev, and as known providers they skip step 2's probe — else they fell to 256K.
|
||||
ctx = _resolve_endpoint_context_length(model, base_url, api_key=api_key)
|
||||
if ctx is not None:
|
||||
return ctx
|
||||
@@ -1924,14 +1928,23 @@ def _is_cjk_token_dense_char(ch: str) -> bool:
|
||||
|
||||
|
||||
def estimate_tokens_rough(text: str) -> int:
|
||||
"""Rough token estimate: ceil(chars/4), CJK/Hangul/Kana codepoints ~1 token each. Ceiling keeps
|
||||
short texts from estimating 0. Runs on every preflight walk, so the all-ASCII case stays O(1)."""
|
||||
"""Rough token estimate: CJK/Hangul/Kana codepoints ~1 token each; everything else ceil(UTF-8 bytes/4).
|
||||
Ceiling keeps short texts from estimating 0. Runs on every preflight walk, so all-ASCII stays O(1).
|
||||
|
||||
Byte-counting (not chars) is the corrective for non-CJK, non-ASCII text: Cyrillic/Greek/Arabic are 2
|
||||
bytes/char so count ~chars/2, matching real BPE cost (~2-3 chars/token) where chars/4 under-counted
|
||||
~2x and let sessions ride the provider ceiling below the compaction threshold. Calibrated vs
|
||||
cl100k/o200k/Qwen2.5 (estimate/real): Russian 0.67->1.24, Arabic 0.53->0.96, Hindi 0.34->0.90,
|
||||
Greek 0.37->0.68; accented Latin barely moves (French 1.02->1.03). errors="replace": lone surrogates
|
||||
(routine in tool output; see message_sanitization) must not turn an estimate into a raise."""
|
||||
if not text:
|
||||
return 0
|
||||
text = str(text)
|
||||
# ``str.isascii()`` is a flag check on CPython; non-ASCII without CJK (accents, Cyrillic, emoji) also gets chars/4.
|
||||
dense = 0 if text.isascii() else len(text) - len(_CJK_DENSE_RE.sub("", text))
|
||||
return dense + ((len(text) - dense + 3) // 4)
|
||||
if text.isascii(): # flag check on CPython; ASCII cannot contain token-dense CJK
|
||||
return (len(text) + 3) // 4
|
||||
stripped = _CJK_DENSE_RE.sub("", text)
|
||||
dense = len(text) - len(stripped)
|
||||
return dense + ((len(stripped.encode("utf-8", "replace")) + 3) // 4)
|
||||
|
||||
|
||||
def estimate_messages_tokens_rough(messages: List[Dict[str, Any]], *, charge_stale_thinking: bool = True) -> int:
|
||||
|
||||
+5
-1
@@ -114,7 +114,11 @@ PROVIDER_TO_MODELS_DEV: Dict[str, str] = {
|
||||
"minimax-oauth": "minimax", "minimax-cn": "minimax-cn", "deepseek": "deepseek",
|
||||
"alibaba": "alibaba", "qwen-oauth": "alibaba", "copilot": "github-copilot",
|
||||
"ai-gateway": "vercel", "opencode-zen": "opencode",
|
||||
"opencode-go": "opencode-go", "kilocode": "kilo", "fireworks": "fireworks-ai",
|
||||
"opencode-go": "opencode-go",
|
||||
# opencode-free is Zen-hosted (hermes_cli/models.py) and models.dev's "opencode" catalog lists
|
||||
# its *-contributor-free SKUs; without this alias every opencode-free lookup missed models.dev.
|
||||
"opencode-free": "opencode",
|
||||
"kilocode": "kilo", "fireworks": "fireworks-ai",
|
||||
"huggingface": "huggingface", "gemini": "google", "google": "google",
|
||||
"xai": "xai",
|
||||
"xai-oauth": "xai", # OAuth is a transport path for the same xAI catalog
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
"""``x-opencode-session`` — OpenCode relay session-affinity header.
|
||||
|
||||
OpenCode (opencode.ai Zen/Go/free relay) pins requests that share an
|
||||
``x-opencode-session`` value to the same upstream backend, which is what
|
||||
keeps its prompt cache warm across the turns of one conversation. The value
|
||||
only has to be opaque and consistent per conversation, so it is derived the
|
||||
same way as the other conversation-affinity hints Hermes already sends
|
||||
(OpenRouter's sticky ``session_id``, xAI's ``x-grok-conv-id``): the
|
||||
host-declared routing scope first, then the ambient conversation root, then
|
||||
the physical session id — normalized through ``_cache_scope_from_session_id``
|
||||
so cron fires of one job share a scope.
|
||||
|
||||
Every OpenCode request — main turn on any transport, auxiliary calls
|
||||
(compression, titles, vision, MoA) — goes through :func:`opencode_session_headers`
|
||||
so the header cannot drift per code path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Optional
|
||||
|
||||
OPENCODE_SESSION_HEADER = "x-opencode-session"
|
||||
|
||||
|
||||
def is_opencode_target(provider: Optional[str], base_url: Optional[str]) -> bool:
|
||||
"""True when *provider* or *base_url* addresses the OpenCode relay.
|
||||
|
||||
Matches the built-in opencode-zen/go/free providers, custom
|
||||
``opencode-<family>-*`` providers, and any base_url hosted on opencode.ai.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.models import opencode_provider_family
|
||||
|
||||
if opencode_provider_family(provider) is not None:
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
from agent.anthropic_endpoints import _is_opencode_endpoint
|
||||
|
||||
return _is_opencode_endpoint(str(base_url or ""))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def opencode_session_headers(
|
||||
provider: Optional[str],
|
||||
base_url: Optional[str],
|
||||
session_id: Optional[str] = None,
|
||||
) -> dict[str, str]:
|
||||
"""Return ``{"x-opencode-session": <key>}`` for OpenCode targets, else ``{}``."""
|
||||
if not is_opencode_target(provider, base_url):
|
||||
return {}
|
||||
try:
|
||||
from agent.portal_tags import get_affinity_scope, get_conversation_context
|
||||
from agent.transports.codex import _cache_scope_from_session_id
|
||||
|
||||
key = _cache_scope_from_session_id(
|
||||
get_affinity_scope() or get_conversation_context() or session_id
|
||||
)
|
||||
except Exception:
|
||||
key = str(session_id or "")
|
||||
return {OPENCODE_SESSION_HEADER: key} if key else {}
|
||||
|
||||
|
||||
def merge_opencode_session_headers(
|
||||
kwargs: dict[str, Any],
|
||||
provider: Optional[str],
|
||||
base_url: Optional[str],
|
||||
session_id: Optional[str] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Merge the affinity header into ``kwargs["extra_headers"]`` (in place).
|
||||
|
||||
Existing per-request headers win, so a caller-pinned value is preserved.
|
||||
Non-OpenCode targets are left untouched.
|
||||
"""
|
||||
headers = opencode_session_headers(provider, base_url, session_id)
|
||||
if headers:
|
||||
existing = kwargs.get("extra_headers")
|
||||
merged = dict(existing) if isinstance(existing, dict) else {}
|
||||
for key, value in headers.items():
|
||||
merged.setdefault(key, value)
|
||||
kwargs["extra_headers"] = merged
|
||||
return kwargs
|
||||
@@ -0,0 +1,118 @@
|
||||
"""One process-wide timer thread for periodic maintenance callbacks.
|
||||
|
||||
Replaces the per-child ``while not stop.wait(interval): body()`` daemon
|
||||
threads (delegate heartbeat, durable turn-lease refresher, turn-liveness
|
||||
watchdog). With ~130 in-process subagents those added 2-3 sleeping OS
|
||||
threads per child; this module runs every periodic body on ONE daemon
|
||||
thread ordered by a heap of due times.
|
||||
|
||||
Semantics match the loop they replace: the first call happens ``interval``
|
||||
seconds after :func:`schedule`, and each following call ``interval`` seconds
|
||||
after the previous body *returned* (drift-free wrt. body duration was never
|
||||
a property of the old loops either). A body that returns ``False`` stops
|
||||
itself; a body that raises is logged at debug and rescheduled — one bad
|
||||
callback must never kill the shared thread.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import heapq
|
||||
import itertools
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Callable, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_THREAD_NAME = "hermes-periodic-scheduler"
|
||||
|
||||
|
||||
class ScheduledHandle:
|
||||
"""Cancel token for one scheduled periodic callback."""
|
||||
|
||||
__slots__ = ("_fn", "_interval", "_cancelled", "_scheduler")
|
||||
|
||||
def __init__(self, scheduler: "PeriodicScheduler", fn: Callable[[], object], interval: float):
|
||||
self._scheduler = scheduler
|
||||
self._fn = fn
|
||||
self._interval = interval
|
||||
self._cancelled = False
|
||||
|
||||
@property
|
||||
def cancelled(self) -> bool:
|
||||
return self._cancelled
|
||||
|
||||
def cancel(self, wait: Optional[float] = None) -> None:
|
||||
"""Stop future runs. ``wait`` (seconds) additionally blocks until an
|
||||
in-flight run of this callback finishes — the analogue of
|
||||
``thread.join(timeout=wait)`` on the old per-child thread."""
|
||||
self._scheduler._cancel(self, wait)
|
||||
|
||||
|
||||
class PeriodicScheduler:
|
||||
def __init__(self) -> None:
|
||||
self._cond = threading.Condition()
|
||||
self._heap: list = [] # (due, seq, handle)
|
||||
self._seq = itertools.count()
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._running: Optional[ScheduledHandle] = None
|
||||
|
||||
def schedule(self, fn: Callable[[], object], interval: float) -> ScheduledHandle:
|
||||
handle = ScheduledHandle(self, fn, float(interval))
|
||||
with self._cond:
|
||||
heapq.heappush(self._heap, (time.monotonic() + handle._interval, next(self._seq), handle))
|
||||
if self._thread is None or not self._thread.is_alive():
|
||||
self._thread = threading.Thread(target=self._run, name=_THREAD_NAME, daemon=True)
|
||||
self._thread.start()
|
||||
self._cond.notify()
|
||||
return handle
|
||||
|
||||
def _cancel(self, handle: ScheduledHandle, wait: Optional[float]) -> None:
|
||||
with self._cond:
|
||||
handle._cancelled = True
|
||||
self._cond.notify()
|
||||
if wait and self._running is handle and threading.current_thread() is not self._thread:
|
||||
self._cond.wait_for(lambda: self._running is not handle, timeout=wait)
|
||||
|
||||
def _run(self) -> None:
|
||||
while True:
|
||||
with self._cond:
|
||||
while True:
|
||||
if not self._heap:
|
||||
self._cond.wait()
|
||||
continue
|
||||
due, _, handle = self._heap[0]
|
||||
if handle._cancelled:
|
||||
heapq.heappop(self._heap)
|
||||
continue
|
||||
delay = due - time.monotonic()
|
||||
if delay > 0:
|
||||
self._cond.wait(delay)
|
||||
continue
|
||||
heapq.heappop(self._heap)
|
||||
self._running = handle
|
||||
break
|
||||
stop = False
|
||||
try:
|
||||
stop = handle._fn() is False
|
||||
except Exception:
|
||||
logger.debug("periodic callback %r raised", handle._fn, exc_info=True)
|
||||
with self._cond:
|
||||
self._running = None
|
||||
if stop:
|
||||
handle._cancelled = True
|
||||
elif not handle._cancelled:
|
||||
heapq.heappush(
|
||||
self._heap,
|
||||
(time.monotonic() + handle._interval, next(self._seq), handle),
|
||||
)
|
||||
self._cond.notify_all()
|
||||
|
||||
|
||||
_DEFAULT = PeriodicScheduler()
|
||||
|
||||
|
||||
def schedule(fn: Callable[[], object], interval: float) -> ScheduledHandle:
|
||||
"""Run ``fn()`` every ``interval`` seconds on the shared scheduler thread."""
|
||||
return _DEFAULT.schedule(fn, interval)
|
||||
+129
-6
@@ -13,6 +13,7 @@ import os
|
||||
import selectors
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
from typing import Any, Optional
|
||||
@@ -23,6 +24,19 @@ from utils import base_url_hostname, normalize_proxy_url
|
||||
_OPENAI_CLS_CACHE = None
|
||||
_HAPPY_EYEBALLS_DELAY_SECONDS = 0.25
|
||||
|
||||
# Process-wide pool of sync ``httpx.HTTPTransport`` objects shared by every
|
||||
# keepalive client with the same (verify, proxy, happy-eyeballs) identity.
|
||||
# Each delegated child AIAgent used to get its own transport = its own TLS
|
||||
# pool, so a fan-out of N children held N separate socket sets to the same
|
||||
# provider. Bounded: past the cap, callers get a private transport again.
|
||||
_SHARED_TRANSPORTS: dict[tuple, Any] = {}
|
||||
_SHARED_TRANSPORTS_LOCK = threading.Lock()
|
||||
_SHARED_TRANSPORTS_MAX = 32
|
||||
# ``request.extensions`` key stamped by ``_SharedTransport.handle_request``;
|
||||
# the socket-abort walker in agent_runtime_helpers uses it to find only the
|
||||
# owning client's in-flight connections on a shared pool.
|
||||
HERMES_TRANSPORT_OWNER_EXT = "hermes_transport_owner"
|
||||
|
||||
|
||||
def _interleave_addrinfos(addrinfos: list[tuple]) -> list[tuple]:
|
||||
"""Round-robin the resolved address families (deduped), preserving resolver order within each."""
|
||||
@@ -283,6 +297,86 @@ def _get_proxy_for_base_url(base_url: Optional[str]) -> Optional[str]:
|
||||
return proxy
|
||||
|
||||
|
||||
def _shared_transport_cls():
|
||||
"""Lazily define the per-client transport view (httpx import is deferred)."""
|
||||
global _SharedTransport
|
||||
if _SharedTransport is not None:
|
||||
return _SharedTransport
|
||||
import httpx
|
||||
|
||||
class _SharedTransportImpl(httpx.BaseTransport):
|
||||
"""Per-client view of a process-shared ``httpx.HTTPTransport``.
|
||||
|
||||
``httpx.Client.close()`` closes every mounted transport, and each OpenAI client still
|
||||
owns its own ``httpx.Client`` (closing one client must never poison the next), so the
|
||||
mounted object absorbs that close while the shared pool keeps serving other clients.
|
||||
``handle_request`` stamps the owning view into ``request.extensions`` so socket-abort
|
||||
sweeps target only this client's in-flight connections on the shared pool.
|
||||
"""
|
||||
|
||||
__slots__ = ("_inner", "_closed")
|
||||
|
||||
def __init__(self, inner: Any) -> None:
|
||||
self._inner = inner
|
||||
self._closed = False
|
||||
|
||||
@property
|
||||
def _pool(self) -> Any: # httpx-private; socket walkers and tests introspect it
|
||||
return getattr(self._inner, "_pool", None)
|
||||
|
||||
def handle_request(self, request: Any) -> Any:
|
||||
if self._closed:
|
||||
raise RuntimeError("Cannot send a request, as the client has been closed.")
|
||||
request.extensions[HERMES_TRANSPORT_OWNER_EXT] = id(self)
|
||||
return self._inner.handle_request(request)
|
||||
|
||||
def close(self) -> None:
|
||||
# Never closes the shared ``_inner``; idle connections are reaped by keepalive_expiry
|
||||
# and the pool lives for the process (see ``close_shared_transports``).
|
||||
self._closed = True
|
||||
|
||||
_SharedTransportImpl.__name__ = _SharedTransportImpl.__qualname__ = "_SharedTransport"
|
||||
_SharedTransport = _SharedTransportImpl
|
||||
return _SharedTransport
|
||||
|
||||
|
||||
_SharedTransport: Any = None
|
||||
|
||||
|
||||
def _shared_transport_key(base_url: str, verify: Any, proxy: Optional[str]) -> tuple:
|
||||
"""Identity under which sync direct transports are pooled process-wide."""
|
||||
if verify is True or verify is False:
|
||||
verify_key: Any = verify
|
||||
elif isinstance(verify, str):
|
||||
verify_key = ("path", verify)
|
||||
else:
|
||||
verify_key = ("id", id(verify)) # SSLContext / custom object: share by identity only
|
||||
return (verify_key, proxy, _uses_codex_cloud_transport(base_url))
|
||||
|
||||
|
||||
def _get_shared_transport(key: tuple, build) -> Any:
|
||||
with _SHARED_TRANSPORTS_LOCK:
|
||||
transport = _SHARED_TRANSPORTS.get(key)
|
||||
if transport is None:
|
||||
transport = build()
|
||||
if len(_SHARED_TRANSPORTS) < _SHARED_TRANSPORTS_MAX:
|
||||
_SHARED_TRANSPORTS[key] = transport
|
||||
return transport
|
||||
|
||||
|
||||
def close_shared_transports() -> int:
|
||||
"""Really close every process-shared transport (test teardown / atexit)."""
|
||||
with _SHARED_TRANSPORTS_LOCK:
|
||||
transports = list(_SHARED_TRANSPORTS.values())
|
||||
_SHARED_TRANSPORTS.clear()
|
||||
for transport in transports:
|
||||
try:
|
||||
transport.close()
|
||||
except Exception:
|
||||
pass
|
||||
return len(transports)
|
||||
|
||||
|
||||
def build_keepalive_http_client(base_url: str = "", *, async_mode: bool = False, verify: Any = True) -> Optional[Any]:
|
||||
"""httpx client for OpenAI SDK calls with env-only proxy policy (None on failure).
|
||||
|
||||
@@ -291,6 +385,12 @@ def build_keepalive_http_client(base_url: str = "", *, async_mode: bool = False,
|
||||
reaps idle connections before reverse proxies' 30-60 s timeouts (a custom
|
||||
socket_options transport broke streaming and stripped TCP_NODELAY). ``verify``
|
||||
goes on the client AND the mounts, since a mounted transport owns its SSL context.
|
||||
|
||||
Every call returns a NEW ``httpx.Client`` (per-client close semantics), but sync clients
|
||||
with the same (verify, proxy, happy-eyeballs) identity mount the SAME underlying
|
||||
``HTTPTransport`` through a ``_SharedTransport`` view, so N delegated children share one
|
||||
connection pool + SSL context. Async clients are never shared: an httpcore async pool is
|
||||
bound to the event loop that first used it. Proxy-backed clients keep httpx's own transport.
|
||||
"""
|
||||
try:
|
||||
import httpx
|
||||
@@ -301,12 +401,34 @@ def build_keepalive_http_client(base_url: str = "", *, async_mode: bool = False,
|
||||
client_cls = httpx.AsyncClient if async_mode else httpx.Client
|
||||
mounts = None
|
||||
if proxy is None:
|
||||
mounts = {"http://": transport_cls(verify=verify), "https://": transport_cls(verify=verify)}
|
||||
# Async transports race natively (anyio happy_eyeballs_delay=0.25).
|
||||
if not async_mode and _uses_codex_cloud_transport(base_url):
|
||||
for transport in mounts.values():
|
||||
happy_eyeballs = not async_mode and _uses_codex_cloud_transport(base_url)
|
||||
# One pool serves every agent in the process, so its ceiling must cover a whole
|
||||
# fan-out of concurrently streaming children. (Client-level ``limits`` never reach
|
||||
# mounted transports — they used to run on httpx defaults, keepalive_expiry=5s.)
|
||||
direct_limits = limits if async_mode else httpx.Limits(
|
||||
max_keepalive_connections=50, max_connections=1000, keepalive_expiry=20.0,
|
||||
)
|
||||
|
||||
def _build_direct():
|
||||
transport = transport_cls(verify=verify, limits=direct_limits)
|
||||
# Async transports race natively (anyio happy_eyeballs_delay=0.25).
|
||||
if happy_eyeballs:
|
||||
_enable_happy_eyeballs(transport)
|
||||
return client_cls(limits=limits, timeout=timeout, proxy=proxy, mounts=mounts, verify=verify)
|
||||
return transport
|
||||
|
||||
if async_mode:
|
||||
mounts = {"http://": _build_direct(), "https://": _build_direct()}
|
||||
else:
|
||||
key = _shared_transport_key(base_url, verify, proxy)
|
||||
view_cls = _shared_transport_cls()
|
||||
mounts = {
|
||||
f"{scheme}://": view_cls(_get_shared_transport((scheme, *key), _build_direct))
|
||||
for scheme in ("http", "https")
|
||||
}
|
||||
# Default transport = the https view; otherwise httpx builds a third, never-used
|
||||
# direct transport (pool + SSL context) per client.
|
||||
return client_cls(limits=limits, timeout=timeout, transport=mounts["https://"], mounts=mounts)
|
||||
return client_cls(limits=limits, timeout=timeout, proxy=proxy, mounts=mounts or None, verify=verify)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@@ -325,5 +447,6 @@ OpenAI = _OpenAIProxy()
|
||||
|
||||
__all__ = [
|
||||
"OpenAI", "_OpenAIProxy", "_load_openai_cls", "_SafeWriter", "_install_safe_stdio", "_get_proxy_from_env",
|
||||
"_get_proxy_for_base_url", "build_keepalive_http_client", "enable_happy_eyeballs_on_client",
|
||||
"_get_proxy_for_base_url", "build_keepalive_http_client", "close_shared_transports",
|
||||
"enable_happy_eyeballs_on_client",
|
||||
]
|
||||
|
||||
+75
-8
@@ -8,6 +8,7 @@ import contextvars
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import sys
|
||||
import threading
|
||||
from collections import OrderedDict
|
||||
@@ -31,6 +32,51 @@ from utils import atomic_json_write
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Default read deadline for context files (SOUL.md, AGENTS.md, .cursorrules,
|
||||
# ...); overridable via ``context_file_read_timeout`` in config.yaml.
|
||||
# Intentionally short: network-backed filesystems (iCloud Drive, OneDrive,
|
||||
# NFS) can fault-in an evicted file and block a cold read indefinitely, which
|
||||
# stalls system-prompt assembly before the first turn.
|
||||
_CONTEXT_FILE_READ_TIMEOUT_SECS = 5.0
|
||||
|
||||
|
||||
def _get_context_file_read_timeout() -> float:
|
||||
"""``context_file_read_timeout`` from config.yaml, else the 5s default."""
|
||||
val = _config_readonly("context_file_read_timeout").get("context_file_read_timeout")
|
||||
if isinstance(val, (int, float)) and val > 0:
|
||||
return float(val)
|
||||
return _CONTEXT_FILE_READ_TIMEOUT_SECS
|
||||
|
||||
|
||||
def _read_text_with_timeout(path: Path, timeout: Optional[float] = None) -> Optional[str]:
|
||||
"""``path.read_text()`` on a daemon thread so a slow file can't stall startup.
|
||||
|
||||
Returns the text, or ``None`` after *timeout* seconds (logged at WARNING;
|
||||
the orphaned reader thread finishes on its own). Read errors propagate to
|
||||
the caller exactly as a direct ``read_text`` would, so existing
|
||||
``try/except`` handling at each site is unchanged.
|
||||
"""
|
||||
if timeout is None:
|
||||
timeout = _get_context_file_read_timeout()
|
||||
result: "queue.Queue[tuple[bool, object]]" = queue.Queue(maxsize=1)
|
||||
|
||||
def _reader() -> None:
|
||||
try:
|
||||
result.put((True, path.read_text(encoding="utf-8")))
|
||||
except Exception as exc: # re-raised on the caller thread
|
||||
result.put((False, exc))
|
||||
|
||||
threading.Thread(target=_reader, daemon=True, name=f"context-read:{path.name}").start()
|
||||
try:
|
||||
ok, value = result.get(timeout=timeout)
|
||||
except queue.Empty:
|
||||
logger.warning("Context file %s read timed out after %.1fs; skipping", path, timeout)
|
||||
return None
|
||||
if ok:
|
||||
return value # type: ignore[return-value]
|
||||
raise value # type: ignore[misc]
|
||||
|
||||
|
||||
def _scan_context_content(content: str, filename: str) -> str:
|
||||
"""Scan a context file (AGENTS.md, .cursorrules, SOUL.md) for injection; matches are BLOCKED.
|
||||
|
||||
@@ -245,14 +291,17 @@ TOOL_USE_ENFORCEMENT_GUIDANCE = (
|
||||
"user. Responses that only describe intentions without acting are not acceptable."
|
||||
)
|
||||
|
||||
TOOL_USE_ENFORCEMENT_MODELS = ("gpt", "codex", "gemini", "gemma", "grok", "glm", "qwen", "deepseek")
|
||||
# "muse" = Meta Muse Spark: on defaults it answers in prose with 0 tool calls and the turn closes on
|
||||
# finish_reason=stop (#96550).
|
||||
TOOL_USE_ENFORCEMENT_MODELS = ("gpt", "codex", "gemini", "gemma", "grok", "glm", "qwen", "deepseek", "muse")
|
||||
|
||||
# Models that receive OPENAI_MODEL_EXECUTION_GUIDANCE when agent.execution_guidance is "auto" (agentic-eval
|
||||
# traces showed the same failure modes). Gemini/Gemma get GOOGLE_MODEL_OPERATIONAL_GUIDANCE instead; Claude
|
||||
# does not exhibit these modes. Any model can opt in via config.yaml (`true` or a substring list).
|
||||
# traces showed the same failure modes; Muse Spark stops after a chat-only turn on defaults). Gemini/Gemma get
|
||||
# GOOGLE_MODEL_OPERATIONAL_GUIDANCE instead; Claude does not exhibit these modes. Any model can opt in via
|
||||
# config.yaml (`true` or a substring list).
|
||||
EXECUTION_GUIDANCE_MODELS = (
|
||||
"gpt", "codex", "grok",
|
||||
"deepseek", "kimi", "qwen", "glm", "minimax", "mimo", "mistral",
|
||||
"deepseek", "kimi", "qwen", "glm", "minimax", "mimo", "mistral", "muse",
|
||||
)
|
||||
|
||||
# Universal "finish the job" guidance (ALL models): don't stop after a stub, never
|
||||
@@ -734,17 +783,30 @@ _BACKEND_PROBE_CMD = (
|
||||
|
||||
def _run_backend_probe(env_type: str, terminal_tool) -> str:
|
||||
"""Execute the probe command inside a freshly built backend; "" when it yields nothing."""
|
||||
from tools.terminal_tool_backends import _ssh_config_from_config
|
||||
|
||||
config = terminal_tool._get_env_config()
|
||||
# Mirrors tools/terminal_tool.py's live-command assembly (`_create_environment` is the factory).
|
||||
env = terminal_tool._create_environment(
|
||||
env_type=env_type, image=config.get(_BACKEND_IMAGE_KEYS[env_type], "") if env_type in _BACKEND_IMAGE_KEYS else "", cwd=config.get("cwd", ""),
|
||||
timeout=config.get("timeout", 180),
|
||||
ssh_config=terminal_tool._ssh_config_from_config(config) if env_type == "ssh" else None,
|
||||
ssh_config=_ssh_config_from_config(config) if env_type == "ssh" else None,
|
||||
container_config=({k: config.get(k, d) for k, d in _CONTAINER_CONFIG_DEFAULTS}
|
||||
if terminal_tool._is_container_backend(env_type) else None),
|
||||
task_id="prompt-backend-probe", host_cwd=config.get("host_cwd"),
|
||||
)
|
||||
result = env.execute(_BACKEND_PROBE_CMD, timeout=4)
|
||||
try:
|
||||
result = env.execute(_BACKEND_PROBE_CMD, timeout=4)
|
||||
finally:
|
||||
# One-shot `uname`; without teardown the backend leaves a second idle sandbox
|
||||
# (task_id="prompt-backend-probe") running for the whole process next to the agent's own.
|
||||
# ssh is left alone: no task-scoped sandbox, and its cleanup() closes a ControlMaster socket
|
||||
# (keyed by user@host:port) shared with the agent's real environment; ControlPersist expires it.
|
||||
if env_type != "ssh":
|
||||
try:
|
||||
terminal_tool._cleanup_env(env, force_remove=True)
|
||||
except Exception:
|
||||
logger.debug("Backend probe cleanup failed", exc_info=True)
|
||||
if result.get("returncode") != 0:
|
||||
logger.debug("Backend probe returned non-zero: %r", result)
|
||||
return ""
|
||||
@@ -781,6 +843,11 @@ def _probe_remote_backend(env_type: str) -> str | None:
|
||||
return formatted or None
|
||||
|
||||
|
||||
def _clear_backend_probe_cache() -> None:
|
||||
"""Test helper — drop the backend probe cache so monkeypatched backends take effect."""
|
||||
_BACKEND_PROBE_CACHE.clear()
|
||||
|
||||
|
||||
def _local_host_hints() -> list[str]:
|
||||
"""Host OS / home / cwd block for a local terminal backend (tools run on this host)."""
|
||||
import platform
|
||||
@@ -1295,7 +1362,7 @@ def load_soul_md(context_length: Optional[int] = None, home_override: "Path | No
|
||||
if not soul_path.exists():
|
||||
return None
|
||||
try:
|
||||
content = soul_path.read_text(encoding="utf-8").strip()
|
||||
content = (_read_text_with_timeout(soul_path) or "").strip()
|
||||
if not content:
|
||||
return None
|
||||
return _truncate_content(_scan_context_content(content, "SOUL.md"), "SOUL.md", context_length=context_length,
|
||||
@@ -1310,7 +1377,7 @@ def _read_context_file(path: Path) -> str:
|
||||
if not path.exists():
|
||||
return ""
|
||||
try:
|
||||
return path.read_text(encoding="utf-8").strip()
|
||||
return (_read_text_with_timeout(path) or "").strip()
|
||||
except Exception as e:
|
||||
logger.debug("Could not read %s: %s", path, e)
|
||||
return ""
|
||||
|
||||
@@ -711,6 +711,12 @@ def _provider_request(
|
||||
headers = getattr(request, "headers", None)
|
||||
if isinstance(headers, dict):
|
||||
headers = {k: v for k, v in headers.items() if str(k).lower() not in _RELAY_INTERNAL_PROVIDER_HEADERS}
|
||||
# Relay's managed-call trace header maps to ``extra_headers`` for known SDK adapters and custom
|
||||
# requests that already use that container; other native transports take protocol kwargs directly
|
||||
# and may reject an SDK-only argument. Non-trace middleware headers are preserved as before.
|
||||
supports_extra_headers = _RELAY_PROTOCOL_BY_API_MODE.get(_api_mode(metadata)) is not None or "extra_headers" in original
|
||||
if headers and not supports_extra_headers:
|
||||
headers = {k: v for k, v in headers.items() if str(k).lower() != "traceparent"}
|
||||
if headers:
|
||||
final["extra_headers"] = {**dict(final.get("extra_headers") or {}), **headers}
|
||||
return final
|
||||
|
||||
@@ -269,7 +269,8 @@ class _ProcessRelayPluginConfiguration:
|
||||
except Exception as exc:
|
||||
raise RuntimeError("Hermes Relay dynamic plugin activation failed") from exc
|
||||
if self._activation is None:
|
||||
# Reached only after explicit opt-in; Relay owns any ambient layering.
|
||||
# Reached only after explicit opt-in. Relay 0.8 no longer layers repository-local
|
||||
# configuration onto this explicitly selected payload.
|
||||
_resolve_plugin_awaitable(relay.plugin.initialize(plugin_config))
|
||||
return True
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
def execute(
|
||||
tool_name: str, args: dict[str, Any], callback: Callable[[dict[str, Any]], Any], *,
|
||||
session_id: str, metadata: dict[str, Any] | None = None,
|
||||
session_id: str, tool_call_id: str | None = None, metadata: dict[str, Any] | None = None,
|
||||
) -> tuple[Any, dict[str, Any]]:
|
||||
"""Run one tool call through Relay and return its final arguments."""
|
||||
runtime, session, parent = relay_runtime.resolve_execution_context(session_id)
|
||||
@@ -42,13 +42,13 @@ def execute(
|
||||
callback_error = exc
|
||||
raise
|
||||
raw_result.update(value=result, json=_jsonable(result))
|
||||
return raw_result["json"]
|
||||
return runtime.relay.ToolExecutionResult(raw_result["json"])
|
||||
|
||||
try:
|
||||
managed = _run_awaitable(
|
||||
runtime.run_in_session_async(
|
||||
session, runtime.relay.tools.execute, tool_name, _jsonable(args), invoke,
|
||||
handle=parent, metadata=_jsonable(metadata or {}),
|
||||
handle=parent, metadata=_jsonable(metadata or {}), tool_call_id=tool_call_id or None,
|
||||
)
|
||||
)
|
||||
except BaseException as exc:
|
||||
@@ -61,9 +61,12 @@ def execute(
|
||||
)
|
||||
return raw_result["value"], observed_args
|
||||
raise
|
||||
if "value" in raw_result and _json_equal(managed, raw_result["json"]):
|
||||
managed_result = managed.result
|
||||
if "value" in raw_result and _json_equal(managed_result, raw_result["json"]):
|
||||
return raw_result["value"], observed_args
|
||||
return (managed if isinstance(managed, str) else json.dumps(_jsonable(managed), ensure_ascii=False)), observed_args
|
||||
if isinstance(managed_result, str):
|
||||
return managed_result, observed_args
|
||||
return json.dumps(_jsonable(managed_result), ensure_ascii=False), observed_args
|
||||
|
||||
|
||||
def _jsonable(value: Any) -> Any:
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Shared directory pruning policy for broad recursive scans.
|
||||
|
||||
These names identify version-control internals, dependency trees, generated
|
||||
artifacts, caches, and backup copies that are not useful results for broad
|
||||
agent-facing discovery. Ordinary search callers may still target an explicit
|
||||
path; broad diagnostic probes should apply this policy to recursive walks.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
# Keep this policy conservative and name-based so it works for local and remote
|
||||
# shell backends alike. The same set is used by context discovery and search
|
||||
# probes; adding a directory here protects every broad recursive consumer.
|
||||
SEARCH_PRUNE_DIR_NAMES = frozenset({
|
||||
# Version-control internals.
|
||||
".git", ".hg", ".svn",
|
||||
# Dependency and vendored trees.
|
||||
"node_modules", "venv", ".venv", "site-packages", "dist-packages",
|
||||
"vendor", "third_party",
|
||||
# Generated/build output.
|
||||
"build", "dist", "target", "out", "coverage",
|
||||
".next", ".turbo", ".parcel-cache", ".nuxt", ".svelte-kit",
|
||||
# Python and package-manager caches.
|
||||
"__pycache__", ".cache", ".Trash", ".tox", ".nox", ".mypy_cache",
|
||||
".pytest_cache", ".ruff_cache", ".npm", ".yarn", ".pnpm-store",
|
||||
".gradle", ".m2", ".nuget",
|
||||
# Backup copies.
|
||||
"backups", "backup", ".backups",
|
||||
})
|
||||
+20
-1
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import logging
|
||||
import os
|
||||
import ssl
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
@@ -12,6 +13,24 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_CA_BUNDLE_ENV_VARS = ("HERMES_CA_BUNDLE", "SSL_CERT_FILE", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE")
|
||||
_INSECURE_STRINGS = {"false", "0", "no", "off"}
|
||||
_CA_CONTEXTS: dict[str, ssl.SSLContext] = {}
|
||||
_CA_CONTEXTS_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _context_for_ca_bundle(ca_path: str) -> ssl.SSLContext:
|
||||
"""One ``SSLContext`` per CA bundle path, process-wide.
|
||||
|
||||
``ssl.create_default_context(cafile=...)`` parses the whole bundle each call, and httpx
|
||||
transport sharing keys on context identity — so a per-agent context cost one parsed bundle
|
||||
AND one private connection pool per agent (and per delegated child). An ``SSLContext`` is
|
||||
safe to share across connections.
|
||||
"""
|
||||
with _CA_CONTEXTS_LOCK:
|
||||
ctx = _CA_CONTEXTS.get(ca_path)
|
||||
if ctx is None:
|
||||
ctx = ssl.create_default_context(cafile=ca_path)
|
||||
_CA_CONTEXTS[ca_path] = ctx
|
||||
return ctx
|
||||
|
||||
|
||||
def resolve_httpx_verify(*, ca_bundle: Optional[str] = None, ssl_verify: Any = None, base_url: str = "") -> bool | ssl.SSLContext:
|
||||
@@ -32,6 +51,6 @@ def resolve_httpx_verify(*, ca_bundle: Optional[str] = None, ssl_verify: Any = N
|
||||
if effective_ca:
|
||||
ca_path = str(Path(effective_ca).expanduser())
|
||||
if os.path.isfile(ca_path):
|
||||
return ssl.create_default_context(cafile=ca_path)
|
||||
return _context_for_ca_bundle(ca_path)
|
||||
logger.warning("CA bundle path does not exist: %s — falling back to default certificates", effective_ca)
|
||||
return True
|
||||
|
||||
@@ -65,10 +65,25 @@ class StreamDeliveryMixin:
|
||||
deliver(ctx_scrubber.flush())
|
||||
self._current_streamed_assistant_text = ""
|
||||
|
||||
@property
|
||||
def _current_streamed_assistant_text(self) -> str:
|
||||
"""Visible assistant text streamed so far this turn. Backed by a list of pieces: ``+=`` on a string
|
||||
attribute copies the whole reply on every delta (quadratic). Hot-path emptiness checks look at
|
||||
``_streamed_assistant_text_parts`` so they do not join per token."""
|
||||
parts = getattr(self, "_streamed_assistant_text_parts", None)
|
||||
return "".join(parts) if parts else ""
|
||||
|
||||
@_current_streamed_assistant_text.setter
|
||||
def _current_streamed_assistant_text(self, value: str) -> None:
|
||||
self._streamed_assistant_text_parts = [value] if value else []
|
||||
|
||||
def _record_streamed_assistant_text(self, text: str) -> None:
|
||||
"""Accumulate visible assistant text emitted through stream callbacks (superseded writers excluded)."""
|
||||
if isinstance(text, str) and text and not self._stream_writer_superseded():
|
||||
self._current_streamed_assistant_text = getattr(self, "_current_streamed_assistant_text", "") + text
|
||||
parts = getattr(self, "_streamed_assistant_text_parts", None)
|
||||
if parts is None:
|
||||
parts = self._streamed_assistant_text_parts = []
|
||||
parts.append(text)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_interim_visible_text(text: str) -> str:
|
||||
@@ -263,7 +278,8 @@ class StreamDeliveryMixin:
|
||||
text = think_scrubber.feed(text) if think_scrubber is not None else self._strip_think_blocks(text)
|
||||
text = scrubber.feed(text) if scrubber is not None else sanitize_context(text)
|
||||
# Only strip leading newlines on the first delta — mid-stream "\n" is legitimate markdown.
|
||||
if not prepended_break and not getattr(self, "_current_streamed_assistant_text", ""):
|
||||
# Check the parts list, not the joined property (joining per token copies the whole reply).
|
||||
if not prepended_break and not getattr(self, "_streamed_assistant_text_parts", None):
|
||||
text = text.lstrip("\n")
|
||||
if not text:
|
||||
return
|
||||
|
||||
@@ -14,6 +14,7 @@ import secrets
|
||||
import threading
|
||||
import time
|
||||
import contextlib
|
||||
import weakref
|
||||
from contextlib import contextmanager
|
||||
from concurrent.futures import Future, TimeoutError
|
||||
from typing import Any, Callable, Mapping, Optional
|
||||
@@ -162,8 +163,19 @@ _ACTIVE_PARENT_AGENT: contextvars.ContextVar[Any] = contextvars.ContextVar("herm
|
||||
|
||||
@contextmanager
|
||||
def bind_subagent_parent(parent_agent: Any):
|
||||
"""Bind the host-owned parent for the current agent turn."""
|
||||
token = _ACTIVE_PARENT_AGENT.set(parent_agent)
|
||||
"""Bind the host-owned parent for the current agent turn.
|
||||
|
||||
Stored as a weakref: every asyncio Handle/Future scheduled from the turn
|
||||
(LSP reader loops, kernel pipes, ...) snapshots the Context, and those
|
||||
snapshots outlive the turn. A strong ref there pinned finished delegate
|
||||
children — each of which binds itself here for its own turn — in the
|
||||
parent process heap for the life of the background loop.
|
||||
"""
|
||||
try:
|
||||
ref = weakref.ref(parent_agent)
|
||||
except TypeError:
|
||||
ref = lambda: parent_agent # noqa: E731 — non-weakrefable test doubles
|
||||
token = _ACTIVE_PARENT_AGENT.set(ref)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
@@ -172,7 +184,8 @@ def bind_subagent_parent(parent_agent: Any):
|
||||
|
||||
def get_active_subagent_parent() -> Any:
|
||||
"""Return the parent bound to this execution context, if any."""
|
||||
return _ACTIVE_PARENT_AGENT.get()
|
||||
ref = _ACTIVE_PARENT_AGENT.get()
|
||||
return ref() if ref is not None else None
|
||||
|
||||
|
||||
def _opt_str(value: Any) -> bool:
|
||||
|
||||
@@ -11,7 +11,8 @@ import shlex
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Optional, Set
|
||||
|
||||
from agent.prompt_builder import _scan_context_content
|
||||
from agent.prompt_builder import _read_text_with_timeout, _scan_context_content
|
||||
from agent.search_policy import SEARCH_PRUNE_DIR_NAMES
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -22,13 +23,9 @@ _PATH_ARG_KEYS = {"path", "file_path", "workdir"}
|
||||
_COMMAND_TOOLS = {"terminal"}
|
||||
_MAX_ANCESTOR_WALK = 5 # ancestor levels walked per path — bounds deep-path scans
|
||||
|
||||
# Directories that hold *copies* of context files (backups, vendored deps,
|
||||
# VCS internals, caches), never authoritative project context.
|
||||
_EXCLUDED_DIR_NAMES = frozenset({
|
||||
"node_modules", "venv", ".venv", "__pycache__", ".git", ".hg", ".svn", ".Trash", ".cache", ".tox",
|
||||
".mypy_cache", ".pytest_cache", "site-packages", "dist-packages", "backups", "backup", ".backups",
|
||||
"vendor", "third_party",
|
||||
})
|
||||
# Shared with broad recursive search probes so context discovery and search never drift into
|
||||
# different dependency/cache/build trees (those hold *copies* of context files, never authoritative ones).
|
||||
_EXCLUDED_DIR_NAMES = SEARCH_PRUNE_DIR_NAMES
|
||||
|
||||
|
||||
def _digest(content: str) -> str:
|
||||
@@ -164,7 +161,7 @@ class SubdirectoryHintTracker:
|
||||
except OSError:
|
||||
continue
|
||||
try:
|
||||
content = hint_path.read_text(encoding="utf-8").strip()
|
||||
content = (_read_text_with_timeout(hint_path) or "").strip()
|
||||
if not content:
|
||||
continue
|
||||
digest = _digest(content)
|
||||
|
||||
@@ -701,6 +701,7 @@ def _run_agent_tool_execution_middleware(
|
||||
function_args,
|
||||
_hermes_pipeline,
|
||||
session_id=str(getattr(agent, "session_id", "") or ""),
|
||||
tool_call_id=tool_call_id or None,
|
||||
metadata={
|
||||
"task_id": effective_task_id or "",
|
||||
"turn_id": getattr(agent, "_current_turn_id", "") or "",
|
||||
|
||||
+17
-9
@@ -283,18 +283,26 @@ def _should_run_preflight_estimate(
|
||||
|
||||
def _should_idle_compact(
|
||||
*, enabled: bool, idle_after_seconds: int, idle_gap_seconds: float, tokens: int,
|
||||
floor_tokens: int, cooldown_active: bool,
|
||||
floor_tokens: int, cooldown_active: bool, last_compaction_tokens: int = 0,
|
||||
) -> bool:
|
||||
"""Pure predicate: idle compaction fires after a wall-clock gap of
|
||||
``idle_after_seconds`` (opt-in, <= 0 disables), independent of ``threshold_tokens``;
|
||||
never at/below ``floor_tokens`` or during a compression-failure cooldown."""
|
||||
return bool(
|
||||
enabled
|
||||
and idle_after_seconds > 0
|
||||
and idle_gap_seconds >= idle_after_seconds
|
||||
and not cooldown_active
|
||||
and tokens > floor_tokens
|
||||
)
|
||||
never at/below ``floor_tokens`` or during a compression-failure cooldown.
|
||||
|
||||
``floor_tokens`` (``threshold_tokens × summary_target_ratio``) is a theoretical target a
|
||||
real pass routinely misses (system prompt, tool schemas and protected head/tail are
|
||||
incompressible), so a session compacted to above it would re-summarise on every idle
|
||||
resume without growing. ``last_compaction_tokens`` — what the previous pass actually
|
||||
produced (``ContextCompressor.last_compression_rough_tokens``, same rough shape as
|
||||
``tokens``) — raises the floor to ``last + floor_tokens`` so the transcript must gain a
|
||||
floor's worth of NEW content first. ``0`` (nothing compacted yet / counter reset) keeps
|
||||
the original semantics exactly."""
|
||||
if not enabled or idle_after_seconds <= 0 or idle_gap_seconds < idle_after_seconds or cooldown_active:
|
||||
return False
|
||||
effective_floor = floor_tokens
|
||||
if last_compaction_tokens > 0:
|
||||
effective_floor = max(effective_floor, last_compaction_tokens + floor_tokens)
|
||||
return tokens > effective_floor
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -56,16 +56,26 @@ def _reset_retry_state_after_compaction(agent: Any) -> None:
|
||||
agent._mute_post_response = False
|
||||
|
||||
|
||||
def _blocked_compress_reason(compressor: Any, tokens: int) -> Optional[str]:
|
||||
def _blocked_compress_reason(
|
||||
compressor: Any, tokens: int, attempts_spent: Optional[int] = None
|
||||
) -> Optional[str]:
|
||||
"""Why an over-threshold request is blocked (``None`` below threshold or when the
|
||||
engine lacks ``should_compress_info`` / raises)."""
|
||||
engine lacks ``should_compress_info`` / raises).
|
||||
|
||||
``attempts_spent``: when given and the engine says compression SHOULD run
|
||||
(``(True, None)``) yet the caller skipped it, the per-turn attempt budget is
|
||||
spent — name it ``attempts_exhausted:<n>`` instead of dropping the
|
||||
``(True, None)`` on the floor (silent-lockout case, #101889)."""
|
||||
_info = getattr(compressor, "should_compress_info", None)
|
||||
if not callable(_info):
|
||||
return None
|
||||
try:
|
||||
return _info(tokens)[1]
|
||||
_should_now, _reason = _info(tokens)
|
||||
except Exception:
|
||||
return None
|
||||
if attempts_spent is not None and _should_now and not _reason:
|
||||
return f"attempts_exhausted:{attempts_spent}"
|
||||
return _reason
|
||||
|
||||
|
||||
def _apply_grown_window(agent: Any, compressor: Any, grown: int) -> None:
|
||||
@@ -144,15 +154,22 @@ def _idle_compaction(
|
||||
_idle_cooldown = getattr(
|
||||
_compressor, "get_active_compression_failure_cooldown", lambda: None
|
||||
)()
|
||||
# What the previous pass actually produced — the honest floor versus the theoretical
|
||||
# ``_idle_floor``. Type pin: compressor doubles expose truthy non-ints here; only a real
|
||||
# int may raise the floor, anything else falls back to 0 (original semantics).
|
||||
_idle_last_compaction = getattr(_compressor, "last_compression_rough_tokens", 0)
|
||||
if not isinstance(_idle_last_compaction, int) or isinstance(_idle_last_compaction, bool):
|
||||
_idle_last_compaction = 0
|
||||
if not _tc._should_idle_compact(
|
||||
enabled=agent.compression_enabled, idle_after_seconds=_idle_after,
|
||||
idle_gap_seconds=_idle_gap, tokens=_idle_tokens, floor_tokens=_idle_floor,
|
||||
cooldown_active=bool(_idle_cooldown),
|
||||
cooldown_active=bool(_idle_cooldown), last_compaction_tokens=_idle_last_compaction,
|
||||
):
|
||||
return
|
||||
logger.info(
|
||||
"Idle compaction: %ss idle >= %ss, ~%s tokens > %s floor (session %s)",
|
||||
"Idle compaction: %ss idle >= %ss, ~%s tokens > %s floor (last compaction produced ~%s) (session %s)",
|
||||
int(_idle_gap), _idle_after, f"{_idle_tokens:,}", f"{_idle_floor:,}",
|
||||
f"{_idle_last_compaction:,}" if _idle_last_compaction > 0 else "n/a",
|
||||
agent.session_id or "none",
|
||||
)
|
||||
_idle_status = automatic_compaction_status_message(
|
||||
|
||||
@@ -115,8 +115,14 @@ _PERSISTENCE_CAUSE_EXPLANATIONS: Dict[str, str] = {
|
||||
"have been lost on restart). Freeing disk space will "
|
||||
"not help. Recovery options:\n"
|
||||
"1. Run `hermes doctor --fix`\n"
|
||||
"2. Salvage with: sqlite3 ~/.hermes/state.db \".recover\" "
|
||||
"(then replace state.db)\n"
|
||||
"2. Stop the gateway, then recover with:\n"
|
||||
" hermes sessions recover --source {db_path} --inspect-only\n"
|
||||
" (if it reports recoverable) hermes sessions recover "
|
||||
"--source {db_path} --output recovered-state.db\n"
|
||||
" — recovery snapshots the damaged file first; do NOT "
|
||||
"run `sqlite3 ... \".recover\"` against the live "
|
||||
"state.db, a vulnerable sqlite3 CLI can corrupt it "
|
||||
"further\n"
|
||||
"3. Restore from a backup in ~/.hermes/backups/\n"
|
||||
"Then send your message again."
|
||||
),
|
||||
@@ -287,4 +293,9 @@ class TurnExplainersMixin:
|
||||
body = _PERSISTENCE_CAUSE_EXPLANATIONS.get(
|
||||
persistence_cause or "unknown", _PERSISTENCE_DEFAULT_EXPLANATION
|
||||
)
|
||||
if persistence_cause == "corrupt":
|
||||
# Copy-pasteable, so name the real store (profiles / HERMES_HOME do not live under ~/.hermes).
|
||||
from hermes_state import _default_db_path
|
||||
|
||||
body = body.replace("{db_path}", str(_default_db_path()))
|
||||
return _NO_REPLY + body if body else ""
|
||||
|
||||
+45
-42
@@ -2,8 +2,9 @@
|
||||
|
||||
One process at a time may load -> run -> flush a session shared through state.db (Desktop, CLI
|
||||
resume, gateway, background delivery). ``admit_durable_turn_lease`` acquires the row lease (or
|
||||
returns the early result the façade must hand back); ``DurableTurnLease`` owns the refresher
|
||||
daemon thread, the turn-liveness watchdog wiring, and the lease-loss / stall interrupt plumbing.
|
||||
returns the early result the façade must hand back); ``DurableTurnLease`` owns the periodic
|
||||
refresher, the turn-liveness watchdog wiring, and the lease-loss / stall interrupt plumbing. Both
|
||||
timers run on the shared scheduler thread (``agent/periodic_scheduler.py``), not per-turn threads.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
@@ -20,7 +21,7 @@ LEASE_WAIT_SECONDS = 1800.0
|
||||
|
||||
|
||||
class DurableTurnLease:
|
||||
"""An admitted session turn lease plus the threads that keep it alive and watch the turn.
|
||||
"""An admitted session turn lease plus the periodic timers that keep it alive and watch the turn.
|
||||
|
||||
``stop`` is shared by the refresher and the liveness watchdog; ``turn_active`` gates every
|
||||
interrupt so a late refresher miss can never hard-interrupt the NEXT turn. Both are read and
|
||||
@@ -37,18 +38,15 @@ class DurableTurnLease:
|
||||
self._lock = threading.Lock()
|
||||
self.turn_active = False
|
||||
self.interrupt_message: Optional[str] = None
|
||||
self.refresh_thread: Optional[threading.Thread] = None
|
||||
self.liveness_thread: Optional[threading.Thread] = None
|
||||
self.watchdog = None # TurnLivenessWatchdog when configured
|
||||
self.timer_handles: list = [] # periodic_scheduler handles, cancelled in join_threads
|
||||
|
||||
def _current_session_id(self) -> str:
|
||||
return getattr(self.agent, "session_id", None) or self.session_id
|
||||
|
||||
def build_threads(self) -> None:
|
||||
"""Create (not start) the refresher thread and, when configured, the liveness watchdog:
|
||||
lease renewal is NOT evidence of progress; a silently stalled turn would renew forever."""
|
||||
self.refresh_thread = threading.Thread(
|
||||
target=self.refresh_loop, name="session-turn-lease-refresh", daemon=True
|
||||
)
|
||||
"""Create (not schedule) the liveness watchdog when configured: lease renewal is NOT
|
||||
evidence of progress; a silently stalled turn would renew forever."""
|
||||
try:
|
||||
from hermes_cli.config import load_config_readonly
|
||||
|
||||
@@ -59,13 +57,13 @@ class DurableTurnLease:
|
||||
|
||||
timeout_s, poll_s = turn_liveness.resolve_turn_liveness_settings(liveness_config)
|
||||
if timeout_s is not None:
|
||||
self.liveness_thread = turn_liveness.TurnLivenessWatchdog(
|
||||
self.watchdog = turn_liveness.TurnLivenessWatchdog(
|
||||
self.agent, session_id=self._current_session_id(), timeout_s=timeout_s,
|
||||
poll_s=poll_s, stop_event=self.stop,
|
||||
activity_lock=self.agent._liveness_activity_lock(),
|
||||
is_turn_active=self.is_turn_active, commit_abort=self.commit_liveness_abort,
|
||||
deactivate_turn=self.stop_refresher,
|
||||
).make_thread()
|
||||
)
|
||||
|
||||
def start(self) -> None:
|
||||
with self._lock:
|
||||
@@ -73,9 +71,11 @@ class DurableTurnLease:
|
||||
# Stamp the activity clock at turn entry: `_last_activity_ts` persists across turns, so
|
||||
# without this the watchdog would measure idle from the PREVIOUS turn and abort a fresh one.
|
||||
self.agent._touch_activity("starting new turn")
|
||||
self.refresh_thread.start()
|
||||
if self.liveness_thread is not None:
|
||||
self.liveness_thread.start()
|
||||
from agent.periodic_scheduler import schedule
|
||||
|
||||
self.timer_handles.append(schedule(self.refresh_tick, self.refresh_interval))
|
||||
if self.watchdog is not None:
|
||||
self.timer_handles.append(self.watchdog.schedule())
|
||||
|
||||
def stop_refresher(self) -> None:
|
||||
"""Stop renewal and deactivate the turn. Also the watchdog's deactivate callback: a wedge the
|
||||
@@ -88,9 +88,10 @@ class DurableTurnLease:
|
||||
deactivate_after_liveness_abort = stop_refresher
|
||||
|
||||
def join_threads(self, timeout: float = 1.0) -> None:
|
||||
for thread in (self.refresh_thread, self.liveness_thread):
|
||||
if thread is not None and thread.is_alive():
|
||||
thread.join(timeout=timeout)
|
||||
"""Cancel both timers; ``wait=timeout`` mirrors the old ``thread.join(timeout)`` so an
|
||||
in-flight tick finishes before ``clear_interrupt`` runs."""
|
||||
for handle in self.timer_handles:
|
||||
handle.cancel(wait=timeout)
|
||||
|
||||
def release(self) -> None:
|
||||
"""Release the row and drop the agent's holder attrs (only if they still name this lease)."""
|
||||
@@ -171,34 +172,36 @@ class DurableTurnLease:
|
||||
if agent._execution_thread_id is not None:
|
||||
_set_interrupt(False, agent._execution_thread_id)
|
||||
|
||||
def refresh_loop(self) -> None:
|
||||
"""Renew the lease every ``refresh_interval``; a miss or error interrupts the turn.
|
||||
def refresh_tick(self):
|
||||
"""One periodic renewal (every ``refresh_interval`` on the shared scheduler); a miss or
|
||||
error interrupts the turn. Returning False stops the timer.
|
||||
|
||||
The holder-qualified UPDATE fences a late refresher from a successor lease. The façade's
|
||||
finally sets ``stop`` before releasing, so a holder-fenced miss observed after stop is not
|
||||
a loss."""
|
||||
while not self.stop.wait(self.refresh_interval):
|
||||
try:
|
||||
if self.db.refresh_session_turn_lease(
|
||||
self._current_session_id(), self.holder, ttl_seconds=LEASE_TTL_SECONDS
|
||||
):
|
||||
continue
|
||||
if self.stop.is_set():
|
||||
return
|
||||
logger.error(
|
||||
"Lost session turn lease while turn is active: %s", self._current_session_id()
|
||||
)
|
||||
self._interrupt_turn("Session turn lease lost; stopping to protect the transcript.")
|
||||
except Exception:
|
||||
if self.stop.is_set():
|
||||
return
|
||||
logger.warning(
|
||||
"Failed to refresh session turn lease: %s", self._current_session_id(), exc_info=True,
|
||||
)
|
||||
self._interrupt_turn(
|
||||
"Session turn lease could not be refreshed; stopping to protect the transcript."
|
||||
)
|
||||
return
|
||||
if self.stop.is_set():
|
||||
return False
|
||||
try:
|
||||
if self.db.refresh_session_turn_lease(
|
||||
self._current_session_id(), self.holder, ttl_seconds=LEASE_TTL_SECONDS
|
||||
):
|
||||
return None
|
||||
if self.stop.is_set():
|
||||
return False
|
||||
logger.error(
|
||||
"Lost session turn lease while turn is active: %s", self._current_session_id()
|
||||
)
|
||||
self._interrupt_turn("Session turn lease lost; stopping to protect the transcript.")
|
||||
except Exception:
|
||||
if self.stop.is_set():
|
||||
return False
|
||||
logger.warning(
|
||||
"Failed to refresh session turn lease: %s", self._current_session_id(), exc_info=True,
|
||||
)
|
||||
self._interrupt_turn(
|
||||
"Session turn lease could not be refreshed; stopping to protect the transcript."
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
+27
-22
@@ -86,7 +86,8 @@ def resolve_turn_liveness_settings(
|
||||
|
||||
|
||||
class TurnLivenessWatchdog:
|
||||
"""Sampled-idle watchdog thread bound to one conversation turn.
|
||||
"""Sampled-idle watchdog bound to one conversation turn (polls on the
|
||||
shared periodic scheduler thread).
|
||||
|
||||
``activity_lock`` must be the SAME lock ``AIAgent._touch_activity`` stamps
|
||||
the activity clock with; run_agent owns the lease state and callbacks.
|
||||
@@ -108,28 +109,32 @@ class TurnLivenessWatchdog:
|
||||
self._commit_abort = commit_abort
|
||||
self._deactivate_turn = deactivate_turn
|
||||
|
||||
def make_thread(self) -> threading.Thread:
|
||||
"""Build the (not yet started) watcher thread; started at turn entry, after the
|
||||
turn-active flag and activity clock are stamped."""
|
||||
return threading.Thread(target=self._watch, name="turn-liveness-watchdog", daemon=True)
|
||||
def schedule(self):
|
||||
"""Start polling on the shared periodic scheduler thread; returns the cancel handle.
|
||||
Scheduled at turn entry, after the turn-active flag and activity clock are stamped."""
|
||||
from agent.periodic_scheduler import schedule
|
||||
|
||||
def _watch(self) -> None:
|
||||
while not self._stop_event.wait(self._poll_s):
|
||||
snapshot = self._sample()
|
||||
if snapshot is None:
|
||||
return # turn no longer active
|
||||
if snapshot.idle_seconds < self._timeout_s:
|
||||
continue
|
||||
# Observational only: the commit below can still veto the abort if progress
|
||||
# resumed; the definitive settlement is _surface_committed_abort.
|
||||
self._surface_stall(snapshot)
|
||||
message = f"Turn made no progress for {int(snapshot.idle_seconds)}s; aborting to release the session."
|
||||
if not self._commit_abort(snapshot, message):
|
||||
continue
|
||||
# Stop renewing the lease so a wedge the interrupt cannot unwind expires via TTL.
|
||||
self._deactivate_turn()
|
||||
self._surface_committed_abort(snapshot)
|
||||
return
|
||||
return schedule(self._tick, self._poll_s)
|
||||
|
||||
def _tick(self):
|
||||
"""One poll. Returns False when the watchdog is finished."""
|
||||
if self._stop_event.is_set():
|
||||
return False
|
||||
snapshot = self._sample()
|
||||
if snapshot is None:
|
||||
return False # turn no longer active
|
||||
if snapshot.idle_seconds < self._timeout_s:
|
||||
return None
|
||||
# Observational only: the commit below can still veto the abort if progress
|
||||
# resumed; the definitive settlement is _surface_committed_abort.
|
||||
self._surface_stall(snapshot)
|
||||
message = f"Turn made no progress for {int(snapshot.idle_seconds)}s; aborting to release the session."
|
||||
if not self._commit_abort(snapshot, message):
|
||||
return None
|
||||
# Stop renewing the lease so a wedge the interrupt cannot unwind expires via TTL.
|
||||
self._deactivate_turn()
|
||||
self._surface_committed_abort(snapshot)
|
||||
return False
|
||||
|
||||
def _sample(self) -> Optional[ActivitySnapshot]:
|
||||
with self._activity_lock:
|
||||
|
||||
@@ -315,8 +315,12 @@ def compress_after_tool_results(
|
||||
return _verdict(True)
|
||||
elif agent.compression_enabled:
|
||||
# Over threshold but compression blocked (cooldown/anti-thrash): deduped
|
||||
# warning so context can't silently overflow.
|
||||
_block_reason = _blocked_compress_reason(_compressor, _real_tokens)
|
||||
# warning so context can't silently overflow. ``attempts_spent`` names the
|
||||
# attempts_exhausted lockout when the engine says RUN but the per-turn
|
||||
# budget is spent (#101889).
|
||||
_block_reason = _blocked_compress_reason(
|
||||
_compressor, _real_tokens, attempts_spent=compression_attempts
|
||||
)
|
||||
if _block_reason:
|
||||
agent._warn_context_overflow_blocked(
|
||||
_block_reason, _real_tokens, int(getattr(_compressor, "threshold_tokens", 0) or 0)
|
||||
|
||||
@@ -113,7 +113,7 @@ def assemble_api_request(
|
||||
user merge and surrogate stripping, so the same row's bytes never vary across turns."""
|
||||
from agent.conversation_loop import (
|
||||
_apply_context_engine_selection, _canonicalize_api_tool_calls, _clone_message_for_send,
|
||||
_midturn_request_pressure_tokens, estimate_messages_tokens_rough,
|
||||
_midturn_request_pressure_tokens, _pressure_with_real_floor, estimate_messages_tokens_rough,
|
||||
)
|
||||
|
||||
api_messages, effective_system = build_api_messages(
|
||||
@@ -145,6 +145,13 @@ def assemble_api_request(
|
||||
# Runs unconditionally (not gated on context_compressor) so orphaned tool
|
||||
# results from session loading or manual message edits are always caught.
|
||||
api_messages = agent._sanitize_api_messages(api_messages)
|
||||
# Send-path vision eviction (#89296): compression only strips stale screenshots
|
||||
# when prune fires, and the Anthropic adapter's keep-window never sees
|
||||
# OpenAI-style tool-result image_url parts. The per-call clone is rewritten in
|
||||
# place; persisted history is untouched.
|
||||
from agent.context_compressor import evict_stale_outbound_tool_images
|
||||
|
||||
evict_stale_outbound_tool_images(api_messages)
|
||||
|
||||
# One-time repeated-heal notice goes out via the status/warning callback, NEVER
|
||||
# appended to messages: the cached prompt prefix stays byte-identical.
|
||||
@@ -235,6 +242,13 @@ def assemble_api_request(
|
||||
_anchored_pressure = anchored_context_tokens(messages, getattr(agent, "_usage_anchor", None))
|
||||
if _anchored_pressure is not None:
|
||||
request_pressure_tokens = _anchored_pressure
|
||||
else:
|
||||
# Rough fallback only: floor at the provider's last REAL prompt size (an anchored
|
||||
# figure is provider-exact and is never floored — on MoA turns that would re-add
|
||||
# the fan-out tokens the anchor excludes).
|
||||
request_pressure_tokens = _pressure_with_real_floor(
|
||||
agent.context_compressor, request_pressure_tokens
|
||||
)
|
||||
# Stash the rough estimate so update_from_response() can pair it with the real
|
||||
# count (should_defer_preflight_to_real_usage). getattr: test doubles lack it.
|
||||
_note_rough = getattr(agent.context_compressor, "note_request_rough_estimate", None)
|
||||
|
||||
@@ -422,6 +422,10 @@ def get_pricing_entry(
|
||||
return _INCLUDED_ENTRY
|
||||
if route.provider == "openrouter":
|
||||
return _openrouter_pricing_entry(route)
|
||||
|
||||
bundled_entry = _lookup_official_docs_pricing(route)
|
||||
if bundled_entry:
|
||||
return bundled_entry
|
||||
if route.base_url:
|
||||
entry = _pricing_entry_from_metadata(
|
||||
fetch_endpoint_model_metadata(route.base_url, api_key=api_key or ""), route.model,
|
||||
@@ -430,7 +434,7 @@ def get_pricing_entry(
|
||||
)
|
||||
if entry:
|
||||
return entry
|
||||
return _lookup_official_docs_pricing(route)
|
||||
return None
|
||||
|
||||
|
||||
# Usage-field candidate paths per API shape: (input/prompt total, output, cache
|
||||
|
||||
@@ -36,13 +36,26 @@ export function execText(command: string, args: string[], { timeout = 3000 } = {
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe budget for the ORPHAN-REAP path (matchesParent / matchesIdentity /
|
||||
* stopOwnedBackend). The claim path keeps the full 30s headroom — a freshly
|
||||
* spawned backend's marker is load-bearing and a slow probe must not kill a
|
||||
* healthy child (#93608). Reap only needs to tell "same process" from "gone
|
||||
* or reused" for OLD records, and the ownership file can accumulate dozens of
|
||||
* them (one per profile per launch), so a 30s budget per record would let a
|
||||
* cold PowerShell 5.1 stall boot for minutes (#87169). 5s is plenty for a
|
||||
* warm probe; a timeout degrades to "unknown" and the record is preserved for
|
||||
* the next launch instead of blocking boot.
|
||||
*/
|
||||
export const REAP_PROBE_TIMEOUT_MS = 5_000
|
||||
|
||||
/**
|
||||
* Cross-platform process start marker: a value that changes when a PID is
|
||||
* reused, so `pid + marker` identifies one specific process incarnation.
|
||||
* Throws when the probe fails — callers decide what a failure means (see
|
||||
* `claimDecision` / `probeStartMarker`).
|
||||
*/
|
||||
export async function processStartMarker(pid: number): Promise<string> {
|
||||
export async function processStartMarker(pid: number, timeoutMs: number = 30_000): Promise<string> {
|
||||
// Cheap native dead-PID gate. Windows Get-Process / macOS `ps -p` exit 1
|
||||
// on a missing PID (not ESRCH), so the identity matchers used to keep the
|
||||
// orphan and re-probe it every launch (#92875). ESRCH is the code those
|
||||
@@ -85,7 +98,9 @@ export async function processStartMarker(pid: number): Promise<string> {
|
||||
],
|
||||
// PowerShell 5.1 cold starts routinely exceed the default 3s execText
|
||||
// budget (2.4-8s observed in #87169); give the marker probe headroom.
|
||||
{ timeout: 30_000 }
|
||||
// The claim path keeps this 30s budget; the orphan-reap path passes
|
||||
// REAP_PROBE_TIMEOUT_MS so a slow probe cannot stall boot.
|
||||
{ timeout: timeoutMs }
|
||||
)
|
||||
|
||||
if (!/^\d+$/.test(ticks)) {
|
||||
|
||||
@@ -178,6 +178,51 @@ test('startup reap preserves failed stops for the next launch', async () => {
|
||||
assert.deepEqual(parseBackendOwnership(store.value()), [entry])
|
||||
})
|
||||
|
||||
test('startup reap stops at the deadline and preserves the unprocessed records', async () => {
|
||||
const first = ownershipEntry({ pid: 60 })
|
||||
const second = ownershipEntry({ pid: 61 })
|
||||
const store = memoryStore(stored([first, second]))
|
||||
const stop = vi.fn()
|
||||
|
||||
const ownership = createOwnership(store, {
|
||||
// Each probe is slow enough to blow a 1ms budget after the first entry.
|
||||
matchesIdentity: async () => {
|
||||
await new Promise(resolve => setTimeout(resolve, 10))
|
||||
|
||||
return false
|
||||
},
|
||||
stop,
|
||||
reapDeadlineMs: 1
|
||||
})
|
||||
|
||||
assert.deepEqual(await ownership.reapOrphans(), [])
|
||||
// The first entry was processed (dropped); the second was preserved for the
|
||||
// next launch instead of stalling boot on a slow identity probe.
|
||||
assert.deepEqual(parseBackendOwnership(store.value()), [second])
|
||||
})
|
||||
|
||||
test('startup reap preserves would-be-reaped records when the budget runs out', async () => {
|
||||
const first = ownershipEntry({ pid: 62 })
|
||||
const second = ownershipEntry({ pid: 63 })
|
||||
const store = memoryStore(stored([first, second]))
|
||||
const stop = vi.fn()
|
||||
|
||||
const ownership = createOwnership(store, {
|
||||
matchesIdentity: async () => {
|
||||
await new Promise(resolve => setTimeout(resolve, 10))
|
||||
|
||||
return true
|
||||
},
|
||||
stop,
|
||||
reapDeadlineMs: 1
|
||||
})
|
||||
|
||||
assert.deepEqual(await ownership.reapOrphans(), [62])
|
||||
// The second would have been reaped too, but the budget ran out — it is
|
||||
// preserved so a later launch retries it.
|
||||
assert.deepEqual(parseBackendOwnership(store.value()), [second])
|
||||
})
|
||||
|
||||
test('startup reap never stops a backend whose parent Electron is still alive', async () => {
|
||||
const entry = { ...ownershipEntry({ pid: 54 }), parentPid: 100, parentStartMarker: 'os-start-parent' }
|
||||
const store = memoryStore(stored([entry]))
|
||||
|
||||
@@ -28,8 +28,22 @@ export interface BackendOwnershipDeps {
|
||||
matchesParent: (entry: BackendOwnershipEntry) => Promise<boolean | undefined>
|
||||
stop: (identity: BackendIdentity) => Promise<void> | void
|
||||
store: BackendOwnershipStore
|
||||
/**
|
||||
* Overall time budget for one reap sweep. The ownership file legitimately
|
||||
* accumulates one record per profile per launch, and each record can cost
|
||||
* up to two identity probes (parent + backend) plus a stop — on Windows
|
||||
* those shell out to PowerShell, whose 5.1 cold starts are slow (#87169).
|
||||
* Without a bound, a large roster could stall boot for minutes while the
|
||||
* renderer's 45s backend-boot budget expires and the user stares at the
|
||||
* connecting screen. When the budget is exhausted the sweep preserves the
|
||||
* unprocessed records for the next launch and returns what it reaped.
|
||||
*/
|
||||
reapDeadlineMs?: number
|
||||
}
|
||||
|
||||
/** Default budget for one reap sweep (see `reapDeadlineMs`). */
|
||||
export const REAP_ORPHANS_DEADLINE_MS = 5_000
|
||||
|
||||
export interface BackendClaim extends BackendIdentity {
|
||||
command?: string
|
||||
parentPid?: number
|
||||
@@ -222,8 +236,21 @@ export function createBackendOwnership(deps: BackendOwnershipDeps) {
|
||||
|
||||
const survivors: BackendOwnershipEntry[] = []
|
||||
const reaped: number[] = []
|
||||
const deadline = Date.now() + (deps.reapDeadlineMs ?? REAP_ORPHANS_DEADLINE_MS)
|
||||
|
||||
for (let i = 0; i < entries.length; i += 1) {
|
||||
// Budget exhausted: preserve the unprocessed records so a later launch
|
||||
// can retry them. A slow identity probe must never stall boot — the
|
||||
// renderer's backend-boot budget is 45s and the spawn itself needs
|
||||
// most of it.
|
||||
if (Date.now() >= deadline) {
|
||||
survivors.push(...entries.slice(i))
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
const entry = entries[i]
|
||||
|
||||
for (const entry of entries) {
|
||||
// A backend whose Electron parent is still running is NOT an orphan:
|
||||
// reaping it would kill a live instance's session. This is what stops
|
||||
// a second launch from SIGTERMing the running instance's backend even
|
||||
|
||||
+253
-46
@@ -42,7 +42,8 @@ import {
|
||||
isPidOnlyStartMarker,
|
||||
pidOnlyStartMarker,
|
||||
probeStartMarker,
|
||||
processStartMarker
|
||||
processStartMarker,
|
||||
REAP_PROBE_TIMEOUT_MS
|
||||
} from './backend-claim'
|
||||
import { dashboardFallbackArgs, sourceDeclaresServe } from './backend-command'
|
||||
import { createBackendConnectionState } from './backend-connection-state'
|
||||
@@ -280,6 +281,12 @@ import {
|
||||
undialedSshRouteSeeds
|
||||
} from './plugin-profile-routes'
|
||||
import { selectPoolEvictions } from './pool-eviction'
|
||||
import { clampPoolLimits, parsePoolLimits, POOL_LIMITS_DEFAULTS } from './pool-limits'
|
||||
import {
|
||||
LocalBackendSpawnCoordinator,
|
||||
type LocalBackendSpawnRequest,
|
||||
releaseLocalBackendSlotAfterExit
|
||||
} from './pool-spawn-coordinator'
|
||||
import { createPoolStopper } from './pool-stop'
|
||||
import { poolTouchKeys } from './pool-touch-scope'
|
||||
import { createKeepAwake } from './power-save'
|
||||
@@ -1408,8 +1415,97 @@ const profileDeletionGate = new ProfileDeletionGate()
|
||||
// Keep the pool light: cap concurrent profile backends (LRU eviction) and reap
|
||||
// idle ones. A user idles at exactly the primary backend; pool backends only
|
||||
// exist while a non-primary profile is actively being chatted through.
|
||||
const POOL_MAX_BACKENDS = Math.max(1, Number(process.env.HERMES_DESKTOP_POOL_MAX) || 3)
|
||||
const POOL_IDLE_MS = Math.max(60_000, Number(process.env.HERMES_DESKTOP_POOL_IDLE_MS) || 10 * 60_000)
|
||||
// Pool sizing is a device preference (Settings → Advanced → pool rows), not a
|
||||
// launch constant: mutable at runtime, persisted in userData, applied live.
|
||||
// The legacy HERMES_DESKTOP_POOL_* env vars remain the initial-value fallback
|
||||
// for scripted/headless setups; after launch the stored preference wins.
|
||||
const POOL_LIMITS_PATH = path.join(app.getPath('userData'), 'pool-limits.json')
|
||||
|
||||
function readPersistedPoolLimits() {
|
||||
try {
|
||||
const limits = parsePoolLimits(fs.readFileSync(POOL_LIMITS_PATH, 'utf8'))
|
||||
rememberLog(
|
||||
`[pool-limits] loaded from ${POOL_LIMITS_PATH}: maxBackends=${limits.maxBackends}, idleMs=${limits.idleMs}`
|
||||
)
|
||||
|
||||
return limits
|
||||
} catch {
|
||||
// No persisted file yet — fall back to the legacy env vars so scripted
|
||||
// setups keep working. Log which source won: a silently-ignored env var
|
||||
// here costs a scripted-setup user a debugging session.
|
||||
const fromEnv = clampPoolLimits({
|
||||
maxBackends: Number(process.env.HERMES_DESKTOP_POOL_MAX) || undefined,
|
||||
idleMs: Number(process.env.HERMES_DESKTOP_POOL_IDLE_MS) || undefined
|
||||
})
|
||||
|
||||
if (fromEnv.maxBackends !== POOL_LIMITS_DEFAULTS.maxBackends || fromEnv.idleMs !== POOL_LIMITS_DEFAULTS.idleMs) {
|
||||
rememberLog(
|
||||
`[pool-limits] no saved file; using env-var overrides: maxBackends=${fromEnv.maxBackends}, idleMs=${fromEnv.idleMs}`
|
||||
)
|
||||
} else {
|
||||
rememberLog('[pool-limits] no saved file and no env overrides; using defaults')
|
||||
}
|
||||
|
||||
return fromEnv
|
||||
}
|
||||
}
|
||||
|
||||
function persistPoolLimits(limits) {
|
||||
try {
|
||||
fs.mkdirSync(path.dirname(POOL_LIMITS_PATH), { recursive: true })
|
||||
// Atomic write: write to a temp file in the same directory, then rename.
|
||||
// A crash mid-write would otherwise leave truncated JSON and silently
|
||||
// lose the user's saved sizing.
|
||||
const tmpPath = `${POOL_LIMITS_PATH}.tmp`
|
||||
fs.writeFileSync(tmpPath, JSON.stringify(limits, null, 2), 'utf8')
|
||||
fs.renameSync(tmpPath, POOL_LIMITS_PATH)
|
||||
} catch (error) {
|
||||
rememberLog(`[pool-limits] write failed: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
// rememberLog() state. Declared here, before the top-level
|
||||
// readPersistedPoolLimits() call below, because that call logs during module
|
||||
// evaluation; declaring these later crashed launch with `undefined.push` in
|
||||
// the packaged build (esbuild lowers the TDZ to undefined instead of throwing).
|
||||
const hermesLog = []
|
||||
let desktopLogBuffer = ''
|
||||
let desktopLogFlushTimer = null
|
||||
let desktopLogFlushPromise = Promise.resolve()
|
||||
|
||||
let poolLimits = readPersistedPoolLimits()
|
||||
// Hard cap on local backends that are starting OR running (the LRU eviction
|
||||
// above is soft — it spares keepalive-fresh entries). Follows the live
|
||||
// preference: setPoolLimits() pushes a new max into the coordinator.
|
||||
const localBackendSpawnCoordinator = new LocalBackendSpawnCoordinator(poolLimits.maxBackends)
|
||||
// How long a spawn may wait for a free local slot. Must stay under the
|
||||
// renderer's BACKEND_BOOT_WAIT_TIMEOUT_MS (45s, src/lib/with-timeout.ts) so
|
||||
// the queued ticket fails before the renderer does and the user sees why.
|
||||
const POOL_SLOT_WAIT_MS = 30_000
|
||||
|
||||
function poolMaxBackends() {
|
||||
return poolLimits.maxBackends
|
||||
}
|
||||
|
||||
function poolIdleMs() {
|
||||
return poolLimits.idleMs
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply new limits live: persist, then converge the running pool — evict
|
||||
* LRU backends down to the new max, and let the (already running) idle
|
||||
* reaper handle a shortened idle window on its next tick. Returns the
|
||||
* limits actually in force (post-clamp).
|
||||
*/
|
||||
function setPoolLimits(raw) {
|
||||
poolLimits = clampPoolLimits(raw)
|
||||
persistPoolLimits(poolLimits)
|
||||
localBackendSpawnCoordinator.setLimit(poolLimits.maxBackends)
|
||||
evictLruPoolBackends(poolMaxBackends())
|
||||
startPoolIdleReaper()
|
||||
|
||||
return { ...poolLimits }
|
||||
}
|
||||
|
||||
// A backend touched within this window has a live renderer socket (the keepalive
|
||||
// pings every 60s for every open profile). LRU eviction must spare these — a
|
||||
@@ -1429,7 +1525,7 @@ const POOL_IDLE_MS = Math.max(60_000, Number(process.env.HERMES_DESKTOP_POOL_IDL
|
||||
// re-allocating pooled gateway secondaries ~700×/day).
|
||||
// * 3× ping + 60s headroom = ~4 min, comfortable margin for two missed
|
||||
// pings + WSL2 IPC stall. The hard ceiling for the cap-eligible set is
|
||||
// POOL_IDLE_MS above (default 10 min) — this constant only governs the
|
||||
// pool idle window above (default 10 min) — this constant only governs the
|
||||
// "is this backend plausibly still alive" question for LRU eviction,
|
||||
// not when the idle reaper definitively tears a backend down.
|
||||
const POOL_KEEPALIVE_FRESH_MS = Math.max(
|
||||
@@ -1487,12 +1583,8 @@ let connectionRegistryCache = null
|
||||
let connectionRegistryCacheMtime = null
|
||||
let remoteHeaderRulesInstalled = false
|
||||
const remoteWsHeaderStore = createRemoteWsHeaderStore()
|
||||
const hermesLog = []
|
||||
const previewWatchers = new Map()
|
||||
let previewShortcutActive = false
|
||||
let desktopLogBuffer = ''
|
||||
let desktopLogFlushTimer = null
|
||||
let desktopLogFlushPromise = Promise.resolve()
|
||||
let nativeThemeListenerInstalled = false
|
||||
|
||||
let bootProgressState = {
|
||||
@@ -3401,7 +3493,7 @@ async function backendCommandForPid(pid) {
|
||||
}
|
||||
}
|
||||
|
||||
async function processIdentityMatches(identity) {
|
||||
async function processIdentityMatches(identity, timeoutMs: number = 30_000) {
|
||||
// Degraded PID-only identity (#93608): the start-marker probe failed while
|
||||
// the child was verifiably alive, so only PID liveness can be checked here.
|
||||
// backendIdentityMatches layers the command-line check on top before
|
||||
@@ -3419,14 +3511,14 @@ async function processIdentityMatches(identity) {
|
||||
}
|
||||
|
||||
try {
|
||||
return (await processStartMarker(identity.pid)) === identity.startMarker
|
||||
return (await processStartMarker(identity.pid, timeoutMs)) === identity.startMarker
|
||||
} catch (error) {
|
||||
return error?.code === 'ENOENT' || error?.code === 'ESRCH' ? false : undefined
|
||||
}
|
||||
}
|
||||
|
||||
async function backendIdentityMatches(identity) {
|
||||
const processMatches = await processIdentityMatches(identity)
|
||||
const processMatches = await processIdentityMatches(identity, REAP_PROBE_TIMEOUT_MS)
|
||||
|
||||
if (processMatches !== true) {
|
||||
return processMatches
|
||||
@@ -3447,17 +3539,26 @@ async function backendParentMatches(entry) {
|
||||
}
|
||||
|
||||
try {
|
||||
return (await processStartMarker(entry.parentPid)) === entry.parentStartMarker
|
||||
return (await processStartMarker(entry.parentPid, REAP_PROBE_TIMEOUT_MS)) === entry.parentStartMarker
|
||||
} catch (error) {
|
||||
return error?.code === 'ENOENT' || error?.code === 'ESRCH' ? false : undefined
|
||||
}
|
||||
}
|
||||
|
||||
async function stopOwnedBackend(identity) {
|
||||
if ((await processIdentityMatches(identity)) !== true) {
|
||||
const matches = await processIdentityMatches(identity, REAP_PROBE_TIMEOUT_MS)
|
||||
|
||||
if (matches === false) {
|
||||
return
|
||||
}
|
||||
|
||||
if (matches !== true) {
|
||||
// Identity probe failed (not confirmed gone): preserve the record so a
|
||||
// later launch retries the stop instead of dropping it and leaking the
|
||||
// backend. reapOrphans keeps the entry when stop() throws.
|
||||
throw new Error(`Could not verify backend PID ${identity.pid} before stopping it.`)
|
||||
}
|
||||
|
||||
if (IS_WINDOWS) {
|
||||
forceKillProcessTree(identity.pid)
|
||||
} else {
|
||||
@@ -3474,7 +3575,7 @@ async function stopOwnedBackend(identity) {
|
||||
const deadline = Date.now() + 1500
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
if ((await processIdentityMatches(identity)) !== true) {
|
||||
if ((await processIdentityMatches(identity, REAP_PROBE_TIMEOUT_MS)) !== true) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -3483,7 +3584,7 @@ async function stopOwnedBackend(identity) {
|
||||
|
||||
// Revalidate immediately before escalation so PID reuse cannot target a
|
||||
// replacement process.
|
||||
if ((await processIdentityMatches(identity)) === true) {
|
||||
if ((await processIdentityMatches(identity, REAP_PROBE_TIMEOUT_MS)) === true) {
|
||||
try {
|
||||
process.kill(-identity.pid, 'SIGKILL')
|
||||
} catch {
|
||||
@@ -3493,7 +3594,7 @@ async function stopOwnedBackend(identity) {
|
||||
}
|
||||
|
||||
await new Promise(resolve => setTimeout(resolve, 50))
|
||||
const remaining = await processIdentityMatches(identity)
|
||||
const remaining = await processIdentityMatches(identity, REAP_PROBE_TIMEOUT_MS)
|
||||
|
||||
if (remaining !== false) {
|
||||
throw new Error(`Backend PID ${identity.pid} did not stop cleanly.`)
|
||||
@@ -11251,7 +11352,7 @@ async function ensureBackend(profile) {
|
||||
return connection
|
||||
}
|
||||
|
||||
evictLruPoolBackends(POOL_MAX_BACKENDS - 1)
|
||||
evictLruPoolBackends(poolMaxBackends() - 1)
|
||||
|
||||
const entry = {
|
||||
process: null,
|
||||
@@ -11259,7 +11360,10 @@ async function ensureBackend(profile) {
|
||||
token: null,
|
||||
connectionPromise: null,
|
||||
lastActiveAt: Date.now(),
|
||||
remoteBaseUrl: null
|
||||
remoteBaseUrl: null,
|
||||
releaseLocalBackendSlot: null,
|
||||
localBackendSlotKey: null,
|
||||
localBackendSpawnRequest: null
|
||||
}
|
||||
|
||||
entry.connectionPromise = spawnPoolBackend(key, entry).catch(async error => {
|
||||
@@ -11270,12 +11374,7 @@ async function ensureBackend(profile) {
|
||||
`Hermes backend for profile "${key}" failed to start: ${error instanceof Error ? error.message : String(error)}`
|
||||
)
|
||||
|
||||
if (backendPool.get(key) === entry) {
|
||||
backendPool.delete(key)
|
||||
}
|
||||
|
||||
stopBackendChild(entry.process)
|
||||
await waitForBackendExit(entry.process)
|
||||
await teardownFailedLocalBackend(key, entry)
|
||||
throw error
|
||||
})
|
||||
backendPool.set(key, entry)
|
||||
@@ -11419,7 +11518,7 @@ async function ensureRegistryBackend(connectionId, profile, managedUpdateCorrela
|
||||
return existingLocal.connectionPromise
|
||||
}
|
||||
|
||||
evictLruPoolBackends(POOL_MAX_BACKENDS - 1)
|
||||
evictLruPoolBackends(poolMaxBackends() - 1)
|
||||
|
||||
const localEntry = {
|
||||
process: null,
|
||||
@@ -11427,7 +11526,10 @@ async function ensureRegistryBackend(connectionId, profile, managedUpdateCorrela
|
||||
token: null,
|
||||
connectionPromise: null,
|
||||
lastActiveAt: Date.now(),
|
||||
remoteBaseUrl: null
|
||||
remoteBaseUrl: null,
|
||||
releaseLocalBackendSlot: null,
|
||||
localBackendSlotKey: null,
|
||||
localBackendSpawnRequest: null
|
||||
}
|
||||
|
||||
localEntry.connectionPromise = spawnPoolBackend(profileKey, localEntry, {
|
||||
@@ -11440,12 +11542,7 @@ async function ensureRegistryBackend(connectionId, profile, managedUpdateCorrela
|
||||
`Hermes backend for profile "${profileKey}" (forced-local) failed to start: ${error instanceof Error ? error.message : String(error)}`
|
||||
)
|
||||
|
||||
if (backendPool.get(localRoute.poolKey) === localEntry) {
|
||||
backendPool.delete(localRoute.poolKey)
|
||||
}
|
||||
|
||||
stopBackendChild(localEntry.process)
|
||||
await waitForBackendExit(localEntry.process)
|
||||
await teardownFailedLocalBackend(localRoute.poolKey, localEntry)
|
||||
throw error
|
||||
})
|
||||
backendPool.set(localRoute.poolKey, localEntry)
|
||||
@@ -11492,7 +11589,7 @@ async function ensureRegistryBackend(connectionId, profile, managedUpdateCorrela
|
||||
)
|
||||
}
|
||||
|
||||
evictLruPoolBackends(POOL_MAX_BACKENDS - 1)
|
||||
evictLruPoolBackends(poolMaxBackends() - 1)
|
||||
|
||||
const entry = {
|
||||
process: null,
|
||||
@@ -12147,7 +12244,7 @@ function evictLruPoolBackends(keep) {
|
||||
const evictions = selectPoolEvictions(backendPool.entries(), Math.max(0, keep), Date.now(), POOL_KEEPALIVE_FRESH_MS)
|
||||
|
||||
for (const profile of evictions) {
|
||||
rememberLog(`Evicting idle profile backend "${profile}" (LRU cap ${POOL_MAX_BACKENDS})`)
|
||||
rememberLog(`Evicting idle profile backend "${profile}" (LRU cap ${poolMaxBackends()})`)
|
||||
stopPoolBackend(profile)
|
||||
}
|
||||
}
|
||||
@@ -12161,8 +12258,8 @@ function startPoolIdleReaper() {
|
||||
const now = Date.now()
|
||||
|
||||
for (const [profile, entry] of [...backendPool.entries()]) {
|
||||
if (now - (entry.lastActiveAt || 0) > POOL_IDLE_MS) {
|
||||
rememberLog(`Reaping idle profile backend "${profile}" (idle > ${Math.round(POOL_IDLE_MS / 1000)}s)`)
|
||||
if (now - (entry.lastActiveAt || 0) > poolIdleMs()) {
|
||||
rememberLog(`Reaping idle profile backend "${profile}" (idle > ${Math.round(poolIdleMs() / 1000)}s)`)
|
||||
stopPoolBackend(profile)
|
||||
}
|
||||
}
|
||||
@@ -12178,6 +12275,67 @@ function startPoolIdleReaper() {
|
||||
}
|
||||
}
|
||||
|
||||
function releaseLocalBackendSlot(entry: any) {
|
||||
if (!entry) {
|
||||
return
|
||||
}
|
||||
|
||||
const release = entry.releaseLocalBackendSlot
|
||||
const request = entry.localBackendSpawnRequest as LocalBackendSpawnRequest | null
|
||||
entry.releaseLocalBackendSlot = null
|
||||
entry.localBackendSlotKey = null
|
||||
entry.localBackendSpawnRequest = null
|
||||
|
||||
if (release) {
|
||||
release()
|
||||
} else {
|
||||
request?.cancel()
|
||||
}
|
||||
}
|
||||
|
||||
function assertPoolEntryStillOwned(poolKey: string, entry: any) {
|
||||
if (backendPool.get(poolKey) !== entry) {
|
||||
releaseLocalBackendSlot(entry)
|
||||
throw new Error(`Profile backend start for "${poolKey}" was cancelled before spawn.`)
|
||||
}
|
||||
}
|
||||
|
||||
const failedLocalBackendTeardowns = new WeakMap<object, Promise<void>>()
|
||||
|
||||
function teardownFailedLocalBackend(poolKey: string, entry: any): Promise<void> {
|
||||
const existing = failedLocalBackendTeardowns.get(entry)
|
||||
|
||||
if (existing) {
|
||||
return existing
|
||||
}
|
||||
|
||||
if (backendPool.get(poolKey) === entry) {
|
||||
backendPool.delete(poolKey)
|
||||
}
|
||||
|
||||
const child = entry.process
|
||||
|
||||
const teardown = releaseLocalBackendSlotAfterExit(
|
||||
() => releaseLocalBackendSlot(entry),
|
||||
async () => {
|
||||
stopBackendChild(child)
|
||||
await waitForBackendExit(child)
|
||||
|
||||
if (child && child.exitCode === null && child.signalCode === null) {
|
||||
throw new Error(`Profile backend for "${poolKey}" did not exit; keeping the local slot occupied.`)
|
||||
}
|
||||
|
||||
releaseBackendChild(child)
|
||||
}
|
||||
)
|
||||
|
||||
// Keep the settled promise in the WeakMap for the lifetime of this entry.
|
||||
// Error + exit + outer catch may all request cleanup; none may run it twice.
|
||||
failedLocalBackendTeardowns.set(entry, teardown)
|
||||
|
||||
return teardown
|
||||
}
|
||||
|
||||
// Spawn an additional dashboard backend pinned to a named profile. Mirrors the
|
||||
// local-spawn portion of startHermes() but without the boot-progress UI,
|
||||
// bootstrap, or remote handling (those belong to the primary backend only).
|
||||
@@ -12215,6 +12373,29 @@ async function spawnPoolBackend(profile, entry, opts: { forceLocal?: boolean; po
|
||||
}
|
||||
}
|
||||
|
||||
// Bound the slot wait BELOW the renderer's backend-boot budget (45s): once
|
||||
// the renderer has given up on this spawn, a ticket still queued for the
|
||||
// pool-idle window (10 min) would hold the pool key hostage and every
|
||||
// later click on the profile would join that stale wait. Failing here
|
||||
// surfaces the "all N slots busy" reason instead of a generic boot timeout.
|
||||
const spawnRequest = localBackendSpawnCoordinator.request(poolKey, { timeoutMs: POOL_SLOT_WAIT_MS })
|
||||
entry.localBackendSlotKey = poolKey
|
||||
entry.localBackendSpawnRequest = spawnRequest
|
||||
|
||||
if (localBackendSpawnCoordinator.activeCount >= poolMaxBackends()) {
|
||||
rememberLog(
|
||||
`Profile backend "${profile}" waiting for a free local slot (${localBackendSpawnCoordinator.activeCount}/${poolMaxBackends()} busy, ${localBackendSpawnCoordinator.queuedCount} queued)`
|
||||
)
|
||||
}
|
||||
|
||||
entry.releaseLocalBackendSlot = await spawnRequest.acquired
|
||||
|
||||
if (entry.localBackendSpawnRequest === spawnRequest) {
|
||||
entry.localBackendSpawnRequest = null
|
||||
}
|
||||
|
||||
assertPoolEntryStillOwned(poolKey, entry)
|
||||
|
||||
const token = crypto.randomBytes(32).toString('base64url')
|
||||
|
||||
// Same update mutual exclusion as the primary window's waitForLocalStart
|
||||
@@ -12258,12 +12439,12 @@ async function spawnPoolBackend(profile, entry, opts: { forceLocal?: boolean; po
|
||||
assertLocalProfileCanStart(profile, profileDeletionGate, key =>
|
||||
directoryExists(path.join(HERMES_HOME, 'profiles', key))
|
||||
)
|
||||
|
||||
rememberLog(`Starting Hermes backend for profile "${profile}" via ${backend.label}`)
|
||||
|
||||
const parentStartMarker = await desktopParentStartMarker()
|
||||
const backendNonce = crypto.randomBytes(16).toString('hex')
|
||||
const parentIdentityEnv = parentWatchdogEnv(process.pid, parentStartMarker, backendNonce)
|
||||
assertPoolEntryStillOwned(poolKey, entry)
|
||||
|
||||
const child = spawn(
|
||||
backend.command,
|
||||
@@ -12318,6 +12499,7 @@ async function spawnPoolBackend(profile, entry, opts: { forceLocal?: boolean; po
|
||||
// surface as an unhandled rejection before the Promise.race below attaches.
|
||||
portAnnouncement.catch(() => {})
|
||||
await claimBackendChild(child, `${backend.command} ${backend.args.join(' ')}`, profile, backendNonce, outputTail)
|
||||
assertPoolEntryStillOwned(poolKey, entry)
|
||||
|
||||
child.stdout.on('data', rememberLog)
|
||||
child.stderr.on('data', rememberLog)
|
||||
@@ -12331,14 +12513,21 @@ async function spawnPoolBackend(profile, entry, opts: { forceLocal?: boolean; po
|
||||
|
||||
child.once('error', error => {
|
||||
rememberLog(`Hermes backend for profile "${profile}" failed to start: ${error.message}`)
|
||||
releaseBackendChild(child)
|
||||
backendPool.delete(poolKey)
|
||||
void teardownFailedLocalBackend(poolKey, entry).catch(cleanupError => {
|
||||
rememberLog(
|
||||
`Hermes backend for profile "${profile}" cleanup failed: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`
|
||||
)
|
||||
})
|
||||
rejectStart?.(error)
|
||||
})
|
||||
child.once('exit', (code, signal) => {
|
||||
rememberLog(`Hermes backend for profile "${profile}" exited (${signal || code})`)
|
||||
releaseLocalBackendSlot(entry)
|
||||
releaseBackendChild(child)
|
||||
backendPool.delete(poolKey)
|
||||
|
||||
if (backendPool.get(poolKey) === entry) {
|
||||
backendPool.delete(poolKey)
|
||||
}
|
||||
|
||||
if (!ready) {
|
||||
rejectStart?.(
|
||||
@@ -12406,16 +12595,20 @@ const poolStopper = createPoolStopper({
|
||||
waitForExit: child => waitForBackendExit(child)
|
||||
})
|
||||
|
||||
function stopPoolBackend(profile) {
|
||||
return poolStopper.stop(profile)
|
||||
async function stopPoolBackend(profile: string) {
|
||||
const entry = backendPool.get(profile)
|
||||
await poolStopper.stop(profile)
|
||||
releaseLocalBackendSlot(entry)
|
||||
}
|
||||
|
||||
async function teardownPoolBackendAndWait(profile) {
|
||||
await Promise.all(localProfilePoolKeys(profile).map(key => poolStopper.stop(key)))
|
||||
await Promise.all(localProfilePoolKeys(profile).map(key => stopPoolBackend(key)))
|
||||
}
|
||||
|
||||
function stopAllPoolBackends() {
|
||||
return poolStopper.stopAll()
|
||||
async function stopAllPoolBackends() {
|
||||
const entries = [...backendPool.values()]
|
||||
await poolStopper.stopAll()
|
||||
entries.forEach(releaseLocalBackendSlot)
|
||||
}
|
||||
|
||||
const backendShutdown = createBackendShutdownCoordinator(async () => {
|
||||
@@ -14610,6 +14803,18 @@ ipcMain.handle('hermes:backend:touch', async (_event, profile) => {
|
||||
|
||||
return { ok: true }
|
||||
})
|
||||
// Pool sizing (Settings → Advanced): device-local, live-applied. Main is
|
||||
// authoritative (it owns the pool and the persisted copy); the returned
|
||||
// limits are what actually took effect post-clamp.
|
||||
ipcMain.handle('hermes:pool-limits:get', async () => ({ ...poolLimits }))
|
||||
ipcMain.handle('hermes:pool-limits:set', async (_event, raw) => {
|
||||
const next = setPoolLimits({
|
||||
maxBackends: typeof raw?.maxBackends === 'number' ? raw.maxBackends : poolLimits.maxBackends,
|
||||
idleMs: typeof raw?.idleMs === 'number' ? raw.idleMs : poolLimits.idleMs
|
||||
})
|
||||
|
||||
return { ok: true, limits: next }
|
||||
})
|
||||
ipcMain.handle('hermes:gateway:ws-url', async (_event, profile) => {
|
||||
return gatewayWsUrlIpcResult(() => freshGatewayWsUrl(profile))
|
||||
})
|
||||
@@ -16817,7 +17022,9 @@ ipcMain.on('hermes:translucency:support', event => {
|
||||
// shortcut edit), and survives self-relaunches because collectRelaunchArgs
|
||||
// only strips internal flags.
|
||||
ipcMain.on('hermes:launch-flags', event => {
|
||||
event.returnValue = { localModels: process.argv.includes('--local') }
|
||||
event.returnValue = {
|
||||
localModels: process.argv.includes('--local') || process.platform === 'win32' || process.platform === 'darwin'
|
||||
}
|
||||
})
|
||||
|
||||
ipcMain.on('hermes:translucency', (_event, payload) => {
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import {
|
||||
clampPoolLimits,
|
||||
parsePoolLimits,
|
||||
POOL_LIMITS_BOUNDS,
|
||||
POOL_LIMITS_DEFAULTS,
|
||||
POOL_LIMITS_MIN
|
||||
} from './pool-limits'
|
||||
|
||||
describe('parsePoolLimits', () => {
|
||||
it('falls back to defaults for null/empty/corrupt input', () => {
|
||||
expect(parsePoolLimits(null)).toEqual(POOL_LIMITS_DEFAULTS)
|
||||
expect(parsePoolLimits(undefined)).toEqual(POOL_LIMITS_DEFAULTS)
|
||||
expect(parsePoolLimits('')).toEqual(POOL_LIMITS_DEFAULTS)
|
||||
expect(parsePoolLimits('not json {')).toEqual(POOL_LIMITS_DEFAULTS)
|
||||
})
|
||||
|
||||
it('parses a valid persisted blob', () => {
|
||||
expect(parsePoolLimits(JSON.stringify({ maxBackends: 11, idleMs: 7_200_000 }))).toEqual({
|
||||
maxBackends: 11,
|
||||
idleMs: 7_200_000
|
||||
})
|
||||
})
|
||||
|
||||
it('fills missing keys from defaults', () => {
|
||||
expect(parsePoolLimits(JSON.stringify({ maxBackends: 5 }))).toEqual({ ...POOL_LIMITS_DEFAULTS, maxBackends: 5 })
|
||||
expect(parsePoolLimits('{}')).toEqual(POOL_LIMITS_DEFAULTS)
|
||||
})
|
||||
|
||||
it('ignores non-numeric junk instead of NaN-poisoning the pool', () => {
|
||||
expect(parsePoolLimits(JSON.stringify({ maxBackends: 'lots', idleMs: null }))).toEqual(POOL_LIMITS_DEFAULTS)
|
||||
})
|
||||
})
|
||||
|
||||
describe('clampPoolLimits', () => {
|
||||
it('clamps below the floors', () => {
|
||||
expect(clampPoolLimits({ maxBackends: 0 }).maxBackends).toBe(POOL_LIMITS_MIN.maxBackends)
|
||||
expect(clampPoolLimits({ idleMs: 100 }).idleMs).toBe(POOL_LIMITS_MIN.idleMs)
|
||||
})
|
||||
|
||||
it('clamps absurdly high backend counts', () => {
|
||||
expect(clampPoolLimits({ maxBackends: 10_000 }).maxBackends).toBeLessThanOrEqual(64)
|
||||
})
|
||||
|
||||
it('clamps idleMs to the shared ceiling (7 days)', () => {
|
||||
expect(clampPoolLimits({ idleMs: 999_000_000 }).idleMs).toBe(POOL_LIMITS_BOUNDS.idleMsMax)
|
||||
})
|
||||
|
||||
it('floors fractional values', () => {
|
||||
expect(clampPoolLimits({ maxBackends: 2.9 }).maxBackends).toBe(2)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,81 @@
|
||||
/**
|
||||
* Pool limits — how many bot backends may stay spawned, and how long an
|
||||
* unused one survives.
|
||||
*
|
||||
* A device-local preference (each machine trades RAM against switching
|
||||
* speed for itself), stored in userData like keep-awake. The main process
|
||||
* is authoritative: it owns the pool AND the persisted copy, and applies a
|
||||
* new max IMMEDIATELY by evicting least-recently-used idle backends — no
|
||||
* app restart. The renderer mirrors the values for its UI and prewarm
|
||||
* guard over IPC.
|
||||
*
|
||||
* Defaults preserve the historical hard-coded behavior (3 backends, 10min
|
||||
* idle) so machines that never open Settings behave exactly as before.
|
||||
*/
|
||||
|
||||
export interface PoolLimits {
|
||||
/** Max concurrently spawned non-primary profile backends. */
|
||||
maxBackends: number
|
||||
/** Idle lifetime of an unused pool backend, in milliseconds. */
|
||||
idleMs: number
|
||||
}
|
||||
|
||||
export const POOL_LIMITS_DEFAULTS: PoolLimits = {
|
||||
maxBackends: 3,
|
||||
idleMs: 10 * 60_000
|
||||
}
|
||||
|
||||
/** Hard floors — match the clamps the env-var path always applied. */
|
||||
export const POOL_LIMITS_MIN: PoolLimits = {
|
||||
maxBackends: 1,
|
||||
idleMs: 60_000
|
||||
}
|
||||
|
||||
/** Shared bounds for both pool knobs — imported by the Settings UI so the
|
||||
* advertised input ranges can never drift from what main actually clamps
|
||||
* to. idleMs has no ceiling: a user who wants backends kept warm all week
|
||||
* may have exactly that. */
|
||||
export const POOL_LIMITS_BOUNDS = {
|
||||
maxBackendsMax: 64,
|
||||
/** 7 days, matching the UI's suggestion ceiling. */
|
||||
idleMsMax: 7 * 24 * 60 * 60_000
|
||||
} as const
|
||||
|
||||
const MAX_BACKENDS_CEILING = POOL_LIMITS_BOUNDS.maxBackendsMax
|
||||
const IDLE_MS_CEILING = POOL_LIMITS_BOUNDS.idleMsMax
|
||||
|
||||
/** Clamp a raw partial to the floors/ceilings; missing keys fall to defaults. */
|
||||
export function clampPoolLimits(raw: Partial<PoolLimits>): PoolLimits {
|
||||
const maxBackends = Number.isFinite(raw.maxBackends)
|
||||
? Math.min(MAX_BACKENDS_CEILING, Math.max(POOL_LIMITS_MIN.maxBackends, Math.floor(Number(raw.maxBackends))))
|
||||
: POOL_LIMITS_DEFAULTS.maxBackends
|
||||
|
||||
const idleMs = Number.isFinite(raw.idleMs)
|
||||
? Math.min(IDLE_MS_CEILING, Math.max(POOL_LIMITS_MIN.idleMs, Math.floor(Number(raw.idleMs))))
|
||||
: POOL_LIMITS_DEFAULTS.idleMs
|
||||
|
||||
return { maxBackends, idleMs }
|
||||
}
|
||||
|
||||
function clampLimits(raw: Partial<PoolLimits>): PoolLimits {
|
||||
return clampPoolLimits(raw)
|
||||
}
|
||||
|
||||
/** Parse + clamp a persisted JSON blob; anything unreadable falls back to
|
||||
* defaults so a corrupted file can never wedge the pool. */
|
||||
export function parsePoolLimits(json: string | null | undefined): PoolLimits {
|
||||
if (!json) {
|
||||
return { ...POOL_LIMITS_DEFAULTS }
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = JSON.parse(json)
|
||||
|
||||
return clampLimits({
|
||||
maxBackends: typeof parsed?.maxBackends === 'number' ? parsed.maxBackends : undefined,
|
||||
idleMs: typeof parsed?.idleMs === 'number' ? parsed.idleMs : undefined
|
||||
})
|
||||
} catch {
|
||||
return { ...POOL_LIMITS_DEFAULTS }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,344 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { spawn } from 'node:child_process'
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
import { test } from 'vitest'
|
||||
|
||||
import { LocalBackendSpawnCoordinator, releaseLocalBackendSlotAfterExit } from './pool-spawn-coordinator'
|
||||
|
||||
const deferred = () => {
|
||||
let resolve!: () => void
|
||||
|
||||
const promise = new Promise<void>(done => {
|
||||
resolve = done
|
||||
})
|
||||
|
||||
return { promise, resolve }
|
||||
}
|
||||
|
||||
const flush = () => new Promise<void>(resolve => setImmediate(resolve))
|
||||
|
||||
test('100 concurrent local requests never hold more than the configured slots', async () => {
|
||||
const limit = 12
|
||||
const coordinator = new LocalBackendSpawnCoordinator(limit)
|
||||
const gates = Array.from({ length: 100 }, deferred)
|
||||
let active = 0
|
||||
let maxActive = 0
|
||||
|
||||
const tasks = gates.map(async (gate, index) => {
|
||||
const release = await coordinator.acquire(`profile-${index}`)
|
||||
active += 1
|
||||
maxActive = Math.max(maxActive, active)
|
||||
|
||||
await gate.promise
|
||||
|
||||
active -= 1
|
||||
release()
|
||||
})
|
||||
|
||||
await flush()
|
||||
assert.equal(active, limit)
|
||||
assert.equal(coordinator.activeCount, limit)
|
||||
assert.equal(coordinator.queuedCount, 100 - limit)
|
||||
|
||||
for (let start = 0; start < gates.length; start += limit) {
|
||||
for (const gate of gates.slice(start, start + limit)) {
|
||||
gate.resolve()
|
||||
}
|
||||
|
||||
await flush()
|
||||
}
|
||||
|
||||
await Promise.all(tasks)
|
||||
assert.equal(maxActive, limit)
|
||||
assert.equal(coordinator.activeCount, 0)
|
||||
assert.equal(coordinator.queuedCount, 0)
|
||||
})
|
||||
|
||||
test('a queued start can be cancelled without waiting for an active backend', async () => {
|
||||
const coordinator = new LocalBackendSpawnCoordinator(1)
|
||||
const releaseFirst = await coordinator.acquire('first')
|
||||
const queued = coordinator.request('cancelled')
|
||||
|
||||
assert.equal(coordinator.queuedCount, 1)
|
||||
assert.equal(queued.cancel(), true)
|
||||
await assert.rejects(queued.acquired, /cancelled while queued/)
|
||||
assert.equal(coordinator.activeCount, 1)
|
||||
assert.equal(coordinator.queuedCount, 0)
|
||||
|
||||
releaseFirst()
|
||||
assert.equal(coordinator.activeCount, 0)
|
||||
})
|
||||
|
||||
test('cancelling an old same-key request never rejects a newer waiter', async () => {
|
||||
const coordinator = new LocalBackendSpawnCoordinator(1)
|
||||
const blocker = coordinator.request('blocker')
|
||||
const releaseBlocker = await blocker.acquired
|
||||
const old = coordinator.request('same-profile')
|
||||
|
||||
releaseBlocker()
|
||||
const newer = coordinator.request('same-profile')
|
||||
|
||||
assert.equal(old.cancel(), false, 'the old request was already granted')
|
||||
assert.equal(coordinator.queuedCount, 1, 'the newer same-key waiter must remain queued')
|
||||
|
||||
const releaseOld = await old.acquired
|
||||
releaseOld()
|
||||
const releaseNewer = await newer.acquired
|
||||
releaseNewer()
|
||||
|
||||
assert.equal(coordinator.activeCount, 0)
|
||||
assert.equal(coordinator.queuedCount, 0)
|
||||
})
|
||||
|
||||
test('a queued start times out with a clear error and frees its queue position', async () => {
|
||||
const coordinator = new LocalBackendSpawnCoordinator(1)
|
||||
const releaseFirst = await coordinator.acquire('first')
|
||||
const queued = coordinator.request('timed-out', { timeoutMs: 10 })
|
||||
|
||||
await assert.rejects(queued.acquired, /timed out while waiting for a free slot/)
|
||||
assert.equal(coordinator.activeCount, 1)
|
||||
assert.equal(coordinator.queuedCount, 0)
|
||||
|
||||
releaseFirst()
|
||||
assert.equal(coordinator.activeCount, 0)
|
||||
})
|
||||
|
||||
test('100 real child processes never exceed twelve simultaneous local slots', async () => {
|
||||
const limit = 12
|
||||
const coordinator = new LocalBackendSpawnCoordinator(limit)
|
||||
const livePids = new Set<number>()
|
||||
const seenPids = new Set<number>()
|
||||
let maxLive = 0
|
||||
|
||||
await Promise.all(
|
||||
Array.from({ length: 100 }, async (_, index) => {
|
||||
const release = await coordinator.acquire(`real-profile-${index}`)
|
||||
|
||||
try {
|
||||
const child = spawn(process.execPath, ['-e', 'setTimeout(() => {}, 40)'], {
|
||||
stdio: 'ignore'
|
||||
})
|
||||
|
||||
assert.ok(child.pid)
|
||||
livePids.add(child.pid)
|
||||
seenPids.add(child.pid)
|
||||
maxLive = Math.max(maxLive, livePids.size)
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
child.once('error', reject)
|
||||
child.once('exit', code => {
|
||||
if (code === 0) {
|
||||
resolve()
|
||||
} else {
|
||||
reject(new Error(`child ${child.pid} exited with ${code}`))
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
livePids.delete(child.pid)
|
||||
} finally {
|
||||
release()
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
assert.equal(seenPids.size, 100)
|
||||
assert.equal(maxLive, limit)
|
||||
assert.equal(livePids.size, 0)
|
||||
assert.equal(coordinator.activeCount, 0)
|
||||
assert.equal(coordinator.queuedCount, 0)
|
||||
})
|
||||
|
||||
test('failed start keeps its slot until the child has actually exited', async () => {
|
||||
const coordinator = new LocalBackendSpawnCoordinator(1)
|
||||
const childExit = deferred()
|
||||
const releaseFailed = await coordinator.acquire('failed')
|
||||
let successorEntered = false
|
||||
|
||||
const successor = coordinator.acquire('successor').then(release => {
|
||||
successorEntered = true
|
||||
|
||||
return release
|
||||
})
|
||||
|
||||
const cleanup = releaseLocalBackendSlotAfterExit(releaseFailed, () => childExit.promise)
|
||||
await flush()
|
||||
|
||||
assert.equal(successorEntered, false)
|
||||
assert.equal(coordinator.activeCount, 1)
|
||||
assert.equal(coordinator.queuedCount, 1)
|
||||
|
||||
childExit.resolve()
|
||||
await cleanup
|
||||
const releaseSuccessor = await successor
|
||||
|
||||
assert.equal(successorEntered, true)
|
||||
assert.equal(coordinator.activeCount, 1)
|
||||
assert.equal(coordinator.queuedCount, 0)
|
||||
|
||||
releaseSuccessor()
|
||||
assert.equal(coordinator.activeCount, 0)
|
||||
})
|
||||
|
||||
test('a rejected wait keeps the slot occupied', async () => {
|
||||
const coordinator = new LocalBackendSpawnCoordinator(1)
|
||||
const releaseFailed = await coordinator.acquire('failed')
|
||||
let successorEntered = false
|
||||
|
||||
const successor = coordinator.acquire('successor').then(release => {
|
||||
successorEntered = true
|
||||
|
||||
return release
|
||||
})
|
||||
|
||||
const cleanup = releaseLocalBackendSlotAfterExit(releaseFailed, async () => {
|
||||
throw new Error('exit unproven')
|
||||
})
|
||||
|
||||
await assert.rejects(cleanup, /exit unproven/)
|
||||
await flush()
|
||||
|
||||
assert.equal(successorEntered, false)
|
||||
assert.equal(coordinator.activeCount, 1)
|
||||
assert.equal(coordinator.queuedCount, 1)
|
||||
|
||||
releaseFailed()
|
||||
const releaseSuccessor = await successor
|
||||
assert.equal(successorEntered, true)
|
||||
releaseSuccessor()
|
||||
assert.equal(coordinator.activeCount, 0)
|
||||
assert.equal(coordinator.queuedCount, 0)
|
||||
})
|
||||
|
||||
test('an invalid timeout never enqueues a waiter', async () => {
|
||||
const coordinator = new LocalBackendSpawnCoordinator(1)
|
||||
const releaseFirst = await coordinator.acquire('first')
|
||||
|
||||
assert.throws(() => coordinator.request('invalid', { timeoutMs: 0 }), /timeout must be a positive number/)
|
||||
assert.throws(() => coordinator.request('invalid', { timeoutMs: Number.NaN }), /timeout must be a positive number/)
|
||||
assert.throws(() => coordinator.request('invalid', { timeoutMs: -5 }), /timeout must be a positive number/)
|
||||
|
||||
assert.equal(coordinator.activeCount, 1)
|
||||
assert.equal(coordinator.queuedCount, 0)
|
||||
|
||||
releaseFirst()
|
||||
assert.equal(coordinator.activeCount, 0)
|
||||
})
|
||||
|
||||
test('a failed or repeated cleanup releases exactly one slot', async () => {
|
||||
const coordinator = new LocalBackendSpawnCoordinator(1)
|
||||
const releaseFirst = await coordinator.acquire('first')
|
||||
let secondEntered = false
|
||||
|
||||
const second = coordinator.acquire('second').then(release => {
|
||||
secondEntered = true
|
||||
|
||||
return release
|
||||
})
|
||||
|
||||
await flush()
|
||||
assert.equal(secondEntered, false)
|
||||
assert.equal(coordinator.activeCount, 1)
|
||||
assert.equal(coordinator.queuedCount, 1)
|
||||
|
||||
releaseFirst()
|
||||
releaseFirst()
|
||||
const releaseSecond = await second
|
||||
|
||||
assert.equal(secondEntered, true)
|
||||
assert.equal(coordinator.activeCount, 1)
|
||||
assert.equal(coordinator.queuedCount, 0)
|
||||
|
||||
releaseSecond()
|
||||
assert.equal(coordinator.activeCount, 0)
|
||||
})
|
||||
|
||||
test('raising the limit at runtime drains queued waiters into the new slots', async () => {
|
||||
const coordinator = new LocalBackendSpawnCoordinator(1)
|
||||
const first = await coordinator.acquire('a')
|
||||
const queuedB = coordinator.request('b')
|
||||
const queuedC = coordinator.request('c')
|
||||
await flush()
|
||||
assert.equal(coordinator.activeCount, 1)
|
||||
assert.equal(coordinator.queuedCount, 2)
|
||||
|
||||
coordinator.setLimit(2)
|
||||
const releaseB = await queuedB.acquired
|
||||
assert.equal(coordinator.activeCount, 2)
|
||||
assert.equal(coordinator.queuedCount, 1)
|
||||
|
||||
first()
|
||||
const releaseC = await queuedC.acquired
|
||||
assert.equal(coordinator.activeCount, 2)
|
||||
releaseB()
|
||||
releaseC()
|
||||
assert.equal(coordinator.activeCount, 0)
|
||||
})
|
||||
|
||||
test('lowering the limit never revokes granted slots; new requests queue until under cap', async () => {
|
||||
const coordinator = new LocalBackendSpawnCoordinator(3)
|
||||
const releases = await Promise.all(['a', 'b', 'c'].map(key => coordinator.acquire(key)))
|
||||
coordinator.setLimit(1)
|
||||
assert.equal(coordinator.activeCount, 3, 'granted slots stay granted')
|
||||
|
||||
const queued = coordinator.request('d')
|
||||
await flush()
|
||||
assert.equal(coordinator.queuedCount, 1)
|
||||
|
||||
releases[0]()
|
||||
releases[1]()
|
||||
await flush()
|
||||
assert.equal(coordinator.queuedCount, 1, 'still over the new cap of 1')
|
||||
|
||||
releases[2]()
|
||||
const releaseD = await queued.acquired
|
||||
assert.equal(coordinator.activeCount, 1)
|
||||
releaseD()
|
||||
})
|
||||
|
||||
test('setLimit rejects a non-positive or fractional cap', () => {
|
||||
const coordinator = new LocalBackendSpawnCoordinator(2)
|
||||
assert.throws(() => coordinator.setLimit(0), RangeError)
|
||||
assert.throws(() => coordinator.setLimit(1.5), RangeError)
|
||||
assert.equal(coordinator.limit, 2)
|
||||
})
|
||||
|
||||
// ── main.ts wiring ──────────────────────────────────────────────────────────
|
||||
// The coordinator is only as good as the timeout main.ts hands it. A queued
|
||||
// ticket that outlives the renderer's backend-boot budget holds the pool key
|
||||
// hostage: the renderer has already reported "backend didn't come up", and
|
||||
// every later click on that profile joins the stale wait instead of failing
|
||||
// fast with a reason.
|
||||
{
|
||||
const here = path.dirname(fileURLToPath(import.meta.url))
|
||||
const mainSource = fs.readFileSync(path.join(here, 'main.ts'), 'utf8').replace(/\r\n/g, '\n')
|
||||
|
||||
const withTimeoutSource = fs
|
||||
.readFileSync(path.join(here, '..', 'src', 'lib', 'with-timeout.ts'), 'utf8')
|
||||
.replace(/\r\n/g, '\n')
|
||||
|
||||
test('main.ts bounds the slot wait below the renderer backend-boot budget', () => {
|
||||
const slotWait = Number(/const POOL_SLOT_WAIT_MS = ([\d_]+)/.exec(mainSource)?.[1]?.replace(/_/g, ''))
|
||||
|
||||
const bootBudget = Number(
|
||||
/export const BACKEND_BOOT_WAIT_TIMEOUT_MS = ([\d_]+)/.exec(withTimeoutSource)?.[1]?.replace(/_/g, '')
|
||||
)
|
||||
|
||||
assert.ok(Number.isFinite(slotWait) && slotWait > 0, 'POOL_SLOT_WAIT_MS must be a literal in main.ts')
|
||||
assert.ok(Number.isFinite(bootBudget), 'BACKEND_BOOT_WAIT_TIMEOUT_MS must be a literal')
|
||||
assert.ok(slotWait < bootBudget, `slot wait ${slotWait}ms must be below the boot budget ${bootBudget}ms`)
|
||||
assert.match(mainSource, /localBackendSpawnCoordinator\.request\(poolKey, \{ timeoutMs: POOL_SLOT_WAIT_MS \}\)/)
|
||||
assert.doesNotMatch(mainSource, /request\(poolKey, \{ timeoutMs: POOL_IDLE_MS \}\)/)
|
||||
})
|
||||
|
||||
test('main.ts pushes the live pool max into the coordinator when the preference changes', () => {
|
||||
// Pool sizing is a live device preference (#92581); the hard cap must
|
||||
// follow it, otherwise raising the max in Settings would leave spawns
|
||||
// queued behind the launch-time value.
|
||||
assert.match(mainSource, /new LocalBackendSpawnCoordinator\(poolLimits\.maxBackends\)/)
|
||||
assert.match(mainSource, /localBackendSpawnCoordinator\.setLimit\(poolLimits\.maxBackends\)/)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
export type ReleaseLocalBackendSlot = () => void
|
||||
|
||||
export type LocalBackendSpawnRequest = {
|
||||
acquired: Promise<ReleaseLocalBackendSlot>
|
||||
cancel: () => boolean
|
||||
}
|
||||
|
||||
type Waiter = {
|
||||
key: string
|
||||
resolve: (release: ReleaseLocalBackendSlot) => void
|
||||
reject: (error: Error) => void
|
||||
timer: ReturnType<typeof setTimeout> | null
|
||||
}
|
||||
|
||||
export async function releaseLocalBackendSlotAfterExit(
|
||||
release: ReleaseLocalBackendSlot,
|
||||
waitForExit: () => Promise<void>
|
||||
): Promise<void> {
|
||||
await waitForExit()
|
||||
release()
|
||||
}
|
||||
|
||||
/**
|
||||
* Bounds the number of local profile backends that are starting or running.
|
||||
*
|
||||
* A lease is acquired immediately before local start work and is held until
|
||||
* the child exits or the start fails. Remote descriptors never call request().
|
||||
*/
|
||||
export class LocalBackendSpawnCoordinator {
|
||||
#limit: number
|
||||
#active = 0
|
||||
#queue: Waiter[] = []
|
||||
|
||||
constructor(limit: number) {
|
||||
if (!Number.isInteger(limit) || limit < 1) {
|
||||
throw new RangeError('Local backend spawn limit must be a positive integer.')
|
||||
}
|
||||
|
||||
this.#limit = limit
|
||||
}
|
||||
|
||||
get activeCount(): number {
|
||||
return this.#active
|
||||
}
|
||||
|
||||
get limit(): number {
|
||||
return this.#limit
|
||||
}
|
||||
|
||||
/**
|
||||
* Adopt a new cap at runtime (the pool size is a live device preference).
|
||||
* Raising it drains waiters into the newly freed slots immediately; lowering
|
||||
* it never revokes a granted slot — the running backends simply stay over
|
||||
* the cap until they exit, and LRU eviction (main.ts) converges the pool.
|
||||
*/
|
||||
setLimit(limit: number): void {
|
||||
if (!Number.isInteger(limit) || limit < 1) {
|
||||
throw new RangeError('Local backend spawn limit must be a positive integer.')
|
||||
}
|
||||
|
||||
this.#limit = limit
|
||||
this.#drain()
|
||||
}
|
||||
|
||||
get queuedCount(): number {
|
||||
return this.#queue.length
|
||||
}
|
||||
|
||||
request(key: string, options: { timeoutMs?: number } = {}): LocalBackendSpawnRequest {
|
||||
if (options.timeoutMs !== undefined && (!Number.isFinite(options.timeoutMs) || options.timeoutMs < 1)) {
|
||||
throw new RangeError('Local backend spawn timeout must be a positive number.')
|
||||
}
|
||||
|
||||
if (this.#active < this.#limit) {
|
||||
return {
|
||||
acquired: Promise.resolve(this.#grant()),
|
||||
cancel: () => false
|
||||
}
|
||||
}
|
||||
|
||||
let waiter!: Waiter
|
||||
|
||||
const acquired = new Promise<ReleaseLocalBackendSlot>((resolve, reject) => {
|
||||
waiter = { key, resolve, reject, timer: null }
|
||||
this.#queue.push(waiter)
|
||||
|
||||
if (options.timeoutMs !== undefined) {
|
||||
waiter.timer = setTimeout(() => {
|
||||
this.#rejectWaiter(
|
||||
waiter,
|
||||
new Error(`Local backend start for "${key}" timed out while waiting for a free slot.`)
|
||||
)
|
||||
}, options.timeoutMs)
|
||||
waiter.timer.unref?.()
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
acquired,
|
||||
cancel: () =>
|
||||
this.#rejectWaiter(waiter, new Error(`Local backend start for "${key}" was cancelled while queued.`))
|
||||
}
|
||||
}
|
||||
|
||||
acquire(key: string): Promise<ReleaseLocalBackendSlot> {
|
||||
return this.request(key).acquired
|
||||
}
|
||||
|
||||
#rejectWaiter(waiter: Waiter, error: Error): boolean {
|
||||
const index = this.#queue.indexOf(waiter)
|
||||
|
||||
if (index === -1) {
|
||||
return false
|
||||
}
|
||||
|
||||
this.#queue.splice(index, 1)
|
||||
this.#clearTimer(waiter)
|
||||
waiter.reject(error)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
#clearTimer(waiter: Waiter): void {
|
||||
if (waiter.timer) {
|
||||
clearTimeout(waiter.timer)
|
||||
waiter.timer = null
|
||||
}
|
||||
}
|
||||
|
||||
#grant(): ReleaseLocalBackendSlot {
|
||||
this.#active += 1
|
||||
let released = false
|
||||
|
||||
return () => {
|
||||
if (released) {
|
||||
return
|
||||
}
|
||||
|
||||
released = true
|
||||
this.#active -= 1
|
||||
this.#drain()
|
||||
}
|
||||
}
|
||||
|
||||
/** Hand free slots to queued waiters while under the (possibly lowered) cap. */
|
||||
#drain(): void {
|
||||
while (this.#active < this.#limit && this.#queue.length > 0) {
|
||||
const next = this.#queue.shift()!
|
||||
this.#clearTimer(next)
|
||||
next.resolve(this.#grant())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,8 @@ contextBridge.exposeInMainWorld('hermesDesktop', {
|
||||
getProfileRoutes: profiles => ipcRenderer.invoke('hermes:plugin-profile-routes', profiles),
|
||||
revalidateConnection: () => ipcRenderer.invoke('hermes:connection:revalidate'),
|
||||
touchBackend: profile => ipcRenderer.invoke('hermes:backend:touch', profile),
|
||||
getPoolLimits: () => ipcRenderer.invoke('hermes:pool-limits:get'),
|
||||
setPoolLimits: limits => ipcRenderer.invoke('hermes:pool-limits:set', limits),
|
||||
getGatewayWsUrl: profile => ipcRenderer.invoke('hermes:gateway:ws-url', profile),
|
||||
// Registry-scoped fresh WS URL: { connectionId, profile } → result shape of
|
||||
// getGatewayWsUrl, minted against that connection's backend.
|
||||
|
||||
@@ -78,7 +78,7 @@ import {
|
||||
mergeOlderTranscriptPage,
|
||||
transcriptBackfillAvailable
|
||||
} from './transcript-backfill'
|
||||
import { advanceTranscriptWindow, type TranscriptWindowState } from './transcript-window'
|
||||
import { advanceSessionTranscriptWindow, type SessionWindowMemo } from './transcript-window'
|
||||
|
||||
interface ChatViewProps extends Omit<React.ComponentProps<'div'>, 'onSubmit'> {
|
||||
gateway: HermesGateway | null
|
||||
@@ -247,22 +247,37 @@ function ChatRuntimeBoundary({
|
||||
|
||||
const [windowPages, setWindowPages] = useState(1)
|
||||
const [windowSessionKey, setWindowSessionKey] = useState(runtimeId)
|
||||
// Sticky-cut continuity across flushes (advanceTranscriptWindow). A ref, not
|
||||
// state: it is derived from `messages` and must never trigger a render.
|
||||
const windowStateRef = useRef<null | TranscriptWindowState>(null)
|
||||
// Per-session sticky-cut continuity (advanceSessionTranscriptWindow). A ref,
|
||||
// not state: it is derived from `messages` and must never trigger a render.
|
||||
// Keyed by runtime id so a warm switch back to a session whose transcript
|
||||
// is unchanged reuses the previous windowed slice BY REFERENCE — no window
|
||||
// re-index, no runtime-repository rebuild, no per-row re-parse/re-highlight
|
||||
// (#95595). Bounded internally (oldest session evicted).
|
||||
const windowStateRef = useRef(new Map<string, SessionWindowMemo>())
|
||||
// The memo below intentionally skips `runtimeId` in its deps (a switch
|
||||
// always changes the messages array too, which re-runs it), so the current
|
||||
// value must come from a ref rather than the stale render closure.
|
||||
const runtimeIdRef = useRef(runtimeId)
|
||||
runtimeIdRef.current = runtimeId
|
||||
|
||||
// Reset the window on session swap during RENDER, so a large expand from the
|
||||
// previous chat can't leak into the next one's first paint (#55191).
|
||||
// previous chat can't leak into the next one's first paint (#55191). The
|
||||
// per-session map above keeps each session's own cut; only the page count
|
||||
// resets on a switch.
|
||||
if (windowSessionKey !== runtimeId) {
|
||||
setWindowSessionKey(runtimeId)
|
||||
setWindowPages(1)
|
||||
windowStateRef.current = null
|
||||
}
|
||||
|
||||
const { messages: windowedMessages, windowed } = useMemo(() => {
|
||||
const next = advanceTranscriptWindow(windowStateRef.current, messages, windowPages)
|
||||
|
||||
windowStateRef.current = next
|
||||
const next = advanceSessionTranscriptWindow(
|
||||
windowStateRef.current,
|
||||
// Draft state has no runtime id yet; a single shared slot is fine there
|
||||
// (mirrors the old single-slot behaviour for the no-runtime case).
|
||||
runtimeIdRef.current ?? '',
|
||||
messages,
|
||||
windowPages
|
||||
)
|
||||
|
||||
return next.window
|
||||
}, [messages, windowPages])
|
||||
@@ -296,7 +311,7 @@ function ChatRuntimeBoundary({
|
||||
// something older to show. Fire-and-forget: the prepend lands through the
|
||||
// session-state write path and re-renders this boundary.
|
||||
if (
|
||||
!windowStateRef.current?.window.windowed &&
|
||||
!windowStateRef.current.get(runtimeIdRef.current ?? '')?.state.window.windowed &&
|
||||
runtimeId &&
|
||||
storedId &&
|
||||
transcriptBackfillAvailable(storedId, tailProfile)
|
||||
|
||||
@@ -347,19 +347,7 @@ function MarkdownCode({ className, children, ...props }: ComponentProps<'code'>)
|
||||
|
||||
const code = String(children).replace(/\n$/, '')
|
||||
|
||||
const highlighted = (
|
||||
<ShikiHighlighter
|
||||
addDefaultStyles={false}
|
||||
as="div"
|
||||
defaultColor="light-dark()"
|
||||
delay={80}
|
||||
language={language}
|
||||
showLanguage={false}
|
||||
theme={SHIKI_THEME}
|
||||
>
|
||||
{code}
|
||||
</ShikiHighlighter>
|
||||
)
|
||||
const highlighted = <ShikiHighlighter code={code} language={language} theme={SHIKI_THEME} />
|
||||
|
||||
// ```mermaid / ```svg fences route to the shared lazy renderers (same
|
||||
// registry the chat transcript uses); everything else stays on Shiki.
|
||||
@@ -661,17 +649,7 @@ export function SourceView({ filePath, language, text }: { filePath?: string; la
|
||||
})}
|
||||
</div>
|
||||
<div className="preview-source-code min-w-0 [&_pre]:m-0" data-selectable-text="true">
|
||||
<ShikiHighlighter
|
||||
addDefaultStyles={false}
|
||||
as="div"
|
||||
defaultColor="light-dark()"
|
||||
delay={80}
|
||||
language={language || 'text'}
|
||||
showLanguage={false}
|
||||
theme={SHIKI_THEME}
|
||||
>
|
||||
{chunk.text}
|
||||
</ShikiHighlighter>
|
||||
<ShikiHighlighter code={chunk.text} language={language || 'text'} theme={SHIKI_THEME} />
|
||||
</div>
|
||||
</Fragment>
|
||||
))}
|
||||
|
||||
@@ -4,8 +4,10 @@ import type { ChatMessage } from '@/lib/chat-messages'
|
||||
import { RENDER_WEIGHT_CHARS } from '@/lib/render-weight'
|
||||
|
||||
import {
|
||||
advanceSessionTranscriptWindow,
|
||||
advanceTranscriptWindow,
|
||||
alignToBranchGroup,
|
||||
MAX_SESSION_WINDOWS,
|
||||
selectTranscriptWindow,
|
||||
TRANSCRIPT_WINDOW_BUDGET,
|
||||
TRANSCRIPT_WINDOW_MIN_MESSAGES,
|
||||
@@ -214,6 +216,87 @@ describe('advanceTranscriptWindow', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('advanceSessionTranscriptWindow', () => {
|
||||
const heavyChars = RENDER_WEIGHT_CHARS * 40
|
||||
|
||||
it('matches a fresh walk on first visit', () => {
|
||||
const memos = new Map()
|
||||
const messages = transcript(400, heavyChars)
|
||||
|
||||
const state = advanceSessionTranscriptWindow(memos, 'session-a', messages)
|
||||
|
||||
expect(state.window).toEqual(selectTranscriptWindow(messages))
|
||||
expect(state.anchorId).toBe(state.window.messages[0].id)
|
||||
})
|
||||
|
||||
it('returns the SAME windowed slice by reference on a warm re-visit with an unchanged transcript', () => {
|
||||
const memos = new Map()
|
||||
const sessionA = transcript(400, heavyChars)
|
||||
const sessionB = transcript(300, heavyChars).map(m => ({ ...m, id: `b-${m.id}` }))
|
||||
|
||||
// Visit B, then A, then B again — the exact warm-switch shape of #95595.
|
||||
const firstB = advanceSessionTranscriptWindow(memos, 'session-b', sessionB)
|
||||
advanceSessionTranscriptWindow(memos, 'session-a', sessionA)
|
||||
const secondB = advanceSessionTranscriptWindow(memos, 'session-b', sessionB)
|
||||
|
||||
expect(secondB.window.windowed).toBe(true)
|
||||
// THE perf guard: same transcript array => same windowed slice reference,
|
||||
// so the runtime repository and every row keep their identity.
|
||||
expect(secondB.window.messages).toBe(firstB.window.messages)
|
||||
expect(secondB.anchorId).toBe(firstB.anchorId)
|
||||
})
|
||||
|
||||
it('holds the sticky cut when a re-visited session grew while away', () => {
|
||||
const memos = new Map()
|
||||
const messages = transcript(400, heavyChars)
|
||||
const state = advanceSessionTranscriptWindow(memos, 'session-a', messages)
|
||||
|
||||
// While away, the session streamed a few light turns (within slack).
|
||||
const grown = [...messages, ...transcript(10, 100)]
|
||||
const next = advanceSessionTranscriptWindow(memos, 'session-a', grown)
|
||||
|
||||
// Sticky cut survived the switch-away: same anchor, no fresh re-walk.
|
||||
expect(next.anchorId).toBe(state.anchorId)
|
||||
expect(next.window.messages[0].id).toBe(state.window.messages[0].id)
|
||||
// The anchored slice now simply includes the 10 new light turns.
|
||||
expect(next.window.messages.length).toBe(state.window.messages.length + 10)
|
||||
})
|
||||
|
||||
it('falls back to a fresh walk when the anchor vanished while away', () => {
|
||||
const memos = new Map()
|
||||
const messages = transcript(400, heavyChars)
|
||||
advanceSessionTranscriptWindow(memos, 'session-a', messages)
|
||||
|
||||
// Compression rewrite: disjoint ids while the user was elsewhere.
|
||||
const rewritten = transcript(300, heavyChars).map(m => ({ ...m, id: `compressed-${m.id}` }))
|
||||
const next = advanceSessionTranscriptWindow(memos, 'session-a', rewritten)
|
||||
|
||||
expect(next.window).toEqual(selectTranscriptWindow(rewritten))
|
||||
})
|
||||
|
||||
it('re-walks when pages change on re-entry', () => {
|
||||
const memos = new Map()
|
||||
const messages = transcript(400, heavyChars)
|
||||
|
||||
const one = advanceSessionTranscriptWindow(memos, 'session-a', messages, 1)
|
||||
const two = advanceSessionTranscriptWindow(memos, 'session-a', messages, 2)
|
||||
|
||||
expect(two.window.messages.length).toBeGreaterThan(one.window.messages.length)
|
||||
})
|
||||
|
||||
it('keeps sessions independent and evicts the oldest memo past the cap', () => {
|
||||
const memos = new Map()
|
||||
|
||||
for (let i = 0; i < MAX_SESSION_WINDOWS + 5; i++) {
|
||||
advanceSessionTranscriptWindow(memos, `session-${i}`, transcript(400, heavyChars))
|
||||
}
|
||||
|
||||
expect(memos.size).toBeLessThanOrEqual(MAX_SESSION_WINDOWS)
|
||||
expect(memos.has('session-0')).toBe(false)
|
||||
expect(memos.has(`session-${MAX_SESSION_WINDOWS + 4}`)).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('alignToBranchGroup', () => {
|
||||
const messages = [message('u-1', 10), message('a-1', 10, 'g'), message('a-2', 10, 'g'), message('u-2', 10)]
|
||||
|
||||
|
||||
@@ -170,3 +170,65 @@ export function advanceTranscriptWindow(
|
||||
|
||||
return { anchorId: window.windowed ? window.messages[0].id : null, pages, window }
|
||||
}
|
||||
|
||||
/** How many sessions keep a sticky window before the oldest is evicted. */
|
||||
export const MAX_SESSION_WINDOWS = 12
|
||||
|
||||
/**
|
||||
* A window state plus the exact message array it was computed from.
|
||||
* The array identity is load-bearing: when a session is re-entered with the
|
||||
* IDENTICAL transcript (the warm-switch path of #95595), the stored window —
|
||||
* including the exact `window.messages` slice reference — is reused as-is.
|
||||
* The reference reuse is what stops `useRuntimeMessageRepository` from
|
||||
* rebuilding (and every row from re-rendering) on a warm switch.
|
||||
*/
|
||||
export interface SessionWindowMemo {
|
||||
messages: readonly ChatMessage[]
|
||||
state: TranscriptWindowState
|
||||
}
|
||||
|
||||
/**
|
||||
* `advanceTranscriptWindow` with a STICKY cut that survives session switches.
|
||||
*
|
||||
* The previous single-slot state was nulled on every switch, so a warm
|
||||
* re-entry always re-ran the weight walk and rebuilt the windowed slice —
|
||||
* which re-indexed the whole windowed transcript (markdown re-parse +
|
||||
* re-highlight per row) even though nothing had changed. This keeps one memo
|
||||
* per session:
|
||||
*
|
||||
* - Re-entering a session with the same transcript array returns the cached
|
||||
* windowed slice BY REFERENCE — the runtime repository and every message
|
||||
* row stay mounted, so the switch is O(1).
|
||||
* - Re-entering with a changed transcript keeps the sticky cut (anchor still
|
||||
* present, tail within budget + slack) instead of re-walking from scratch.
|
||||
* - The anchor vanishing (compression rewrite) or a pages change falls
|
||||
* through to `advanceTranscriptWindow`'s existing fresh-walk behaviour.
|
||||
*
|
||||
* The map is bounded (oldest session evicted) so an unbounded session list
|
||||
* cannot grow it without limit.
|
||||
*/
|
||||
export function advanceSessionTranscriptWindow(
|
||||
memos: Map<string, SessionWindowMemo>,
|
||||
sessionKey: string,
|
||||
messages: readonly ChatMessage[],
|
||||
pages = 1
|
||||
): TranscriptWindowState {
|
||||
const memo = memos.get(sessionKey)
|
||||
|
||||
// Warm re-visit with the identical transcript and page count: reuse the
|
||||
// cached state wholesale, preserving the windowed slice reference.
|
||||
if (memo && memo.messages === messages && memo.state.pages === pages) {
|
||||
return memo.state
|
||||
}
|
||||
|
||||
const state = advanceTranscriptWindow(memo?.state ?? null, messages, pages)
|
||||
|
||||
memos.set(sessionKey, { messages, state })
|
||||
|
||||
if (memos.size > MAX_SESSION_WINDOWS) {
|
||||
const oldest = memos.keys().next().value as string
|
||||
memos.delete(oldest)
|
||||
}
|
||||
|
||||
return state
|
||||
}
|
||||
|
||||
@@ -46,6 +46,7 @@ import {
|
||||
} from '@/store/gateway-switch'
|
||||
import { checkLocalRuntimeUpdate, watchLocalRuntimeJobs } from '@/store/local-runtime-jobs'
|
||||
import { notify, notifyError } from '@/store/notifications'
|
||||
import { loadPoolLimits } from '@/store/pool-limits'
|
||||
import {
|
||||
$activeGatewayProfile,
|
||||
normalizeProfileKey,
|
||||
@@ -900,6 +901,10 @@ export function useGatewayBoot({
|
||||
// this a socket dropped during sleep sits closed until the user clicks.
|
||||
window.addEventListener('focus', onFocus)
|
||||
|
||||
// Pool limits are main-process state; mirror them once for the Settings
|
||||
// rows and prewarmProfileBackend's saturation guard.
|
||||
void loadPoolLimits()
|
||||
|
||||
// Keep live pool backends alive while this window is open (the main process
|
||||
// can't observe the direct renderer↔backend WS). No-op for the primary.
|
||||
const keepaliveTimer = setInterval(() => {
|
||||
|
||||
@@ -157,6 +157,7 @@ import {
|
||||
isSessionGoneError,
|
||||
overlayConcurrentMessageChanges,
|
||||
patchSessionWorkspace,
|
||||
preserveEquivalentTranscript,
|
||||
preserveLocalPendingTurnMessages,
|
||||
reconcileResumeMessages,
|
||||
removeRepresentedLocalLiveProjection,
|
||||
@@ -1372,26 +1373,37 @@ export function useSessionActions({
|
||||
|
||||
const activatedState = updateSessionState(
|
||||
cachedRuntimeId,
|
||||
state => ({
|
||||
...state,
|
||||
messages: visibleActivatedMessages,
|
||||
transcriptProvenance:
|
||||
acceptedPersistedDisplayTranscript || hasValidProvenance
|
||||
? (expectedProvenance ?? undefined)
|
||||
: undefined,
|
||||
...(pendingClarifyProjection
|
||||
? {
|
||||
awaitingResponse: false,
|
||||
sawAssistantPayload: true,
|
||||
streamId: pendingClarifyProjection.streamId
|
||||
}
|
||||
: {}),
|
||||
...(clearedClarifyProjection
|
||||
? {
|
||||
streamId: state.busy ? (clearedClarifyProjection.streamId ?? state.streamId) : null
|
||||
}
|
||||
: {})
|
||||
}),
|
||||
state => {
|
||||
// #95595: the reconcilers above always produce fresh
|
||||
// message objects, so an unconditional publish replaces the
|
||||
// warm-cached array with new-object equivalents and every
|
||||
// visible row re-normalizes + remounts (markdown re-parse +
|
||||
// shiki re-highlight per row, seconds of main-thread work).
|
||||
// Keep the existing array when the content is unchanged —
|
||||
// same guard the cold-resume path uses below.
|
||||
const messages = preserveEquivalentTranscript(state.messages, visibleActivatedMessages)
|
||||
|
||||
return {
|
||||
...state,
|
||||
messages,
|
||||
transcriptProvenance:
|
||||
acceptedPersistedDisplayTranscript || hasValidProvenance
|
||||
? (expectedProvenance ?? undefined)
|
||||
: undefined,
|
||||
...(pendingClarifyProjection
|
||||
? {
|
||||
awaitingResponse: false,
|
||||
sawAssistantPayload: true,
|
||||
streamId: pendingClarifyProjection.streamId
|
||||
}
|
||||
: {}),
|
||||
...(clearedClarifyProjection
|
||||
? {
|
||||
streamId: state.busy ? (clearedClarifyProjection.streamId ?? state.streamId) : null
|
||||
}
|
||||
: {})
|
||||
}
|
||||
},
|
||||
storedSessionId
|
||||
)
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
goneSessionVerdict,
|
||||
isSessionGoneError,
|
||||
overlayConcurrentMessageChanges,
|
||||
preserveEquivalentTranscript,
|
||||
preserveLocalPendingTurnMessages,
|
||||
reconcileResumeMessages,
|
||||
removeRepresentedLocalLiveProjection,
|
||||
@@ -1717,3 +1718,43 @@ describe('overlayConcurrentMessageChanges', () => {
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('preserveEquivalentTranscript', () => {
|
||||
it('keeps the current array BY REFERENCE when the replacement is content-equivalent', () => {
|
||||
// The exact warm-resume shape of #95595: fresh objects, identical content.
|
||||
const current = [msg('u-1', 'user', 'hello'), msg('a-1', 'assistant', 'const x = 1')]
|
||||
const freshObjects = current.map(message => ({ ...message, parts: [...message.parts] }))
|
||||
|
||||
const preserved = preserveEquivalentTranscript(current, freshObjects)
|
||||
|
||||
expect(preserved).toBe(current)
|
||||
expect(preserved[0]).toBe(current[0])
|
||||
})
|
||||
|
||||
it('keeps the current array when the arrays are the same reference', () => {
|
||||
const current = [msg('u-1', 'user', 'hello')]
|
||||
|
||||
expect(preserveEquivalentTranscript(current, current)).toBe(current)
|
||||
})
|
||||
|
||||
it('accepts the replacement when anything changed', () => {
|
||||
const current = [msg('u-1', 'user', 'hello')]
|
||||
const next = [msg('u-1', 'user', 'hello'), msg('a-1', 'assistant', 'new turn')]
|
||||
|
||||
expect(preserveEquivalentTranscript(current, next)).toBe(next)
|
||||
})
|
||||
|
||||
it('rejects the replacement when a message diverges in content', () => {
|
||||
const current = [msg('u-1', 'user', 'hello')]
|
||||
const next = [msg('u-1', 'user', 'hello world')]
|
||||
|
||||
expect(preserveEquivalentTranscript(current, next)).toBe(next)
|
||||
})
|
||||
|
||||
it('rejects the replacement when metadata a row renders diverges', () => {
|
||||
const current = [msg('u-1', 'user', 'hello')]
|
||||
const next = [msg('u-1', 'user', 'hello', { pending: true })]
|
||||
|
||||
expect(preserveEquivalentTranscript(current, next)).toBe(next)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -298,6 +298,23 @@ export function chatMessageArraysEquivalent(a: ChatMessage[], b: ChatMessage[]):
|
||||
return a.length === b.length && a.every((message, index) => chatMessagesEquivalent(message, b[index]))
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep the CURRENT array when the replacement is content-equivalent.
|
||||
*
|
||||
* The resume reconcilers create fresh `ChatMessage` objects via
|
||||
* `toChatMessages` even when nothing changed. Publishing those unconditionally
|
||||
* replaces the `$messages`/session-slice array with a new reference of fresh
|
||||
* objects — and because `useRuntimeMessageRepository` keys its normalization
|
||||
* cache (and React keys its rows) by object identity, every message in the
|
||||
* window re-normalizes and remounts: full markdown re-parse + shiki
|
||||
* re-highlight per row, on the main thread, per warm session switch (#95595).
|
||||
* Returning `current` when the content is equivalent keeps array AND object
|
||||
* identity, so the warm switch is O(1) paint.
|
||||
*/
|
||||
export function preserveEquivalentTranscript(current: ChatMessage[], next: ChatMessage[]): ChatMessage[] {
|
||||
return chatMessageArraysEquivalent(current, next) ? current : next
|
||||
}
|
||||
|
||||
export function reconcileResumeMessages(nextMessages: ChatMessage[], previousMessages: ChatMessage[]): ChatMessage[] {
|
||||
if (!previousMessages.length) {
|
||||
return nextMessages
|
||||
|
||||
@@ -45,6 +45,7 @@ import {
|
||||
import { MemoryConnect } from './memory/connect'
|
||||
import { ProviderConfigPanel } from './memory/provider-config-panel'
|
||||
import { ModelSettings, ModelSettingsSkeleton } from './model-settings'
|
||||
import { PoolLimitsSetting } from './pool-limits-setting'
|
||||
import { EmptyState, ListRow, SettingsContent, SettingsSkeleton, ToggleRow } from './primitives'
|
||||
import { SettingsProfileScope } from './profile-scope'
|
||||
import { QuickEntrySettings } from './quick-entry-settings'
|
||||
@@ -405,6 +406,7 @@ function ConfigSettingsInner({
|
||||
label={c.disableF12Title}
|
||||
onChange={setDisableF12}
|
||||
/>
|
||||
<PoolLimitsSetting />
|
||||
<QuickEntrySettings />
|
||||
</>
|
||||
)}
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import { useStore } from '@nanostores/react'
|
||||
import { useEffect, useState } from 'react'
|
||||
|
||||
import { ListRow } from '@/app/settings/primitives'
|
||||
import { Input } from '@/components/ui/input'
|
||||
import { $poolLimits, loadPoolLimits, savePoolLimits } from '@/store/pool-limits'
|
||||
|
||||
// Bounds imported from main's clamp module so the advertised input ranges
|
||||
// can never drift from what the pool actually enforces (review note on #92581).
|
||||
import { POOL_LIMITS_BOUNDS } from '../../../electron/pool-limits'
|
||||
|
||||
const MAX_BACKENDS_MAX = POOL_LIMITS_BOUNDS.maxBackendsMax
|
||||
const IDLE_MS_MAX = POOL_LIMITS_BOUNDS.idleMsMax
|
||||
|
||||
/** Settings → Advanced: warm-bot-backends count + backend idle timeout.
|
||||
* Device-local (not profile-scoped): the pool is sized once per machine and
|
||||
* changes apply live — main evicts/reaps to converge without a restart. */
|
||||
export function PoolLimitsSetting() {
|
||||
const limits = useStore($poolLimits)
|
||||
const [maxDraft, setMaxDraft] = useState(String(limits.maxBackends))
|
||||
const [idleDraft, setIdleDraft] = useState(String(limits.idleMs))
|
||||
|
||||
useEffect(() => {
|
||||
void loadPoolLimits()
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
setMaxDraft(String(limits.maxBackends))
|
||||
setIdleDraft(String(limits.idleMs))
|
||||
}, [limits])
|
||||
|
||||
const commitMax = () => {
|
||||
const parsed = Number(maxDraft)
|
||||
|
||||
if (!Number.isFinite(parsed) || parsed === limits.maxBackends) {
|
||||
setMaxDraft(String(limits.maxBackends))
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
void savePoolLimits({ maxBackends: parsed })
|
||||
.then(() => undefined)
|
||||
.catch(() => setMaxDraft(String($poolLimits.get().maxBackends)))
|
||||
}
|
||||
|
||||
const commitIdle = () => {
|
||||
const parsed = Number(idleDraft)
|
||||
|
||||
if (!Number.isFinite(parsed) || parsed === limits.idleMs) {
|
||||
setIdleDraft(String(limits.idleMs))
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
void savePoolLimits({ idleMs: parsed })
|
||||
.then(() => undefined)
|
||||
.catch(() => setIdleDraft(String($poolLimits.get().idleMs)))
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<ListRow
|
||||
action={
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
aria-label="Warm bot backends"
|
||||
className="w-20"
|
||||
inputMode="numeric"
|
||||
max={MAX_BACKENDS_MAX}
|
||||
min={1}
|
||||
onBlur={commitMax}
|
||||
onChange={event => setMaxDraft(event.target.value)}
|
||||
onKeyDown={event => {
|
||||
if (event.key === 'Enter') {
|
||||
event.currentTarget.blur()
|
||||
}
|
||||
}}
|
||||
type="number"
|
||||
value={maxDraft}
|
||||
/>
|
||||
</div>
|
||||
}
|
||||
description="How many bot backends stay running for instant switching. Higher = faster switches, more memory (~60MB per backend). Applies immediately."
|
||||
title="Warm Bot Backends"
|
||||
/>
|
||||
<ListRow
|
||||
action={
|
||||
<div className="flex items-center gap-2">
|
||||
<Input
|
||||
aria-label="Backend idle timeout in milliseconds"
|
||||
className="w-28"
|
||||
inputMode="numeric"
|
||||
max={IDLE_MS_MAX}
|
||||
min={60_000}
|
||||
onBlur={commitIdle}
|
||||
onChange={event => setIdleDraft(event.target.value)}
|
||||
onKeyDown={event => {
|
||||
if (event.key === 'Enter') {
|
||||
event.currentTarget.blur()
|
||||
}
|
||||
}}
|
||||
type="number"
|
||||
value={idleDraft}
|
||||
/>
|
||||
<span className="text-[length:var(--conversation-caption-font-size)] text-(--ui-text-tertiary)">ms</span>
|
||||
</div>
|
||||
}
|
||||
description="How long an unused bot backend stays warm before it is shut down. Raise this so bots you revisit every few minutes never pay a cold start."
|
||||
title="Backend Idle Timeout"
|
||||
/>
|
||||
</>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
import { render } from '@testing-library/react'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
/**
|
||||
* Issue #95595 proposed-fix #3: the `messageComponents` map handed to
|
||||
* ThreadMessageList must keep its REFERENCE IDENTITY across a session switch.
|
||||
* If it re-minted, React would unmount/remount every visible message — async
|
||||
* re-rendered parts (shiki code blocks) collapse and re-expand, and the whole
|
||||
* thread visibly jumps on every tab switch.
|
||||
*
|
||||
* The memo deps are deliberately only the boolean "definedness" gates (the
|
||||
* callbacks themselves reach the composer through a ref), so a plain switch
|
||||
* — sessionId changing, callbacks unchanged — must not change the map.
|
||||
*/
|
||||
let lastComponents: unknown
|
||||
|
||||
vi.mock('@/components/assistant-ui/thread/list', () => ({
|
||||
ThreadMessageList: (props: { components: unknown }) => {
|
||||
lastComponents = props.components
|
||||
|
||||
return null
|
||||
}
|
||||
}))
|
||||
|
||||
vi.mock('@/components/assistant-ui/thread/timeline', () => ({
|
||||
ThreadTimeline: () => null
|
||||
}))
|
||||
|
||||
vi.mock('@/components/assistant-ui/thread/status', () => ({
|
||||
BackgroundResumeNotice: () => null,
|
||||
CenteredThreadSpinner: () => null
|
||||
}))
|
||||
|
||||
vi.mock('@/i18n', () => ({
|
||||
useI18n: () => ({
|
||||
t: {
|
||||
assistant: {
|
||||
thread: {
|
||||
restoreBody: 'restore body',
|
||||
restoreConfirm: 'Restore',
|
||||
restoreTitle: 'Restore this turn?'
|
||||
}
|
||||
},
|
||||
common: {
|
||||
cancel: 'Cancel',
|
||||
confirm: 'Confirm',
|
||||
done: 'Done',
|
||||
loading: 'Loading'
|
||||
}
|
||||
}
|
||||
})
|
||||
}))
|
||||
|
||||
import { Thread } from './index'
|
||||
|
||||
describe('Thread messageComponents identity across session switches', () => {
|
||||
it('does not re-mint messageComponents when only the session changes', () => {
|
||||
const { rerender } = render(<Thread sessionId="session-a" />)
|
||||
const first = lastComponents
|
||||
|
||||
expect(first).toBeDefined()
|
||||
|
||||
rerender(<Thread sessionId="session-b" />)
|
||||
|
||||
// THE guard: a switch must keep the component map reference, so the
|
||||
// incoming transcript reconciles instead of remounting.
|
||||
expect(lastComponents).toBe(first)
|
||||
|
||||
rerender(<Thread sessionId="session-c" />)
|
||||
|
||||
expect(lastComponents).toBe(first)
|
||||
})
|
||||
|
||||
it('keeps the map stable across a plain parent re-render', () => {
|
||||
const { rerender } = render(<Thread sessionId="session-a" />)
|
||||
const first = lastComponents
|
||||
|
||||
rerender(<Thread sessionId="session-a" />)
|
||||
|
||||
expect(lastComponents).toBe(first)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,122 @@
|
||||
import { cleanup, render, screen } from '@testing-library/react'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
/**
|
||||
* Perf regression guard for #95595: switching to a warm session remounts the
|
||||
* incoming transcript, and every mounted code block used to re-tokenize from
|
||||
* scratch on the main thread. The content-keyed cache must make a remount of
|
||||
* an unchanged block a cache hit — ZERO highlighter calls.
|
||||
*
|
||||
* shiki itself is mocked (jsdom cannot run the oniguruma wasm engine); the
|
||||
* mock counts `codeToHtml` invocations, which is the cost we are guarding.
|
||||
*/
|
||||
const { codeToHtml } = vi.hoisted(() => ({
|
||||
codeToHtml: vi.fn((code: string) => {
|
||||
const escaped = String(code).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
|
||||
|
||||
return `<pre class="shiki"><code>${escaped}</code></pre>`
|
||||
})
|
||||
}))
|
||||
|
||||
vi.mock('shiki', () => ({
|
||||
bundledLanguages: { typescript: 'typescript-loader', text: 'text-loader' },
|
||||
getSingletonHighlighter: vi.fn(async () => ({
|
||||
codeToHtml: (code: string) => codeToHtml(code),
|
||||
getLoadedLanguages: () => ['text', 'typescript'],
|
||||
loadLanguage: vi.fn(async () => undefined)
|
||||
}))
|
||||
}))
|
||||
|
||||
vi.mock('shiki/engine/oniguruma', () => ({
|
||||
createOnigurumaEngine: vi.fn(() => ({}) as never)
|
||||
}))
|
||||
|
||||
import CachedShikiBlock from '@/components/chat/shiki-block'
|
||||
import { highlightCache } from '@/components/chat/shiki-highlight-cache'
|
||||
|
||||
const TS_BLOCK = { language: 'typescript', code: 'const answer: number = 42\n' }
|
||||
|
||||
async function waitForHighlighted(): Promise<void> {
|
||||
await screen.findByTestId('shiki-container', undefined, { timeout: 2_000 })
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
codeToHtml.mockClear()
|
||||
highlightCache.clear()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
cleanup()
|
||||
})
|
||||
|
||||
describe('CachedShikiBlock (warm-switch perf guard)', () => {
|
||||
it('highlights on first mount and reuses the cached HTML on remount', async () => {
|
||||
const { unmount } = render(<CachedShikiBlock {...TS_BLOCK} />)
|
||||
await waitForHighlighted()
|
||||
|
||||
expect(codeToHtml).toHaveBeenCalledTimes(1)
|
||||
expect(screen.getByTestId('shiki-container').innerHTML).toContain('const answer')
|
||||
|
||||
// Warm session switch: the row unmounts and the SAME block mounts again.
|
||||
unmount()
|
||||
render(<CachedShikiBlock {...TS_BLOCK} />)
|
||||
await waitForHighlighted()
|
||||
|
||||
// The guard: remounting an unchanged block must NOT re-tokenize.
|
||||
expect(codeToHtml).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('re-highlights a block whose code changed (cache miss)', async () => {
|
||||
const { unmount } = render(<CachedShikiBlock {...TS_BLOCK} />)
|
||||
await waitForHighlighted()
|
||||
|
||||
unmount()
|
||||
render(<CachedShikiBlock code="const other = true\n" language="typescript" />)
|
||||
await waitForHighlighted()
|
||||
|
||||
expect(codeToHtml).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('keeps blocks independent: N blocks highlight N times across two mounts', async () => {
|
||||
const { unmount } = render(
|
||||
<>
|
||||
<CachedShikiBlock {...TS_BLOCK} />
|
||||
<CachedShikiBlock code="function two(): void {}\n" language="typescript" />
|
||||
</>
|
||||
)
|
||||
|
||||
await screen.findAllByTestId('shiki-container', undefined, { timeout: 2_000 })
|
||||
|
||||
expect(codeToHtml).toHaveBeenCalledTimes(2)
|
||||
|
||||
unmount()
|
||||
render(
|
||||
<>
|
||||
<CachedShikiBlock {...TS_BLOCK} />
|
||||
<CachedShikiBlock code="function two(): void {}\n" language="typescript" />
|
||||
</>
|
||||
)
|
||||
await screen.findAllByTestId('shiki-container', undefined, { timeout: 2_000 })
|
||||
|
||||
// Two mounts of the same two blocks: exactly two tokenizations total.
|
||||
expect(codeToHtml).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('does not cache a failed highlight, so a retry can succeed', async () => {
|
||||
codeToHtml.mockRejectedValueOnce(new Error('boom'))
|
||||
|
||||
const { unmount } = render(<CachedShikiBlock {...TS_BLOCK} />)
|
||||
await waitForHighlighted()
|
||||
|
||||
// The failure degrades to escaped plain text (and is NOT cached).
|
||||
expect(screen.getByTestId('shiki-container').innerHTML).toContain('const answer')
|
||||
expect(codeToHtml).toHaveBeenCalledTimes(1)
|
||||
|
||||
unmount()
|
||||
render(<CachedShikiBlock {...TS_BLOCK} />)
|
||||
await waitForHighlighted()
|
||||
|
||||
// Second mount tries the highlighter again instead of serving stale HTML.
|
||||
expect(codeToHtml).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
})
|
||||
@@ -1,15 +1,173 @@
|
||||
'use client'
|
||||
|
||||
/**
|
||||
* The ONLY static importer of `react-shiki` (and through it the multi-MB
|
||||
* shiki language/theme bundle). Every consumer reaches this module through
|
||||
* The ONLY static importer of shiki (and through it the multi-MB shiki
|
||||
* language/theme/wasm bundle). Every consumer reaches this module through
|
||||
* `React.lazy(() => import('./shiki-block'))` — see `LazyShiki` in
|
||||
* shiki-highlighter.tsx — so the shiki chunk stays entirely off the
|
||||
* cold-start path and loads on the first highlighted code block instead.
|
||||
*
|
||||
* Do NOT import this module statically from anything the entry graph
|
||||
* reaches, or the chunk moves back into boot.
|
||||
*
|
||||
* Unlike the previous pass-through of `react-shiki`'s component, this module
|
||||
* is cache-aware: highlighted output is stored in a module-level LRU cache
|
||||
* keyed by (theme scope, language, code), so a REMOUNT of an unchanged code
|
||||
* block (the warm-session-switch path of #95595) paints the cached HTML
|
||||
* synchronously and never re-tokenizes. Only cache misses run shiki, and
|
||||
* misses are debounced so a streaming block settles before the heavy work
|
||||
* starts.
|
||||
*/
|
||||
import ShikiHighlighter from 'react-shiki'
|
||||
import { useEffect, useMemo, useState } from 'react'
|
||||
import { bundledLanguages, getSingletonHighlighter } from 'shiki'
|
||||
import type { BundledLanguage, BundledTheme, Highlighter } from 'shiki'
|
||||
import { createOnigurumaEngine } from 'shiki/engine/oniguruma'
|
||||
|
||||
export default ShikiHighlighter
|
||||
import { SHIKI_HIGHLIGHT_SCOPE, SHIKI_THEME } from '@/components/chat/shiki-config'
|
||||
import { highlightCache, highlightCacheKey } from '@/components/chat/shiki-highlight-cache'
|
||||
|
||||
/** Same debounce react-shiki's `delay` used to throttle highlight work with. */
|
||||
const HIGHLIGHT_DELAY_MS = 120
|
||||
|
||||
// Stable identity for "no color replacements" so the memo/effect deps below
|
||||
// never churn on renders that don't pass the prop.
|
||||
const NO_COLOR_REPLACEMENTS: Record<string, Record<string, string>> = {}
|
||||
|
||||
export interface CachedShikiBlockProps {
|
||||
language: string
|
||||
code: string
|
||||
/** Theme override; defaults to the shared SHIKI_THEME. */
|
||||
theme?: { dark: string; light: string }
|
||||
/** Color replacements; defaults to none (the chat passes its own). */
|
||||
colorReplacements?: Record<string, Record<string, string>>
|
||||
}
|
||||
|
||||
function isLoadableLanguage(language: string): boolean {
|
||||
return language === 'text' || language in bundledLanguages
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache scope for one theme configuration. Part of the cache key: a block
|
||||
* highlighted under a different theme is a different render.
|
||||
*/
|
||||
function highlightScope(
|
||||
theme: { dark: string; light: string },
|
||||
colorReplacements: Record<string, Record<string, string>>
|
||||
): string {
|
||||
return `${SHIKI_HIGHLIGHT_SCOPE}:${theme.dark}:${theme.light}:${JSON.stringify(colorReplacements)}`
|
||||
}
|
||||
|
||||
let highlighterPromise: Promise<Highlighter> | null = null
|
||||
let loadedThemes = new Set<string>([SHIKI_THEME.dark, SHIKI_THEME.light])
|
||||
|
||||
/**
|
||||
* Lazily-created shiki singleton, mirroring react-shiki's full bundle: only
|
||||
* the languages actually seen are loaded into it (the singleton is created
|
||||
* with the first block's language, later ones are `loadLanguage`d on demand;
|
||||
* override themes are `loadTheme`d the same way). Unknown languages are left
|
||||
* unloaded and fall through to shiki's plain-text handling, as before.
|
||||
*/
|
||||
async function highlightToHtml(
|
||||
language: string,
|
||||
code: string,
|
||||
theme: { dark: string; light: string },
|
||||
colorReplacements: Record<string, Record<string, string>>
|
||||
): Promise<string> {
|
||||
if (!highlighterPromise) {
|
||||
highlighterPromise = getSingletonHighlighter({
|
||||
// Only bundled languages are ever passed here (isLoadableLanguage
|
||||
// guards both call sites), so the cast is safe.
|
||||
langs: isLoadableLanguage(language) ? [language as BundledLanguage] : [],
|
||||
themes: [SHIKI_THEME.dark, SHIKI_THEME.light],
|
||||
engine: createOnigurumaEngine(import('shiki/wasm'))
|
||||
})
|
||||
}
|
||||
|
||||
const highlighter = await highlighterPromise
|
||||
|
||||
if (isLoadableLanguage(language) && !highlighter.getLoadedLanguages().includes(language)) {
|
||||
await highlighter.loadLanguage(language as BundledLanguage)
|
||||
}
|
||||
|
||||
const missingThemes = [theme.dark, theme.light].filter(name => !loadedThemes.has(name))
|
||||
|
||||
if (missingThemes.length > 0) {
|
||||
await highlighter.loadTheme(...(missingThemes as BundledTheme[]))
|
||||
missingThemes.forEach(name => loadedThemes.add(name))
|
||||
}
|
||||
|
||||
return highlighter.codeToHtml(code, {
|
||||
lang: language,
|
||||
themes: { dark: theme.dark, light: theme.light },
|
||||
defaultColor: 'light-dark()',
|
||||
colorReplacements
|
||||
})
|
||||
}
|
||||
|
||||
function escapeHtml(text: string): string {
|
||||
return text.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"')
|
||||
}
|
||||
|
||||
/** Never let a highlight failure blank a block — degrade to escaped plain text. */
|
||||
function plainTextHtml(code: string): string {
|
||||
return `<pre class="shiki" style="background-color:transparent;margin:0"><code>${escapeHtml(code)}</code></pre>`
|
||||
}
|
||||
|
||||
export default function CachedShikiBlock({ language, code, theme, colorReplacements }: CachedShikiBlockProps) {
|
||||
const themeConfig = theme ?? SHIKI_THEME
|
||||
const replacements = colorReplacements ?? NO_COLOR_REPLACEMENTS
|
||||
|
||||
const cacheKey = useMemo(
|
||||
() => highlightCacheKey(highlightScope(themeConfig, replacements), language, code),
|
||||
[language, code, replacements, themeConfig]
|
||||
)
|
||||
|
||||
const [html, setHtml] = useState<string | null>(() => highlightCache.get(cacheKey) ?? null)
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false
|
||||
|
||||
// Cache hit — no highlighter work at all. This is the warm-switch path:
|
||||
// the previous visit already rendered this block, so paint it again.
|
||||
const cached = highlightCache.get(cacheKey)
|
||||
|
||||
if (cached !== undefined) {
|
||||
setHtml(cached)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
const timer = window.setTimeout(() => {
|
||||
highlightToHtml(language, code, themeConfig, replacements)
|
||||
.then(result => {
|
||||
if (cancelled) {
|
||||
return
|
||||
}
|
||||
|
||||
highlightCache.set(cacheKey, result)
|
||||
setHtml(result)
|
||||
})
|
||||
.catch(error => {
|
||||
if (cancelled) {
|
||||
return
|
||||
}
|
||||
|
||||
console.error('shiki highlight failed; rendering plain code', error)
|
||||
setHtml(plainTextHtml(code))
|
||||
})
|
||||
}, HIGHLIGHT_DELAY_MS)
|
||||
|
||||
return () => {
|
||||
cancelled = true
|
||||
window.clearTimeout(timer)
|
||||
}
|
||||
}, [cacheKey, code, language, replacements, themeConfig])
|
||||
|
||||
if (html === null) {
|
||||
// Nothing to paint yet (miss, debounce pending). Matches react-shiki's
|
||||
// own empty render while the highlight is in flight.
|
||||
return null
|
||||
}
|
||||
|
||||
return <div className="rs-root not-prose" dangerouslySetInnerHTML={{ __html: html }} data-testid="shiki-container" />
|
||||
}
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
// Shiki theme/color constants shared by the chat code-block renderer
|
||||
// (shiki-highlighter.tsx) and the lazy shiki chunk (shiki-block.tsx). Kept in
|
||||
// their own dependency-free module so the lazy chunk can import them without
|
||||
// pulling the main chat module (or react-shiki) into the shiki bundle.
|
||||
|
||||
// `github-dark-dimmed` is GitHub's lower-contrast dark palette — the vivid
|
||||
// `github-dark-default` tokens read harsh at our small code size. Shared by the
|
||||
// inline diff renderer too (see diff-lines.tsx) so code + diffs match.
|
||||
export const SHIKI_THEME = { dark: 'github-dark-dimmed', light: 'github-light-default' } as const
|
||||
|
||||
/**
|
||||
* `github-light-default` colors comments `#6e7781` (~4.2:1 against the code
|
||||
* card background) — borderline unreadable at our 11px code size, and worst of
|
||||
* all for shell snippets where a single `#` turns the rest of the line into one
|
||||
* long comment span. Remap light-mode comments to GitHub's darker muted gray
|
||||
* (`#57606a`, ~6.4:1). Dark mode (`#8b949e`, ~6.1:1) already reads fine, so we
|
||||
* leave it untouched. Keyed per theme name so the bump only applies in light.
|
||||
*/
|
||||
export const SHIKI_COLOR_REPLACEMENTS: Record<string, Record<string, string>> = {
|
||||
'github-light-default': { '#6e7781': '#57606a' }
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache-key scope for the content-addressed highlight cache. Bumping this
|
||||
* invalidates every cached highlight at once — bump it whenever the rendering
|
||||
* options (themes, color replacements) change, because keys are NOT allowed to
|
||||
* silently produce a different DOM than the one they were computed with.
|
||||
*/
|
||||
export const SHIKI_HIGHLIGHT_SCOPE = `hermes-shiki-v1:${JSON.stringify({
|
||||
dark: SHIKI_THEME.dark,
|
||||
light: SHIKI_THEME.light,
|
||||
colorReplacements: SHIKI_COLOR_REPLACEMENTS
|
||||
})}`
|
||||
@@ -0,0 +1,95 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import {
|
||||
HIGHLIGHT_CACHE_MAX_CHARS,
|
||||
HIGHLIGHT_CACHE_MAX_ENTRIES,
|
||||
HighlightCache,
|
||||
highlightCacheKey
|
||||
} from '@/components/chat/shiki-highlight-cache'
|
||||
|
||||
describe('highlightCacheKey', () => {
|
||||
it('separates scope, language and code so distinct blocks never collide', () => {
|
||||
const a = highlightCacheKey('scope-1', 'ts', 'const x = 1')
|
||||
const b = highlightCacheKey('scope-1', 'ts', 'const x = 2')
|
||||
|
||||
expect(a).not.toBe(b)
|
||||
expect(highlightCacheKey('scope-1', 'js', 'const x = 1')).not.toBe(a)
|
||||
expect(highlightCacheKey('scope-2', 'ts', 'const x = 1')).not.toBe(a)
|
||||
})
|
||||
})
|
||||
|
||||
describe('HighlightCache', () => {
|
||||
it('round-trips an entry and refreshes recency on get', () => {
|
||||
const cache = new HighlightCache(3, 1000)
|
||||
|
||||
cache.set('a', 'A')
|
||||
cache.set('b', 'B')
|
||||
cache.set('c', 'C')
|
||||
// Touch the oldest entry so it becomes the newest.
|
||||
expect(cache.get('a')).toBe('A')
|
||||
cache.set('d', 'D')
|
||||
|
||||
// 'b' is now the least recently used and must be evicted first.
|
||||
expect(cache.has('b')).toBe(false)
|
||||
expect(cache.get('a')).toBe('A')
|
||||
expect(cache.get('c')).toBe('C')
|
||||
expect(cache.get('d')).toBe('D')
|
||||
})
|
||||
|
||||
it('evicts oldest-first past the entry cap', () => {
|
||||
const cache = new HighlightCache(2, 1_000_000)
|
||||
|
||||
cache.set('a', 'A')
|
||||
cache.set('b', 'B')
|
||||
cache.set('c', 'C')
|
||||
|
||||
expect(cache.size).toBe(2)
|
||||
expect(cache.has('a')).toBe(false)
|
||||
expect(cache.has('b')).toBe(true)
|
||||
expect(cache.has('c')).toBe(true)
|
||||
})
|
||||
|
||||
it('evicts oldest-first past the total-char cap', () => {
|
||||
const cache = new HighlightCache(HIGHLIGHT_CACHE_MAX_ENTRIES, 10)
|
||||
|
||||
cache.set('a', '12345')
|
||||
cache.set('b', '123456')
|
||||
|
||||
expect(cache.size).toBe(1)
|
||||
expect(cache.has('a')).toBe(false)
|
||||
expect(cache.has('b')).toBe(true)
|
||||
expect(cache.totalChars).toBeLessThanOrEqual(10)
|
||||
})
|
||||
|
||||
it('replaces an existing key in place without double-counting chars', () => {
|
||||
const cache = new HighlightCache(2, 100)
|
||||
|
||||
cache.set('a', '12345')
|
||||
cache.set('a', '1234567890')
|
||||
|
||||
expect(cache.size).toBe(1)
|
||||
expect(cache.totalChars).toBe(10)
|
||||
})
|
||||
|
||||
it('stays within both caps for a large burst of unique blocks', () => {
|
||||
const cache = new HighlightCache(HIGHLIGHT_CACHE_MAX_ENTRIES, HIGHLIGHT_CACHE_MAX_CHARS)
|
||||
|
||||
for (let i = 0; i < 2_000; i++) {
|
||||
cache.set(`block-${i}`, `html-${i}`.repeat(100))
|
||||
}
|
||||
|
||||
expect(cache.size).toBeLessThanOrEqual(HIGHLIGHT_CACHE_MAX_ENTRIES)
|
||||
expect(cache.totalChars).toBeLessThanOrEqual(HIGHLIGHT_CACHE_MAX_CHARS)
|
||||
})
|
||||
|
||||
it('clear drops everything', () => {
|
||||
const cache = new HighlightCache()
|
||||
|
||||
cache.set('a', 'A')
|
||||
cache.clear()
|
||||
|
||||
expect(cache.size).toBe(0)
|
||||
expect(cache.totalChars).toBe(0)
|
||||
expect(cache.get('a')).toBeUndefined()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,90 @@
|
||||
// ── Content-addressed syntax-highlight cache (#95595) ────────────────────────
|
||||
// Switching to a warm session remounts the incoming transcript, and every
|
||||
// mounted code block used to be re-tokenized from scratch by shiki — N blocks
|
||||
// × full tokenization on the renderer main thread, on every switch, even
|
||||
// though the code had not changed. The fix is a module-level LRU cache keyed
|
||||
// by (scope, language, code) holding the final highlighted HTML, so a remount
|
||||
// of an unchanged block paints the cached markup synchronously and never
|
||||
// touches the highlighter.
|
||||
//
|
||||
// Bounds: shiki's tokenized HTML is ~5-10x the source size, so an unbounded
|
||||
// cache would leak renderer memory over a long session list. Cap both the
|
||||
// entry count and the total cached characters; evict oldest-first.
|
||||
//
|
||||
// This module is intentionally dependency-free (no React, no shiki) so the
|
||||
// cache logic can be unit-tested in isolation.
|
||||
|
||||
export const HIGHLIGHT_CACHE_MAX_ENTRIES = 512
|
||||
export const HIGHLIGHT_CACHE_MAX_CHARS = 6 * 1024 * 1024
|
||||
|
||||
/** Unique key for one highlighted block: scope + language + exact code. */
|
||||
export function highlightCacheKey(scope: string, language: string, code: string): string {
|
||||
return `${scope}\u0000${language}\u0000${code}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Bounded LRU map of highlight cache keys to rendered HTML. `get` refreshes
|
||||
* recency (Map insertion order is used as the LRU clock); `set` evicts the
|
||||
* oldest entries until both caps hold.
|
||||
*/
|
||||
export class HighlightCache {
|
||||
private readonly entries = new Map<string, string>()
|
||||
private chars = 0
|
||||
|
||||
constructor(
|
||||
private readonly maxEntries: number = HIGHLIGHT_CACHE_MAX_ENTRIES,
|
||||
private readonly maxChars: number = HIGHLIGHT_CACHE_MAX_CHARS
|
||||
) {}
|
||||
|
||||
get size(): number {
|
||||
return this.entries.size
|
||||
}
|
||||
|
||||
get totalChars(): number {
|
||||
return this.chars
|
||||
}
|
||||
|
||||
has(key: string): boolean {
|
||||
return this.entries.has(key)
|
||||
}
|
||||
|
||||
get(key: string): string | undefined {
|
||||
const value = this.entries.get(key)
|
||||
|
||||
if (value !== undefined) {
|
||||
// Refresh recency: re-inserting moves the entry to the newest position.
|
||||
this.entries.delete(key)
|
||||
this.entries.set(key, value)
|
||||
}
|
||||
|
||||
return value
|
||||
}
|
||||
|
||||
set(key: string, html: string): void {
|
||||
if (this.entries.has(key)) {
|
||||
this.chars -= this.entries.get(key)!.length
|
||||
this.entries.delete(key)
|
||||
}
|
||||
|
||||
this.entries.set(key, html)
|
||||
this.chars += html.length
|
||||
this.evict()
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
this.entries.clear()
|
||||
this.chars = 0
|
||||
}
|
||||
|
||||
private evict(): void {
|
||||
while ((this.entries.size > this.maxEntries || this.chars > this.maxChars) && this.entries.size > 0) {
|
||||
const oldestKey = this.entries.keys().next().value as string
|
||||
const oldest = this.entries.get(oldestKey)!
|
||||
this.entries.delete(oldestKey)
|
||||
this.chars -= oldest.length
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The renderer-wide highlight cache. Lives for the lifetime of the module. */
|
||||
export const highlightCache = new HighlightCache()
|
||||
@@ -1,15 +1,20 @@
|
||||
'use client'
|
||||
|
||||
import type { SyntaxHighlighterProps } from '@assistant-ui/react-streamdown'
|
||||
import { type ComponentProps, type FC, lazy, Suspense, useMemo } from 'react'
|
||||
import type ShikiHighlighter from 'react-shiki'
|
||||
import { type FC, lazy, Suspense, useMemo } from 'react'
|
||||
|
||||
import { CodeCard, CodeCardBody } from '@/components/chat/code-card'
|
||||
import { ExpandableBlock } from '@/components/chat/expandable-block'
|
||||
// Theme constants live in shiki-config (dependency-free) so the lazy shiki
|
||||
// chunk can import them without pulling this module into the shiki bundle.
|
||||
import { SHIKI_COLOR_REPLACEMENTS } from '@/components/chat/shiki-config'
|
||||
import { CopyButton } from '@/components/ui/copy-button'
|
||||
import { useI18n } from '@/i18n'
|
||||
import { isLikelyProseCodeBlock } from '@/lib/markdown-code'
|
||||
|
||||
import type { CachedShikiBlockProps } from './shiki-block'
|
||||
export { SHIKI_COLOR_REPLACEMENTS, SHIKI_THEME } from '@/components/chat/shiki-config'
|
||||
|
||||
/**
|
||||
* Streamdown's code adapter renders header + body as inline siblings, so we
|
||||
* own the wrapping `<CodeCard>` here and neutralize the upstream
|
||||
@@ -17,46 +22,34 @@ import { isLikelyProseCodeBlock } from '@/lib/markdown-code'
|
||||
* background-only — no header row, no language label — so a fence reads as a
|
||||
* tinted slab of the reply; copy is a hover-reveal control in the corner.
|
||||
*
|
||||
* `react-shiki` full bundle so all `bundledLanguages` work; theme switches
|
||||
* follow the document `color-scheme` via `defaultColor="light-dark()"`.
|
||||
* The heavy lifting lives in the lazy `shiki-block` chunk (full bundle so all
|
||||
* `bundledLanguages` work; theme switches follow the document `color-scheme`
|
||||
* via `defaultColor="light-dark()"`), and its output is cached by content so
|
||||
* warm-session switches never re-tokenize unchanged blocks (#95595).
|
||||
*/
|
||||
interface HermesSyntaxHighlighterProps extends SyntaxHighlighterProps {
|
||||
defer?: boolean
|
||||
}
|
||||
|
||||
// `github-dark-dimmed` is GitHub's lower-contrast dark palette — the vivid
|
||||
// `github-dark-default` tokens read harsh at our small code size. Shared by the
|
||||
// inline diff renderer too (see diff-lines.tsx) so code + diffs match.
|
||||
export const SHIKI_THEME = { dark: 'github-dark-dimmed', light: 'github-light-default' } as const
|
||||
|
||||
/**
|
||||
* `github-light-default` colors comments `#6e7781` (~4.2:1 against the code
|
||||
* card background) — borderline unreadable at our 11px code size, and worst of
|
||||
* all for shell snippets where a single `#` turns the rest of the line into one
|
||||
* long comment span. Remap light-mode comments to GitHub's darker muted gray
|
||||
* (`#57606a`, ~6.4:1). Dark mode (`#8b949e`, ~6.1:1) already reads fine, so we
|
||||
* leave it untouched. Keyed per theme name so the bump only applies in light.
|
||||
*/
|
||||
const SHIKI_COLOR_REPLACEMENTS: Record<string, Record<string, string>> = {
|
||||
'github-light-default': { '#6e7781': '#57606a' }
|
||||
}
|
||||
|
||||
const MAX_HIGHLIGHT_CHARS = 150_000
|
||||
const MAX_HIGHLIGHT_LINES = 3_000
|
||||
const CHUNK_LINES = 200
|
||||
const EST_LINE_PX = 16
|
||||
|
||||
// react-shiki (and through it the multi-MB shiki grammar/theme bundle) is the
|
||||
// shiki (and through it the multi-MB grammar/theme/wasm bundle) is the
|
||||
// heaviest dependency in the renderer. `shiki-block.tsx` is its only static
|
||||
// importer, so this lazy() is the single seam that keeps shiki out of the
|
||||
// entry chunk — it loads on the first highlighted code block, not at boot.
|
||||
// The lazy module is cache-aware (#95595): unchanged blocks paint from a
|
||||
// content-keyed cache instead of re-tokenizing on every mount.
|
||||
const ShikiBlock = lazy(() => import('./shiki-block'))
|
||||
|
||||
/** Drop-in ShikiHighlighter that suspends on first use and renders the code
|
||||
* as plain preformatted text until the shiki chunk arrives. */
|
||||
export const LazyShiki: FC<ComponentProps<typeof ShikiHighlighter>> = props => (
|
||||
<Suspense fallback={<PlainCode code={String(props.children ?? '')} />}>
|
||||
<ShikiBlock {...props} />
|
||||
/** Suspends on first use and renders the code as plain preformatted text
|
||||
* until the shiki chunk arrives. Highlighted output is cached by
|
||||
* (theme, language, code), so revisits never re-tokenize (#95595). */
|
||||
export const LazyShiki: FC<CachedShikiBlockProps> = ({ language, code, theme, colorReplacements }) => (
|
||||
<Suspense fallback={<PlainCode code={code} />}>
|
||||
<ShikiBlock code={code} colorReplacements={colorReplacements} language={language} theme={theme} />
|
||||
</Suspense>
|
||||
)
|
||||
|
||||
@@ -160,18 +153,7 @@ export const SyntaxHighlighter: FC<HermesSyntaxHighlighterProps> = ({
|
||||
{plain ? (
|
||||
<PlainCode code={trimmed} />
|
||||
) : (
|
||||
<LazyShiki
|
||||
addDefaultStyles={false}
|
||||
as="div"
|
||||
colorReplacements={SHIKI_COLOR_REPLACEMENTS}
|
||||
defaultColor="light-dark()"
|
||||
delay={120}
|
||||
language={language || 'text'}
|
||||
showLanguage={false}
|
||||
theme={SHIKI_THEME}
|
||||
>
|
||||
{trimmed}
|
||||
</LazyShiki>
|
||||
<LazyShiki code={trimmed} colorReplacements={SHIKI_COLOR_REPLACEMENTS} language={language || 'text'} />
|
||||
)}
|
||||
</Pre>
|
||||
</ExpandableBlock>
|
||||
|
||||
@@ -236,9 +236,9 @@ function NotificationItem({ notification }: { notification: AppNotification }) {
|
||||
notification.action?.onClick()
|
||||
dismissNotification(notification.id)
|
||||
}}
|
||||
size="xs"
|
||||
size="sm"
|
||||
type="button"
|
||||
variant="textStrong"
|
||||
variant="default"
|
||||
>
|
||||
{notification.action.label}
|
||||
</Button>
|
||||
|
||||
@@ -1040,6 +1040,13 @@ export function revealTreePane(paneId: string) {
|
||||
// Reveal beats a Close: un-dismiss and let adoption put the pane back.
|
||||
if ($dismissedPanes.get().has(paneId)) {
|
||||
setDismissed(paneId, false)
|
||||
}
|
||||
|
||||
// A layout replacement can omit a still-registered pane without dismissing
|
||||
// it. Reconcile that saved contribution before claiming to reveal it.
|
||||
const currentTree = $layoutTree.get()
|
||||
|
||||
if (currentTree && !findGroupOfPane(currentTree, paneId)) {
|
||||
adoptContributedPanes()
|
||||
}
|
||||
|
||||
@@ -1064,8 +1071,8 @@ export function revealTreePane(paneId: string) {
|
||||
|
||||
if (hiddenNow.has(paneId)) {
|
||||
setTreePaneHidden(paneId, false)
|
||||
|
||||
return
|
||||
// Reactive unhide preserves a visible sibling. Explicit reveal must also
|
||||
// front this pane and restore its group below.
|
||||
}
|
||||
|
||||
const tree = $layoutTree.get()
|
||||
|
||||
Vendored
+10
@@ -1,6 +1,8 @@
|
||||
import type { GatewayWsUrlResult } from '@hermes/shared'
|
||||
import type { TranslucencyState } from '@hermes/shared/translucency'
|
||||
|
||||
import type { PoolLimits } from '../electron/pool-limits'
|
||||
|
||||
import type { WakeIndicatorState } from './lib/wake-indicator'
|
||||
import type {
|
||||
PetOverlayBounds,
|
||||
@@ -46,6 +48,14 @@ declare global {
|
||||
// Keepalive: mark a pool profile backend as recently used so the idle
|
||||
// reaper spares it while its chat is active.
|
||||
touchBackend: (profile?: string | null) => Promise<{ ok: boolean }>
|
||||
// Pool sizing (Settings → Advanced): device-local, live-applied by the
|
||||
// main process. get resolves the limits currently in force; set applies
|
||||
// (and persists) new ones, evicting/reaping to converge immediately.
|
||||
getPoolLimits: () => Promise<PoolLimits>
|
||||
setPoolLimits: (limits: { maxBackends?: number; idleMs?: number }) => Promise<{
|
||||
ok: boolean
|
||||
limits: PoolLimits
|
||||
}>
|
||||
getGatewayWsUrl: (profile?: null | string) => Promise<GatewayWsUrlResult>
|
||||
// Open (or focus) a standalone OS window for a single chat session so
|
||||
// the user can work with multiple chats side by side. Returns ok:false
|
||||
|
||||
@@ -71,6 +71,24 @@ describe('a row click lands on the canonical chat, never a remembered side tab',
|
||||
})
|
||||
})
|
||||
|
||||
it('fronting an already-open Bot Chat refreshes its transcript in place', async () => {
|
||||
// The front is presentation-only: the pane keeps whatever transcript it
|
||||
// last painted, which can predate rows the bot wrote while the user was
|
||||
// elsewhere (a cron delivery, a teammate's message_agent, another bot's
|
||||
// turn). Fronting must force a registry open so forceResume re-pulls the
|
||||
// latest rows instead of leaving a stale snapshot until the next turn
|
||||
// (#99393 class; #95600 only covered the not-yet-open path).
|
||||
host.focusOpenWorkspaceSession = vi.fn((_key: string, _probe: unknown, only?: readonly string[]) =>
|
||||
only?.includes('bot-chat-tip') ? 'bot-chat-tip' : null
|
||||
) as never
|
||||
$selectedStoredSessionId.set('bot-chat-tip')
|
||||
|
||||
await expect(openRosterBot(canonicalBot)).resolves.toBe(true)
|
||||
|
||||
expect(openBotCanonicalChat).toHaveBeenCalledWith(canonicalBot, expect.any(Function))
|
||||
$selectedStoredSessionId.set(null)
|
||||
})
|
||||
|
||||
it('resolves the registry when only a side thread is open', async () => {
|
||||
// The shell would happily front 'side-thread' — the allowlist excludes it.
|
||||
host.focusOpenWorkspaceSession = vi.fn((_key: string, _probe: unknown, only?: readonly string[]) =>
|
||||
|
||||
@@ -253,14 +253,37 @@ describe('the mention middleware', () => {
|
||||
expect(hostMock.requestProfile).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('hands the agent the connection-qualified message_agent target', async () => {
|
||||
it('hands the agent a relay-resolvable canonical target', async () => {
|
||||
const { handler } = await contributions()
|
||||
const result = await handler({ text: 'ping @default-vera' })
|
||||
|
||||
expect(result.text).toMatch(/message_agent target: "default-vera@vera"/)
|
||||
// The UI alias ('default-vera') is not a relay identity: resolve_remote_target()
|
||||
// accepts only a roster row's handle/profile, optionally @connection-qualified.
|
||||
// The annotation must carry the canonical profile@connection form (#97678).
|
||||
expect(result.text).toMatch(/message_agent target: "default@vera"/)
|
||||
expect(result.text).not.toMatch(/message_agent target: "default-vera/)
|
||||
expect(result.text).toMatch(/on Vera/)
|
||||
})
|
||||
|
||||
it('annotates the resolvable handle for a local row whose UI alias differs', async () => {
|
||||
// The reporter's shape (#97678 / Discord video): the LOCAL twin carries
|
||||
// the 'default-this-device' alias when the remote gateway is active.
|
||||
// The local resolver only knows bare profile names / 'hermes'.
|
||||
const { handler } = await contributions({
|
||||
focused: 'ops',
|
||||
profiles: [
|
||||
{ connectionId: 'local', connectionKind: 'local', handle: 'default-this-device', name: 'default' },
|
||||
{ name: 'ops' }
|
||||
]
|
||||
})
|
||||
|
||||
const result = await handler({ text: 'ping @default-this-device' })
|
||||
|
||||
expect(result.text).toMatch(/@default-this-device = agent profile "default"/)
|
||||
expect(result.text).toMatch(/message_agent target: "hermes"/)
|
||||
expect(result.text).not.toMatch(/message_agent target: "default-this-device/)
|
||||
})
|
||||
|
||||
it('passes a draft with no mention straight through', async () => {
|
||||
const { handler } = await contributions()
|
||||
const draft = { text: 'no tags here' }
|
||||
|
||||
@@ -734,11 +734,23 @@ export default {
|
||||
botRosterMeta(bot, $botMeta.get())?.title || bot.ui_meta?.['hermes-bots']?.title || bot.title || ''
|
||||
).trim()
|
||||
|
||||
const target = bot.remoteSource && bot.connectionId ? `${handle}@${bot.connectionId}` : handle
|
||||
// message_agent only resolves canonical identities: the relay
|
||||
// matches a roster row's handle/profile (± @connection-id), the
|
||||
// local path a bare profile name or 'hermes'. botHandle() prefers
|
||||
// the row's source-qualified UI alias ('default-vera'), which
|
||||
// neither resolver accepts — annotate the canonical form instead.
|
||||
const target =
|
||||
bot.remoteSource && bot.connectionId ? `${bot.name}@${bot.connectionId}` : botHandle(bot.name)
|
||||
|
||||
// Local rows get the same annotation whenever their UI alias
|
||||
// ('default-this-device') differs from the resolvable handle —
|
||||
// otherwise the agent has only the alias to go on and the local
|
||||
// path rejects it the same way (#97678).
|
||||
const where = bot.remoteSource
|
||||
? ` — on ${bot.connectionLabel || bot.connectionId} (message_agent target: "${target}")`
|
||||
: ''
|
||||
: handle !== target
|
||||
? ` (message_agent target: "${target}")`
|
||||
: ''
|
||||
|
||||
return `@${handle} = agent profile "${bot.name}"${title ? ` ("${title}")` : ''}${where}`
|
||||
})
|
||||
|
||||
@@ -246,6 +246,14 @@ export async function openRosterBot(bot: RosterRow): Promise<boolean> {
|
||||
// the roster-activity refresh treat it exactly like a registry open.
|
||||
$openBotChat.set({ key, openedRegistryId: fronted.registryId, openedSessionId: fronted.storedSessionId })
|
||||
|
||||
// Fronting is presentation-only: the pane keeps whatever transcript it
|
||||
// last painted, which can predate rows the bot wrote while the user was
|
||||
// elsewhere (another bot's turn, a cron delivery, a teammate's
|
||||
// message_agent). Force a registry open so forceResume re-pulls the
|
||||
// latest transcript instead of leaving a stale snapshot until the next
|
||||
// user turn (#99393 class; #95600 only covered the not-yet-open path).
|
||||
refreshOpenBotChat(bot)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -1584,6 +1584,24 @@ export function reconnectSecondaryGateways({ forceOpenSockets = false }: { force
|
||||
}
|
||||
}
|
||||
|
||||
// How many non-primary backends currently hold an open socket. Hover-intent
|
||||
// prewarming consults this before spawning: a speculative spawn that pushes
|
||||
// the pool past its cap causes the Electron main to LRU-evict a warm backend
|
||||
// — often one the user is about to click — turning the prewarm into churn
|
||||
// (the #91545 evict/respawn cascade). The active gateway's backend is
|
||||
// primary-routed and never counts toward the pool cap.
|
||||
export function openSecondaryCount(): number {
|
||||
let count = 0
|
||||
|
||||
for (const entry of g.secondaries.values()) {
|
||||
if (isOpen(entry.gateway)) {
|
||||
count += 1
|
||||
}
|
||||
}
|
||||
|
||||
return count
|
||||
}
|
||||
|
||||
// Keep the idle reaper from killing a backend we still need: ping every live
|
||||
// secondary. The active one is pinged separately (touchActiveGatewayBackend).
|
||||
export function touchSecondaryGateways(): void {
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* Pool limits — how many bot backends may stay spawned, and how long an
|
||||
* unused one survives before it is shut down.
|
||||
*
|
||||
* A device-local preference (each machine trades RAM against switching
|
||||
* speed for itself). The MAIN process is authoritative: it owns the pool
|
||||
* and the persisted copy, and applies a new max immediately by evicting
|
||||
* least-recently-used idle backends — no restart. This store mirrors the
|
||||
* live values for the Settings rows and feeds prewarmProfileBackend's
|
||||
* saturation guard.
|
||||
*/
|
||||
|
||||
import { atom } from 'nanostores'
|
||||
|
||||
export interface PoolLimits {
|
||||
/** Max concurrently spawned non-primary profile backends. */
|
||||
maxBackends: number
|
||||
/** Idle lifetime of an unused pool backend, in milliseconds. */
|
||||
idleMs: number
|
||||
}
|
||||
|
||||
export const POOL_LIMITS_DEFAULTS: PoolLimits = {
|
||||
maxBackends: 3,
|
||||
idleMs: 10 * 60_000
|
||||
}
|
||||
|
||||
export const $poolLimits = atom<PoolLimits>({ ...POOL_LIMITS_DEFAULTS })
|
||||
|
||||
/** Seed from main's authoritative state once at startup; no-op without the
|
||||
* bridge (web/older builds just keep the defaults for the UI). */
|
||||
export async function loadPoolLimits(): Promise<void> {
|
||||
try {
|
||||
const limits = await window.hermesDesktop?.getPoolLimits?.()
|
||||
|
||||
if (limits) {
|
||||
$poolLimits.set(limits)
|
||||
}
|
||||
} catch {
|
||||
// Keep defaults — Settings rows still render and can retry on save.
|
||||
}
|
||||
}
|
||||
|
||||
/** Push new limits to main; adopt the post-clamp values it reports. */
|
||||
export async function savePoolLimits(next: { maxBackends?: number; idleMs?: number }): Promise<void> {
|
||||
const current = $poolLimits.get()
|
||||
|
||||
const optimistic: PoolLimits = {
|
||||
maxBackends: next.maxBackends ?? current.maxBackends,
|
||||
idleMs: next.idleMs ?? current.idleMs
|
||||
}
|
||||
|
||||
// Optimistic paint, then honest reconciliation with the clamped result.
|
||||
$poolLimits.set(optimistic)
|
||||
|
||||
try {
|
||||
const result = await window.hermesDesktop?.setPoolLimits?.(next)
|
||||
|
||||
if (result?.limits) {
|
||||
$poolLimits.set(result.limits)
|
||||
}
|
||||
} catch {
|
||||
$poolLimits.set(current)
|
||||
throw new Error('Applying pool limits failed')
|
||||
}
|
||||
}
|
||||
@@ -9,10 +9,24 @@ import type { ProfileInfo } from '@/types/hermes'
|
||||
const ensureGatewayForProfile = vi.fn(async () => undefined)
|
||||
const ensureGatewayForAgent = vi.fn(async () => undefined)
|
||||
const openGatewayForProfile = vi.fn(async (_profile: string) => undefined)
|
||||
const openSecondaryCount = vi.fn(() => 0)
|
||||
const $gateway = atom<unknown>({ id: 'live-socket', connectionState: 'open' })
|
||||
const resetStarmapGraph = vi.fn()
|
||||
|
||||
vi.mock('@/store/gateway', () => ({ $gateway, ensureGatewayForAgent, ensureGatewayForProfile, openGatewayForProfile }))
|
||||
vi.mock('@/store/gateway', () => ({
|
||||
$gateway,
|
||||
ensureGatewayForAgent,
|
||||
ensureGatewayForProfile,
|
||||
openGatewayForProfile,
|
||||
openSecondaryCount
|
||||
}))
|
||||
// The pool-limits atom is profile.ts's live saturation signal — keep the real
|
||||
// one so tests can move the cap via the store, but stub its IPC bridge.
|
||||
vi.mock('@/store/pool-limits', async () => {
|
||||
const { atom } = await import('nanostores')
|
||||
|
||||
return { $poolLimits: atom({ idleMs: 600_000, maxBackends: 3 }) }
|
||||
})
|
||||
vi.mock('@/hermes', () => ({
|
||||
getProfiles: vi.fn(async () => ({ profiles: [] })),
|
||||
setApiRequestProfile: vi.fn()
|
||||
@@ -29,6 +43,8 @@ const {
|
||||
refreshProfiles
|
||||
} = await import('./profile')
|
||||
|
||||
const { $poolLimits } = await import('@/store/pool-limits')
|
||||
|
||||
const { $connection } = await import('./session')
|
||||
const { invalidateProfileScopedQueries } = await import('@/lib/query-client')
|
||||
const { getProfiles } = await import('@/hermes')
|
||||
@@ -55,6 +71,7 @@ beforeEach(() => {
|
||||
getConnection.mockReset()
|
||||
ensureGatewayForProfile.mockClear()
|
||||
openGatewayForProfile.mockClear()
|
||||
openSecondaryCount.mockReturnValue(0)
|
||||
$gateway.set({ id: 'live-socket', connectionState: 'open' })
|
||||
$activeGatewayProfile.set('default')
|
||||
$connection.set(localConn())
|
||||
@@ -169,6 +186,43 @@ describe('prewarmProfileBackend (hover-intent pool spawn)', () => {
|
||||
|
||||
expect(() => prewarmProfileBackend('warm-failing')).not.toThrow()
|
||||
})
|
||||
|
||||
it('skips pre-warm when the pool is saturated (#91545 evict/respawn cascade)', () => {
|
||||
// Every pool slot occupied: a speculative spawn would LRU-evict a warm
|
||||
// backend — often the one the user is about to click. Default limit 3,
|
||||
// 3 open secondaries → the next spawn would exceed the cap.
|
||||
openSecondaryCount.mockReturnValue(3)
|
||||
|
||||
prewarmProfileBackend('warm-saturated')
|
||||
|
||||
expect(openGatewayForProfile).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('pre-warms while pool slots are free', () => {
|
||||
openSecondaryCount.mockReturnValue(1)
|
||||
|
||||
prewarmProfileBackend('warm-slot-free')
|
||||
|
||||
expect(openGatewayForProfile).toHaveBeenCalledWith('warm-slot-free')
|
||||
})
|
||||
|
||||
it('follows the live pool-limit atom, not a hard-coded cap', () => {
|
||||
// User raises Warm Bot Backends to 8 in Settings: prewarming must keep
|
||||
// working well past the old default of 3.
|
||||
openSecondaryCount.mockReturnValue(5)
|
||||
$poolLimits.set({ idleMs: 600_000, maxBackends: 8 })
|
||||
|
||||
prewarmProfileBackend('warm-raised-cap')
|
||||
|
||||
expect(openGatewayForProfile).toHaveBeenCalledWith('warm-raised-cap')
|
||||
|
||||
// And lowering the cap re-engages the guard at the new boundary.
|
||||
$poolLimits.set({ idleMs: 600_000, maxBackends: 2 })
|
||||
|
||||
prewarmProfileBackend('warm-lowered-cap')
|
||||
|
||||
expect(openGatewayForProfile).not.toHaveBeenCalledWith('warm-lowered-cap')
|
||||
})
|
||||
})
|
||||
|
||||
describe('refreshProfiles shared rail list (#49289)', () => {
|
||||
|
||||
@@ -21,9 +21,11 @@ import {
|
||||
ensureGatewayForAgent,
|
||||
ensureGatewayForProfile,
|
||||
openGatewayForAgent,
|
||||
openGatewayForProfile
|
||||
openGatewayForProfile,
|
||||
openSecondaryCount
|
||||
} from '@/store/gateway'
|
||||
import { notifyError } from '@/store/notifications'
|
||||
import { $poolLimits } from '@/store/pool-limits'
|
||||
import { notifyRemoteOverrideAuthFailure } from '@/store/profile-remote-override'
|
||||
import { clearComposerSelectionOwner, setComposerSelectionOwner, setConnection } from '@/store/session'
|
||||
import type { SessionOwnerRoute } from '@/store/session-request-router'
|
||||
@@ -423,6 +425,17 @@ export function prewarmProfileBackend(name: string): void {
|
||||
return
|
||||
}
|
||||
|
||||
// Prewarm/cap harmony (#91545): the pool caps spawned backends at the
|
||||
// configured max, and a spawn over the cap LRU-evicts the warmest idle
|
||||
// backend. A hover sweep across the rail therefore evicted backends for
|
||||
// profiles the user was about to click — prewarming caused the exact churn
|
||||
// it exists to prevent. Skip speculative spawns once every pool slot is
|
||||
// occupied by an open socket; the real click still spawns on demand, it
|
||||
// just doesn't get a head start.
|
||||
if (openSecondaryCount() + 1 > $poolLimits.get().maxBackends) {
|
||||
return
|
||||
}
|
||||
|
||||
prewarmedAt.set(key, now)
|
||||
openGatewayForProfile(key).catch(() => undefined)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
async function setup() {
|
||||
const tree = await import('@/components/pane-shell/tree/store')
|
||||
const model = await import('@/components/pane-shell/tree/model')
|
||||
const { registry } = await import('@/contrib/registry')
|
||||
const session = await import('@/store/session')
|
||||
const states = await import('@/store/session-states')
|
||||
const { paneMirror } = await import('@/app/chat/pane-mirror')
|
||||
|
||||
registry.register({
|
||||
area: 'panes',
|
||||
data: { placement: 'main', uncloseable: true },
|
||||
id: 'workspace',
|
||||
render: () => null,
|
||||
title: 'Chat'
|
||||
})
|
||||
tree.declareDefaultTree(model.group(['workspace'], { active: 'workspace', id: 'main' }))
|
||||
tree.watchContributedPanes()
|
||||
paneMirror({
|
||||
source: states.$sessionTiles,
|
||||
key: tile => tile.storedSessionId,
|
||||
prefix: 'session-tile',
|
||||
dir: () => 'center',
|
||||
minWidth: '20rem',
|
||||
title: id => id,
|
||||
render: () => null,
|
||||
close: states.closeSessionTile
|
||||
})()
|
||||
session.$selectedStoredSessionId.set('previous-chat')
|
||||
|
||||
const scope = {
|
||||
ownerRoute: { connectionId: 'remote-a', mode: 'remote' as const, profile: 'writer' },
|
||||
workspaceMode: 'bots' as const,
|
||||
workspaceOwnerKey: 'remote-a::writer',
|
||||
workspaceTabTitle: 'Bot Chat'
|
||||
}
|
||||
|
||||
states.openSessionTile('canonical-chat', 'center', 'workspace', undefined, scope)
|
||||
|
||||
return { model, scope, session, states, tree }
|
||||
}
|
||||
|
||||
describe('focusing a saved Bot Chat requires a visible pane', () => {
|
||||
let ctx: Awaited<ReturnType<typeof setup>>
|
||||
const paneId = 'session-tile:canonical-chat'
|
||||
|
||||
beforeEach(async () => {
|
||||
window.localStorage.clear()
|
||||
vi.resetModules()
|
||||
ctx = await setup()
|
||||
})
|
||||
|
||||
it('re-adopts a saved tab after a profile overlay replaces the layout', async () => {
|
||||
const { applyDesktopOverlay } = await import('@/store/profile-share')
|
||||
const { model, scope, states, tree } = ctx
|
||||
const saved = states.$sessionTiles.get()
|
||||
applyDesktopOverlay('imported-profile', {
|
||||
version: 1,
|
||||
layoutTree: model.group(['workspace'], { active: 'workspace', id: 'imported-main' })
|
||||
})
|
||||
expect(model.findGroupOfPane(tree.$layoutTree.get()!, paneId)).toBeNull()
|
||||
|
||||
expect(states.focusWorkspaceOwnerSessionTile(scope.workspaceOwnerKey, undefined, ['canonical-chat'])).toBe(
|
||||
'canonical-chat'
|
||||
)
|
||||
expect(tree.isPaneVisible(paneId)).toBe(true)
|
||||
expect(tree.$activeTreeGroup.get()).toBe('imported-main')
|
||||
expect(states.$sessionTiles.get()).toEqual(saved)
|
||||
expect(states.sessionTileOwnerRoute('canonical-chat')).toEqual(scope.ownerRoute)
|
||||
})
|
||||
|
||||
it('reports a miss through both helpers if the layout cannot place the saved tab', () => {
|
||||
const { scope, session, states, tree } = ctx
|
||||
tree.$layoutTree.set(null)
|
||||
|
||||
expect(states.focusOpenSession('canonical-chat', scope)).toBeNull()
|
||||
expect(states.focusWorkspaceOwnerSessionTile(scope.workspaceOwnerKey, undefined, ['canonical-chat'])).toBeNull()
|
||||
expect(session.$selectedStoredSessionId.get()).toBe('previous-chat')
|
||||
expect(states.$sessionTiles.get().map(tile => tile.storedSessionId)).toEqual(['canonical-chat'])
|
||||
})
|
||||
})
|
||||
@@ -320,6 +320,7 @@ describe('SessionTile workspace scope', () => {
|
||||
|
||||
$selectedStoredSessionId.set('bot-chat')
|
||||
openSessionTile('bot-chat', 'center', undefined, undefined, scope)
|
||||
$layoutTree.set(group(['workspace', tilePane('bot-chat')], { active: 'workspace', id: 'main' }))
|
||||
|
||||
expect($sessionTiles.get()).toEqual([
|
||||
expect.objectContaining({
|
||||
@@ -337,6 +338,7 @@ describe('SessionTile workspace scope', () => {
|
||||
// new tip must front that tile, not open the same chat twice.
|
||||
setSessions([{ _lineage_ids: ['seg-1', 'seg-2', 'seg-3'], _lineage_root_id: 'seg-1', id: 'seg-3' } as never])
|
||||
openSessionTile('seg-2')
|
||||
$layoutTree.set(group(['workspace', tilePane('seg-2')], { active: 'workspace', id: 'main' }))
|
||||
|
||||
expect(focusOpenSession('seg-3')).toBe('tile')
|
||||
expect($sessionTiles.get().map(t => t.storedSessionId)).toEqual(['seg-2'])
|
||||
@@ -487,6 +489,7 @@ describe('focusWorkspaceOwnerSessionTile', () => {
|
||||
openSessionTile('thread', 'center', 'workspace', undefined, botA)
|
||||
rememberActivePane(workspaceScopeKey('bots', 'bot:a'), tilePane('closed-bot-chat'))
|
||||
$sessionTiles.set($sessionTiles.get().filter(t => t.storedSessionId !== 'closed-bot-chat'))
|
||||
$layoutTree.set(group(['workspace', tilePane('thread')], { active: 'workspace', id: 'main' }))
|
||||
|
||||
expect(focusWorkspaceOwnerSessionTile('bot:a')).toBe('thread')
|
||||
})
|
||||
@@ -533,6 +536,7 @@ describe('focusWorkspaceOwnerSessionTile', () => {
|
||||
|
||||
it('a throwing probe keeps the tile — reconciliation must not break the click', () => {
|
||||
openSessionTile('bot-chat', 'center', 'workspace', undefined, botA)
|
||||
$layoutTree.set(group(['workspace', tilePane('bot-chat')], { active: 'workspace', id: 'main' }))
|
||||
|
||||
expect(
|
||||
focusWorkspaceOwnerSessionTile('bot:a', () => {
|
||||
@@ -542,8 +546,9 @@ describe('focusWorkspaceOwnerSessionTile', () => {
|
||||
expect($sessionTiles.get().map(t => t.storedSessionId)).toEqual(['bot-chat'])
|
||||
})
|
||||
|
||||
it('no probe keeps the old behavior byte for byte', () => {
|
||||
it('fronts a visible tile without a probe', () => {
|
||||
openSessionTile('bot-chat', 'center', 'workspace', undefined, botA)
|
||||
$layoutTree.set(group(['workspace', tilePane('bot-chat')], { active: 'workspace', id: 'main' }))
|
||||
|
||||
expect(focusWorkspaceOwnerSessionTile('bot:a')).toBe('bot-chat')
|
||||
expect($sessionTiles.get().map(t => t.storedSessionId)).toEqual(['bot-chat'])
|
||||
|
||||
@@ -25,6 +25,7 @@ import {
|
||||
$activeTreeGroup,
|
||||
$layoutTree,
|
||||
focusedSessionTabAnchor,
|
||||
isPaneVisible,
|
||||
moveTreePane,
|
||||
noteActiveTreeGroup,
|
||||
revealTreePane
|
||||
@@ -1551,10 +1552,12 @@ export function focusOpenSession(
|
||||
const tree = $layoutTree.get()
|
||||
const group = tree ? findGroupOfPane(tree, paneId) : null
|
||||
|
||||
if (group) {
|
||||
noteActiveTreeGroup(group.id)
|
||||
if (!group || !isPaneVisible(paneId)) {
|
||||
return null
|
||||
}
|
||||
|
||||
noteActiveTreeGroup(group.id)
|
||||
|
||||
return 'tile'
|
||||
}
|
||||
|
||||
@@ -1634,9 +1637,9 @@ export function focusWorkspaceOwnerSessionTile(
|
||||
const paneId = resolveRememberedActivePane(workspaceScopeKey('bots', workspaceOwnerKey), paneIds) ?? paneIds[0]
|
||||
const storedSessionId = paneId.slice(TILE_PANE_PREFIX.length)
|
||||
|
||||
focusOpenSession(storedSessionId, { workspaceMode: 'bots', workspaceOwnerKey })
|
||||
|
||||
return storedSessionId
|
||||
return focusOpenSession(storedSessionId, { workspaceMode: 'bots', workspaceOwnerKey }) === 'tile'
|
||||
? storedSessionId
|
||||
: null
|
||||
}
|
||||
|
||||
/** Does a sidebar click still need to navigate after `focusOpenSession`? A miss
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
Fatmylin
|
||||
# PR #95964 salvage
|
||||
@@ -0,0 +1,2 @@
|
||||
Christopher-Schulze
|
||||
# PR #85806 salvage
|
||||
@@ -0,0 +1,2 @@
|
||||
ciabata-git
|
||||
# PR #96011 author email preserved by PR #97330
|
||||
@@ -0,0 +1,2 @@
|
||||
pierrenode
|
||||
# PR #84168 salvage
|
||||
@@ -0,0 +1,2 @@
|
||||
JackHunzicker
|
||||
# PR #98371 salvage
|
||||
@@ -0,0 +1,2 @@
|
||||
agentdynamic
|
||||
# PR #92413 salvage
|
||||
@@ -0,0 +1,2 @@
|
||||
slash1andy
|
||||
# PR #88217 salvage
|
||||
@@ -0,0 +1,2 @@
|
||||
benperry6
|
||||
# PR #97330 author email
|
||||
@@ -0,0 +1 @@
|
||||
csreyes
|
||||
@@ -0,0 +1,2 @@
|
||||
EmanueleCornaggia
|
||||
# PR #101778 salvage
|
||||
@@ -0,0 +1,2 @@
|
||||
SelfParody
|
||||
# PR #19000 salvage
|
||||
@@ -0,0 +1 @@
|
||||
GTHell
|
||||
@@ -0,0 +1,2 @@
|
||||
EdderTalmor
|
||||
# PR #10110 salvage
|
||||
@@ -0,0 +1,2 @@
|
||||
yaojiejia
|
||||
# PR #95160 salvage
|
||||
@@ -0,0 +1,379 @@
|
||||
"""Profile-local durable handoff for cron delivery through live gateway adapters.
|
||||
|
||||
A restart-safe cron worker executes outside the gateway cgroup. It cannot own
|
||||
relay/E2EE adapter objects, so it queues the final send here. A gateway claims
|
||||
each row at most once. If that gateway dies after claiming, the outcome is
|
||||
marked unknown and never retried: losing a delivery is safer than duplicating a
|
||||
possibly-completed send.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterator, Optional
|
||||
|
||||
from agent.redact import redact_sensitive_text
|
||||
from cron.executions import _owner_is_live, _process_start_time
|
||||
from hermes_cli.sqlite_util import add_column_if_missing
|
||||
from hermes_constants import get_hermes_home
|
||||
from hermes_time import now as _hermes_now
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DELIVERY_DB: Optional[Path] = None
|
||||
_PROCESS_ID = uuid.uuid4().hex
|
||||
_lock = threading.RLock()
|
||||
_ACTIVE_DELIVERIES: set[str] = set()
|
||||
_TERMINAL = ("delivered", "failed", "unknown")
|
||||
MAX_TERMINAL_DELIVERIES = 1000
|
||||
DEFAULT_DELIVERY_WAIT_TIMEOUT_SECONDS = 300.0
|
||||
|
||||
|
||||
def _prune_terminal_unlocked(conn: sqlite3.Connection) -> None:
|
||||
"""Redact terminal payloads and retain only bounded outcome metadata."""
|
||||
conn.execute(
|
||||
"""UPDATE deliveries SET job_json='{}', content=''
|
||||
WHERE status IN ('delivered','failed','unknown')
|
||||
AND (job_json != '{}' OR content != '')"""
|
||||
)
|
||||
keep = max(0, int(MAX_TERMINAL_DELIVERIES))
|
||||
terminal_count = int(
|
||||
conn.execute(
|
||||
"SELECT COUNT(*) FROM deliveries "
|
||||
"WHERE status IN ('delivered','failed','unknown')"
|
||||
).fetchone()[0]
|
||||
)
|
||||
excess = terminal_count - keep
|
||||
if excess > 0:
|
||||
conn.execute(
|
||||
"""INSERT OR IGNORE INTO delivery_tombstones
|
||||
(execution_id, terminal_status, finished_at)
|
||||
SELECT execution_id, status, finished_at FROM deliveries
|
||||
WHERE status IN ('delivered','failed','unknown')
|
||||
ORDER BY finished_at, created_at, execution_id
|
||||
LIMIT ?""",
|
||||
(excess,),
|
||||
)
|
||||
conn.execute(
|
||||
"""DELETE FROM deliveries WHERE execution_id IN (
|
||||
SELECT execution_id FROM deliveries
|
||||
WHERE status IN ('delivered','failed','unknown')
|
||||
ORDER BY finished_at, created_at, execution_id
|
||||
LIMIT ?
|
||||
)""",
|
||||
(excess,),
|
||||
)
|
||||
|
||||
|
||||
def _path() -> Path:
|
||||
return DELIVERY_DB or (get_hermes_home().resolve() / "cron" / "deliveries.db")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _transaction() -> Iterator[sqlite3.Connection]:
|
||||
with _lock:
|
||||
path = _path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(path, timeout=5)
|
||||
try:
|
||||
path.chmod(0o600)
|
||||
except OSError:
|
||||
pass
|
||||
conn.row_factory = sqlite3.Row
|
||||
try:
|
||||
from hermes_state import apply_wal_with_fallback
|
||||
|
||||
conn.execute("PRAGMA busy_timeout=5000")
|
||||
apply_wal_with_fallback(conn, db_label="cron/deliveries.db")
|
||||
conn.execute("PRAGMA synchronous=FULL")
|
||||
conn.execute(
|
||||
"""CREATE TABLE IF NOT EXISTS deliveries (
|
||||
execution_id TEXT PRIMARY KEY,
|
||||
job_json TEXT NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
for_failure INTEGER NOT NULL DEFAULT 0,
|
||||
status TEXT NOT NULL CHECK(status IN
|
||||
('pending','delivering','delivered','failed','unknown')),
|
||||
owner_process_id TEXT,
|
||||
owner_pid INTEGER,
|
||||
owner_started_at INTEGER,
|
||||
created_at TEXT NOT NULL,
|
||||
finished_at TEXT,
|
||||
error TEXT
|
||||
)"""
|
||||
)
|
||||
conn.execute(
|
||||
"""CREATE TABLE IF NOT EXISTS delivery_tombstones (
|
||||
execution_id TEXT PRIMARY KEY,
|
||||
terminal_status TEXT NOT NULL CHECK(terminal_status IN
|
||||
('delivered','failed','unknown')),
|
||||
finished_at TEXT
|
||||
)"""
|
||||
)
|
||||
add_column_if_missing(
|
||||
conn, "deliveries", "for_failure",
|
||||
"for_failure INTEGER NOT NULL DEFAULT 0",
|
||||
)
|
||||
# Pruning is done explicitly by the paths that create terminal
|
||||
# rows (_finish / recover_abandoned / _terminalize_wait_timeout);
|
||||
# read-only polls must not pay for a full-table UPDATE + COUNT.
|
||||
with conn:
|
||||
yield conn
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def enqueue(
|
||||
execution_id: str,
|
||||
job: dict,
|
||||
content: str,
|
||||
*,
|
||||
for_failure: bool = False,
|
||||
) -> dict:
|
||||
"""Persist one idempotent delivery request before the worker waits."""
|
||||
with _transaction() as conn:
|
||||
tombstone = conn.execute(
|
||||
"SELECT terminal_status, finished_at FROM delivery_tombstones "
|
||||
"WHERE execution_id=?",
|
||||
(str(execution_id),),
|
||||
).fetchone()
|
||||
if tombstone is not None:
|
||||
return {
|
||||
"execution_id": str(execution_id),
|
||||
"status": tombstone["terminal_status"],
|
||||
"finished_at": tombstone["finished_at"],
|
||||
}
|
||||
conn.execute(
|
||||
"""INSERT OR IGNORE INTO deliveries
|
||||
(execution_id, job_json, content, for_failure, status, created_at)
|
||||
VALUES (?, ?, ?, ?, 'pending', ?)""",
|
||||
(
|
||||
str(execution_id),
|
||||
json.dumps(job, ensure_ascii=False, sort_keys=True),
|
||||
str(content),
|
||||
int(bool(for_failure)),
|
||||
_hermes_now().isoformat(),
|
||||
),
|
||||
)
|
||||
row = conn.execute(
|
||||
"SELECT * FROM deliveries WHERE execution_id=?", (str(execution_id),)
|
||||
).fetchone()
|
||||
return dict(row)
|
||||
|
||||
|
||||
def get_status(execution_id: str) -> Optional[dict]:
|
||||
with _transaction() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM deliveries WHERE execution_id=?", (str(execution_id),)
|
||||
).fetchone()
|
||||
if row is not None:
|
||||
return dict(row)
|
||||
tombstone = conn.execute(
|
||||
"SELECT execution_id, terminal_status, finished_at "
|
||||
"FROM delivery_tombstones WHERE execution_id=?",
|
||||
(str(execution_id),),
|
||||
).fetchone()
|
||||
if tombstone is None:
|
||||
return None
|
||||
return {
|
||||
"execution_id": tombstone["execution_id"],
|
||||
"status": tombstone["terminal_status"],
|
||||
"finished_at": tombstone["finished_at"],
|
||||
"error": None,
|
||||
}
|
||||
|
||||
|
||||
def claim_next() -> Optional[dict]:
|
||||
"""Atomically claim one pending send before touching the transport."""
|
||||
pid = os.getpid()
|
||||
started = _process_start_time(pid)
|
||||
with _transaction() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT execution_id FROM deliveries WHERE status='pending' "
|
||||
"ORDER BY created_at, execution_id LIMIT 1"
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
cur = conn.execute(
|
||||
"""UPDATE deliveries SET status='delivering', owner_process_id=?,
|
||||
owner_pid=?, owner_started_at=?
|
||||
WHERE execution_id=? AND status='pending'""",
|
||||
(_PROCESS_ID, pid, started, row["execution_id"]),
|
||||
)
|
||||
if cur.rowcount != 1:
|
||||
return None
|
||||
claimed = conn.execute(
|
||||
"SELECT * FROM deliveries WHERE execution_id=?", (row["execution_id"],)
|
||||
).fetchone()
|
||||
_ACTIVE_DELIVERIES.add(row["execution_id"])
|
||||
result = dict(claimed)
|
||||
result["job"] = json.loads(result.pop("job_json"))
|
||||
return result
|
||||
|
||||
|
||||
def _finish(execution_id: str, *, error: Optional[str]) -> bool:
|
||||
status = "failed" if error else "delivered"
|
||||
safe_error = (
|
||||
redact_sensitive_text(str(error), force=True, redact_url_credentials=True)
|
||||
if error
|
||||
else None
|
||||
)
|
||||
with _transaction() as conn:
|
||||
cur = conn.execute(
|
||||
"""UPDATE deliveries SET status=?, finished_at=?, error=?
|
||||
WHERE execution_id=? AND status='delivering'
|
||||
AND owner_process_id=? AND owner_pid=?""",
|
||||
(
|
||||
status,
|
||||
_hermes_now().isoformat(),
|
||||
safe_error,
|
||||
execution_id,
|
||||
_PROCESS_ID,
|
||||
os.getpid(),
|
||||
),
|
||||
)
|
||||
_prune_terminal_unlocked(conn)
|
||||
return cur.rowcount == 1
|
||||
|
||||
|
||||
def recover_abandoned() -> int:
|
||||
"""Fence dead delivery owners as unknown; never replay uncertain sends."""
|
||||
changed = 0
|
||||
with _transaction() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT execution_id, owner_process_id, owner_pid, owner_started_at "
|
||||
"FROM deliveries WHERE status='delivering'"
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
same_process = row["owner_process_id"] == _PROCESS_ID
|
||||
if same_process:
|
||||
with _lock:
|
||||
if row["execution_id"] in _ACTIVE_DELIVERIES:
|
||||
continue
|
||||
elif _owner_is_live(int(row["owner_pid"]), row["owner_started_at"]):
|
||||
continue
|
||||
error = (
|
||||
"Gateway finished delivery but could not persist its outcome; "
|
||||
"send was not retried."
|
||||
if same_process
|
||||
else "Gateway exited during delivery; send outcome is unknown and was not retried."
|
||||
)
|
||||
cur = conn.execute(
|
||||
"""UPDATE deliveries SET status='unknown', finished_at=?, error=?
|
||||
WHERE execution_id=? AND status='delivering'""",
|
||||
(
|
||||
_hermes_now().isoformat(),
|
||||
error,
|
||||
row["execution_id"],
|
||||
),
|
||||
)
|
||||
changed += cur.rowcount
|
||||
_prune_terminal_unlocked(conn)
|
||||
return changed
|
||||
|
||||
|
||||
def drain(
|
||||
send: Callable[[dict, str, bool], Optional[str]], *, limit: int = 20
|
||||
) -> int:
|
||||
"""Deliver pending rows through *send*, terminalizing every claimed row."""
|
||||
recover_abandoned()
|
||||
processed = 0
|
||||
for _ in range(max(0, limit)):
|
||||
row = claim_next()
|
||||
if row is None:
|
||||
break
|
||||
with _lock:
|
||||
_ACTIVE_DELIVERIES.add(row["execution_id"])
|
||||
try:
|
||||
try:
|
||||
error = send(
|
||||
row["job"], row["content"], bool(row["for_failure"])
|
||||
)
|
||||
except BaseException as exc:
|
||||
error = f"{type(exc).__name__}: {exc}"
|
||||
_finish(row["execution_id"], error=error)
|
||||
finally:
|
||||
with _lock:
|
||||
_ACTIVE_DELIVERIES.discard(row["execution_id"])
|
||||
processed += 1
|
||||
return processed
|
||||
|
||||
|
||||
def _terminalize_wait_timeout(execution_id: str) -> str:
|
||||
"""Fence a delivery whose worker can no longer wait for confirmation.
|
||||
|
||||
A row still ``pending`` was provably never attempted, so it is left queued
|
||||
for whichever gateway comes up next (a restart that includes an update can
|
||||
easily exceed the worker's wait budget). That is a deferral, not a
|
||||
failure: report success so the job is not recorded ``delivery_failed`` for
|
||||
a message the drain will still send. Only a row caught mid-send is
|
||||
uncertain and gets fenced ``unknown``.
|
||||
"""
|
||||
now = _hermes_now().isoformat()
|
||||
uncertain_error = (
|
||||
"timed out while gateway delivery was in progress; outcome is unknown and "
|
||||
"was not retried"
|
||||
)
|
||||
with _transaction() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT status FROM deliveries WHERE execution_id=?",
|
||||
(str(execution_id),),
|
||||
).fetchone()
|
||||
if row is not None and row["status"] == "pending":
|
||||
logger.warning(
|
||||
"Cron delivery %s: no live gateway within the wait budget; "
|
||||
"left queued for the next gateway",
|
||||
execution_id,
|
||||
)
|
||||
return ""
|
||||
conn.execute(
|
||||
"""UPDATE deliveries SET status='unknown', finished_at=?, error=?
|
||||
WHERE execution_id=? AND status='delivering'""",
|
||||
(now, uncertain_error, str(execution_id)),
|
||||
)
|
||||
row = conn.execute(
|
||||
"SELECT status, error FROM deliveries WHERE execution_id=?",
|
||||
(str(execution_id),),
|
||||
).fetchone()
|
||||
_prune_terminal_unlocked(conn)
|
||||
if row is None:
|
||||
return "timed out waiting for live gateway delivery"
|
||||
if row["status"] == "delivered":
|
||||
return ""
|
||||
return str(row["error"] or f"delivery {row['status']}")
|
||||
|
||||
|
||||
def enqueue_and_wait(
|
||||
execution_id: str,
|
||||
job: dict,
|
||||
content: str,
|
||||
*,
|
||||
for_failure: bool = False,
|
||||
timeout: Optional[float] = None,
|
||||
) -> Optional[str]:
|
||||
"""Queue delivery and wait for a gateway's terminal at-most-once outcome."""
|
||||
queued = enqueue(execution_id, job, content, for_failure=for_failure)
|
||||
if queued["status"] in _TERMINAL:
|
||||
return None if queued["status"] == "delivered" else str(
|
||||
queued.get("error") or f"delivery {queued['status']}"
|
||||
)
|
||||
wait_timeout = (
|
||||
DEFAULT_DELIVERY_WAIT_TIMEOUT_SECONDS if timeout is None else max(0.0, timeout)
|
||||
)
|
||||
deadline = time.monotonic() + wait_timeout
|
||||
while time.monotonic() < deadline:
|
||||
row = get_status(execution_id)
|
||||
if row and row["status"] in _TERMINAL:
|
||||
return None if row["status"] == "delivered" else str(
|
||||
row.get("error") or f"delivery {row['status']}"
|
||||
)
|
||||
time.sleep(1.0)
|
||||
return _terminalize_wait_timeout(execution_id) or None
|
||||
+98
-11
@@ -10,6 +10,7 @@ from __future__ import annotations
|
||||
import os
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
@@ -23,6 +24,7 @@ from hermes_time import now as _hermes_now
|
||||
# home.
|
||||
EXECUTIONS_FILE: Optional[Path] = None
|
||||
MAX_TERMINAL_EXECUTIONS = 1000
|
||||
HANDOFF_ADOPTION_GRACE_SECONDS = 30.0
|
||||
_TERMINAL_STATES = ("completed", "failed", "unknown")
|
||||
_lock = threading.RLock()
|
||||
_PROCESS_ID = uuid.uuid4().hex
|
||||
@@ -88,12 +90,23 @@ def _initialize_schema(conn: sqlite3.Connection) -> None:
|
||||
process_started_at INTEGER,
|
||||
status TEXT NOT NULL CHECK(status IN
|
||||
('claimed','running','completed','failed','unknown')),
|
||||
handoff_pending INTEGER NOT NULL DEFAULT 0,
|
||||
handoff_started_at REAL,
|
||||
claimed_at TEXT NOT NULL,
|
||||
started_at TEXT,
|
||||
finished_at TEXT,
|
||||
error TEXT
|
||||
)"""
|
||||
)
|
||||
from hermes_cli.sqlite_util import add_column_if_missing
|
||||
|
||||
add_column_if_missing(
|
||||
conn, "executions", "handoff_pending",
|
||||
"handoff_pending INTEGER NOT NULL DEFAULT 0",
|
||||
)
|
||||
add_column_if_missing(
|
||||
conn, "executions", "handoff_started_at", "handoff_started_at REAL"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE INDEX IF NOT EXISTS idx_executions_job_claimed "
|
||||
"ON executions(job_id, claimed_at DESC, id DESC)"
|
||||
@@ -153,7 +166,7 @@ def _prune_unlocked(conn: sqlite3.Connection) -> None:
|
||||
"""DELETE FROM executions WHERE id IN (
|
||||
SELECT id FROM executions
|
||||
WHERE status IN ('completed','failed','unknown')
|
||||
ORDER BY claimed_at DESC, id DESC LIMIT -1 OFFSET ?
|
||||
ORDER BY finished_at DESC, claimed_at DESC, id DESC LIMIT -1 OFFSET ?
|
||||
)""",
|
||||
(max(0, int(MAX_TERMINAL_EXECUTIONS)),),
|
||||
)
|
||||
@@ -178,14 +191,60 @@ def create_execution(job_id: str, *, source: str) -> Dict[str, Any]:
|
||||
return record # type: ignore[return-value]
|
||||
|
||||
|
||||
def mark_execution_handoff_pending(execution_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Fence restart recovery while an external worker is adopting a claim."""
|
||||
with _transaction() as conn:
|
||||
cur = conn.execute(
|
||||
"""UPDATE executions
|
||||
SET handoff_pending=1, handoff_started_at=?
|
||||
WHERE id=? AND status='claimed'
|
||||
AND process_id=? AND pid=?""",
|
||||
(time.time(), execution_id, _PROCESS_ID, os.getpid()),
|
||||
)
|
||||
if cur.rowcount != 1:
|
||||
return None
|
||||
record = _fetch(conn, execution_id)
|
||||
_emit_execution_state(record)
|
||||
return record
|
||||
|
||||
|
||||
def adopt_claimed_execution(execution_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Atomically transfer and start an attempt in its worker process.
|
||||
|
||||
The dispatching gateway creates the row before spawning a restart-safe
|
||||
worker. Adoption is the single ``claimed`` → ``running`` gate: only the
|
||||
winner may acknowledge ownership or run side effects.
|
||||
"""
|
||||
pid = os.getpid()
|
||||
process_started_at = _process_start_time(pid)
|
||||
now = _hermes_now().isoformat()
|
||||
with _transaction() as conn:
|
||||
cur = conn.execute(
|
||||
"""UPDATE executions
|
||||
SET process_id=?, pid=?, process_started_at=?,
|
||||
status='running', started_at=?, handoff_pending=0,
|
||||
handoff_started_at=NULL
|
||||
WHERE id=? AND status='claimed' AND handoff_pending=1""",
|
||||
(_PROCESS_ID, pid, process_started_at, now, execution_id),
|
||||
)
|
||||
if cur.rowcount != 1:
|
||||
return None
|
||||
record = _fetch(conn, execution_id)
|
||||
_emit_execution_state(record)
|
||||
return record
|
||||
|
||||
|
||||
def mark_execution_running(execution_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Transition one claimed attempt to running exactly once."""
|
||||
now = _hermes_now().isoformat()
|
||||
with _transaction() as conn:
|
||||
cur = conn.execute(
|
||||
"""UPDATE executions SET status='running', started_at=?
|
||||
WHERE id=? AND status='claimed'""",
|
||||
(now, execution_id),
|
||||
"""UPDATE executions
|
||||
SET status='running', started_at=?, handoff_pending=0,
|
||||
handoff_started_at=NULL
|
||||
WHERE id=? AND status='claimed' AND handoff_pending=0
|
||||
AND process_id=? AND pid=?""",
|
||||
(now, execution_id, _PROCESS_ID, os.getpid()),
|
||||
)
|
||||
if cur.rowcount != 1:
|
||||
return None
|
||||
@@ -204,9 +263,12 @@ def finish_execution(
|
||||
detail = None if success else (str(error) if error else "unknown failure")
|
||||
with _transaction() as conn:
|
||||
cur = conn.execute(
|
||||
"""UPDATE executions SET status=?, finished_at=?, error=?
|
||||
WHERE id=? AND status IN ('claimed','running')""",
|
||||
(status, now, detail, execution_id),
|
||||
"""UPDATE executions
|
||||
SET status=?, finished_at=?, error=?, handoff_pending=0,
|
||||
handoff_started_at=NULL
|
||||
WHERE id=? AND status IN ('claimed','running')
|
||||
AND process_id=? AND pid=?""",
|
||||
(status, now, detail, execution_id, _PROCESS_ID, os.getpid()),
|
||||
)
|
||||
if cur.rowcount != 1:
|
||||
return None
|
||||
@@ -223,7 +285,9 @@ def recover_interrupted_executions() -> int:
|
||||
recovered: List[Dict[str, Any]] = []
|
||||
with _transaction() as conn:
|
||||
rows = conn.execute(
|
||||
"""SELECT id, process_id, pid, process_started_at FROM executions
|
||||
"""SELECT id, status, process_id, pid, process_started_at,
|
||||
handoff_pending, handoff_started_at
|
||||
FROM executions
|
||||
WHERE status IN ('claimed','running')"""
|
||||
).fetchall()
|
||||
for row in rows:
|
||||
@@ -231,13 +295,26 @@ def recover_interrupted_executions() -> int:
|
||||
continue
|
||||
if _owner_is_live(int(row["pid"]), row["process_started_at"]):
|
||||
continue
|
||||
handoff_started_at = row["handoff_started_at"]
|
||||
if (
|
||||
row["handoff_pending"]
|
||||
and handoff_started_at is not None
|
||||
and time.time() - float(handoff_started_at)
|
||||
< HANDOFF_ADOPTION_GRACE_SECONDS
|
||||
):
|
||||
continue
|
||||
cur = conn.execute(
|
||||
"""UPDATE executions SET status='unknown', finished_at=?, error=?
|
||||
WHERE id=? AND status IN ('claimed','running')""",
|
||||
"""UPDATE executions
|
||||
SET status='unknown', finished_at=?, error=?,
|
||||
handoff_pending=0, handoff_started_at=NULL
|
||||
WHERE id=? AND status=? AND process_id=? AND pid=?
|
||||
AND handoff_pending=?
|
||||
AND handoff_started_at IS ?""",
|
||||
(now,
|
||||
"Scheduler restarted after this execution's owner exited before a durable "
|
||||
"terminal state; whether side effects ran is unknown.",
|
||||
row["id"]),
|
||||
row["id"], row["status"], row["process_id"], row["pid"],
|
||||
row["handoff_pending"], row["handoff_started_at"]),
|
||||
)
|
||||
changed += cur.rowcount
|
||||
if cur.rowcount:
|
||||
@@ -274,6 +351,16 @@ def list_executions(
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def get_execution(execution_id: str) -> Optional[Dict[str, Any]]:
|
||||
"""Return one exact execution attempt, or ``None`` when it is absent."""
|
||||
with _transaction() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM executions WHERE id=?",
|
||||
(str(execution_id),),
|
||||
).fetchone()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
def latest_execution(job_id: str) -> Optional[Dict[str, Any]]:
|
||||
rows = list_executions(job_id=job_id, limit=1)
|
||||
return rows[0] if rows else None
|
||||
|
||||
+131
-12
@@ -249,6 +249,95 @@ def contains_gateway_lifecycle_command(text: str) -> bool:
|
||||
return _contains_launchctl_gateway_lifecycle(normalized)
|
||||
|
||||
|
||||
# Whole-walk work limits. The per-file cap and depth bound above limit one read, not the walk: a
|
||||
# command can reference arbitrarily many scripts, and the pure-Python shlex pass (quadratic on a
|
||||
# giant token) once held the GIL for minutes. These caps bound one whole walk and are charged
|
||||
# BEFORE any text reaches shlex. Exhaustion fails closed (an unscanned script could hide a
|
||||
# lifecycle command) and is logged at WARNING so an operator can tell it from a real block. Sizes
|
||||
# sit well above any legitimate wrapper graph; remote reads are a backend roundtrip each, so they
|
||||
# get a far tighter cap.
|
||||
_MAX_LIFECYCLE_SCAN_BYTES = _MAX_REFERENCED_SCRIPT_BYTES # 1 MiB across the walk
|
||||
_MAX_LIFECYCLE_SCAN_LINES = 16384
|
||||
_MAX_LIFECYCLE_SCAN_LINE_BYTES = 64 * 1024
|
||||
_MAX_LIFECYCLE_SCAN_PATHS = 1024
|
||||
_MAX_LIFECYCLE_SCAN_REMOTE_READS = 64
|
||||
|
||||
|
||||
class _LifecycleScanBudget:
|
||||
"""Shared work budget for one complete referenced-script walk."""
|
||||
|
||||
__slots__ = ("bytes_remaining", "lines_remaining", "paths_remaining", "remote_reads_remaining")
|
||||
|
||||
def __init__(self) -> None:
|
||||
# Read the module constants at construction so tests/operators can lower them at runtime.
|
||||
self.bytes_remaining = _MAX_LIFECYCLE_SCAN_BYTES
|
||||
self.lines_remaining = _MAX_LIFECYCLE_SCAN_LINES
|
||||
self.paths_remaining = _MAX_LIFECYCLE_SCAN_PATHS
|
||||
self.remote_reads_remaining = _MAX_LIFECYCLE_SCAN_REMOTE_READS
|
||||
|
||||
def charge_text(self, text: str) -> bool:
|
||||
"""Charge *text* before tokenization; False when it does not fit."""
|
||||
# UTF-8 is >= one byte per code point, so the char count is a free lower bound.
|
||||
if len(text) > self.bytes_remaining:
|
||||
return False
|
||||
encoded = len(text.encode("utf-8", errors="replace"))
|
||||
if encoded > self.bytes_remaining:
|
||||
return False
|
||||
lines = text.count("\n") + 1
|
||||
if lines > self.lines_remaining:
|
||||
return False
|
||||
# One huge token is the quadratic shlex case; bound the longest physical line (chars, a
|
||||
# lower bound on bytes — tight enough for a DoS bound without a per-line encode).
|
||||
longest = max((len(line) for line in text.split("\n")), default=0)
|
||||
if longest > _MAX_LIFECYCLE_SCAN_LINE_BYTES:
|
||||
return False
|
||||
self.bytes_remaining -= encoded
|
||||
self.lines_remaining -= lines
|
||||
return True
|
||||
|
||||
def charge_path(self) -> bool:
|
||||
"""Charge one unique referenced path before any local/remote read."""
|
||||
if self.paths_remaining <= 0:
|
||||
return False
|
||||
self.paths_remaining -= 1
|
||||
return True
|
||||
|
||||
def charge_remote_read(self) -> bool:
|
||||
"""Charge one remote-backend read (a network roundtrip each)."""
|
||||
if self.remote_reads_remaining <= 0:
|
||||
return False
|
||||
self.remote_reads_remaining -= 1
|
||||
return True
|
||||
|
||||
|
||||
def _capped_read_limit(max_bytes: Optional[int]) -> int:
|
||||
"""Per-read byte cap: never above the per-file cap, never negative. One definition so local and
|
||||
remote reads cannot diverge."""
|
||||
if max_bytes is None:
|
||||
return _MAX_REFERENCED_SCRIPT_BYTES
|
||||
return min(_MAX_REFERENCED_SCRIPT_BYTES, max(0, int(max_bytes)))
|
||||
|
||||
|
||||
def lifecycle_scan_root_within_budget(text: str) -> bool:
|
||||
"""Whether *text* may safely enter an optional tokenizer pass (``tools/terminal_tool.py`` gates
|
||||
its launchctl pre-scan on this). A FRESH budget, independent of the full guard's walk: the
|
||||
pre-scan may pass while the walk later exhausts, still fail-closed — only the friendlier
|
||||
launchctl diagnostic is lost. ``False`` is not a verdict: callers must still run the full guard."""
|
||||
try:
|
||||
return _LifecycleScanBudget().charge_text(text)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _budget_exhausted(what: str, depth: int) -> bool:
|
||||
logger.warning(
|
||||
"lifecycle guard scan budget exhausted (%s at depth %d); "
|
||||
"failing closed — see _MAX_LIFECYCLE_SCAN_* in cron/lifecycle_guard.py",
|
||||
what, depth,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
# --- shell tokenization -----------------------------------------------------------------------
|
||||
|
||||
def _split_logical_lines(text: str) -> list[str]:
|
||||
@@ -622,13 +711,17 @@ def _has_binary_magic(data: bytes) -> bool:
|
||||
return data.startswith(_BINARY_MAGICS)
|
||||
|
||||
|
||||
def _read_referenced_script(path: Path) -> tuple[Optional[str], bool]:
|
||||
def _read_referenced_script(
|
||||
path: Path, *, max_bytes: Optional[int] = None
|
||||
) -> tuple[Optional[str], bool]:
|
||||
"""Return ``(text, unsafe)`` using bounded, regular-file-only reads.
|
||||
|
||||
Shared choke point for every local script read, so the cloud-placeholder refusal lives here: a
|
||||
FileProvider path is never opened — not even to check hydration — because an evicted
|
||||
placeholder's ``open()`` can hang preflight. Lexical check: direct paths; resolved: symlinks.
|
||||
``max_bytes`` lowers the per-file cap to what the calling walk can still afford.
|
||||
"""
|
||||
byte_limit = _capped_read_limit(max_bytes)
|
||||
if _on_cloud_path(path):
|
||||
return None, True
|
||||
flags = os.O_RDONLY | getattr(os, "O_NONBLOCK", 0)
|
||||
@@ -649,9 +742,13 @@ def _read_referenced_script(path: Path) -> tuple[Optional[str], bool]:
|
||||
data = os.read(descriptor, _BINARY_SNIFF_BYTES)
|
||||
if _has_binary_magic(data):
|
||||
return None, False
|
||||
# A regular file whose size already exceeds the cap fails closed without reading it (the
|
||||
# walk budget can be far below 1 MiB).
|
||||
if metadata.st_size > byte_limit:
|
||||
return None, True
|
||||
# Read the remainder (bounded); loop because os.read may return short.
|
||||
while len(data) <= _MAX_REFERENCED_SCRIPT_BYTES:
|
||||
chunk = os.read(descriptor, _MAX_REFERENCED_SCRIPT_BYTES + 1 - len(data))
|
||||
while len(data) <= byte_limit:
|
||||
chunk = os.read(descriptor, byte_limit + 1 - len(data))
|
||||
if not chunk:
|
||||
break
|
||||
data += chunk
|
||||
@@ -663,21 +760,26 @@ def _read_referenced_script(path: Path) -> tuple[Optional[str], bool]:
|
||||
return None, False
|
||||
# Size check BEFORE NUL stripping: stripping shrinks the buffer and would let an oversized file
|
||||
# slip under the threshold past this fail-closed branch.
|
||||
if len(data) > _MAX_REFERENCED_SCRIPT_BYTES:
|
||||
if len(data) > byte_limit:
|
||||
return None, True
|
||||
if b"\x00" in data:
|
||||
data = data.replace(b"\x00", b"")
|
||||
return data.decode("utf-8", errors="replace"), False
|
||||
|
||||
|
||||
def _sanitize_remote_script_text(text: Optional[str]) -> tuple[Optional[str], bool]:
|
||||
def _sanitize_remote_script_text(
|
||||
text: Optional[str], *, max_bytes: Optional[int] = None
|
||||
) -> tuple[Optional[str], bool]:
|
||||
"""Apply the local-read contract to text from an untrusted ``read_remote_script`` callback: NUL
|
||||
means binary (nothing to scan, checked first); oversized fails closed. Size compares re-encoded
|
||||
*bytes* (matching the ``head -c`` wire bound): a >1 MiB multibyte file truncated at the byte cap
|
||||
decodes to fewer chars, and a char count would scan instead of failing."""
|
||||
if not text or "\x00" in text:
|
||||
return None, False
|
||||
if len(text.encode("utf-8", errors="replace")) > _MAX_REFERENCED_SCRIPT_BYTES:
|
||||
byte_limit = _capped_read_limit(max_bytes)
|
||||
if len(text) > byte_limit:
|
||||
return None, True # chars <= bytes: over the cap without encoding
|
||||
if len(text.encode("utf-8", errors="replace")) > byte_limit:
|
||||
return None, True
|
||||
return text, False
|
||||
|
||||
@@ -698,9 +800,12 @@ def _read_script_for_scanning(script_path: str) -> str:
|
||||
# --- recursive walk ---------------------------------------------------------------------------
|
||||
|
||||
def _contains_unsafe_gateway_action(
|
||||
command: str, *, cwd: Optional[str], depth: int, visited: set[Path],
|
||||
command: str, *, cwd: Optional[str], depth: int, visited: set[Path], budget: _LifecycleScanBudget,
|
||||
read_remote_script: Optional[_ReadRemoteScriptFn] = None,
|
||||
) -> bool:
|
||||
# Charge BEFORE _direct_lifecycle_scan: every scan in it tokenizes with shlex.
|
||||
if not budget.charge_text(command):
|
||||
return _budget_exhausted("text", depth)
|
||||
if _direct_lifecycle_scan(command):
|
||||
return True
|
||||
if depth >= _MAX_REFERENCED_SCRIPT_DEPTH:
|
||||
@@ -708,7 +813,8 @@ def _contains_unsafe_gateway_action(
|
||||
|
||||
def recurse(text: str, cwd: Optional[str]) -> bool:
|
||||
return _contains_unsafe_gateway_action(
|
||||
text, cwd=cwd, depth=depth + 1, visited=visited, read_remote_script=read_remote_script
|
||||
text, cwd=cwd, depth=depth + 1, visited=visited, budget=budget,
|
||||
read_remote_script=read_remote_script,
|
||||
)
|
||||
|
||||
for payload in _iter_shell_command_payloads(command):
|
||||
@@ -722,14 +828,22 @@ def _contains_unsafe_gateway_action(
|
||||
resolved = _resolve_lenient(script_path)
|
||||
if resolved in visited:
|
||||
continue
|
||||
if not budget.charge_path():
|
||||
return _budget_exhausted("paths", depth)
|
||||
visited.add(resolved)
|
||||
script_text, unsafe = _read_referenced_script(script_path)
|
||||
# Never read more than the walk can still afford to tokenize; a file larger than the
|
||||
# remainder fails closed exactly like an oversized one.
|
||||
script_text, unsafe = _read_referenced_script(script_path, max_bytes=budget.bytes_remaining)
|
||||
if unsafe:
|
||||
return True
|
||||
if script_text is None and read_remote_script is not None:
|
||||
# Local path missing; the remote backend's output crosses the same trust boundary as a
|
||||
# local read — sanitize identically (binary skip + size fail-closed).
|
||||
script_text, unsafe = _sanitize_remote_script_text(read_remote_script(str(script_path)))
|
||||
if not budget.charge_remote_read():
|
||||
return _budget_exhausted("remote reads", depth)
|
||||
script_text, unsafe = _sanitize_remote_script_text(
|
||||
read_remote_script(str(script_path)), max_bytes=budget.bytes_remaining
|
||||
)
|
||||
if unsafe:
|
||||
return True
|
||||
if not script_text:
|
||||
@@ -752,7 +866,8 @@ def contains_gateway_lifecycle_command_or_referenced_script(
|
||||
"""
|
||||
try:
|
||||
return _contains_unsafe_gateway_action(
|
||||
command, cwd=cwd, depth=0, visited=set(), read_remote_script=read_remote_script
|
||||
command, cwd=cwd, depth=0, visited=set(), budget=_LifecycleScanBudget(),
|
||||
read_remote_script=read_remote_script,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
@@ -795,7 +910,11 @@ def check_gateway_lifecycle(prompt: Optional[str], script: Optional[str] = None)
|
||||
# Python runs via the interpreter, never a POSIX shell, and the shell reference walk is a
|
||||
# false-positive generator on Python sources (pathlib "/" resolves to the filesystem root).
|
||||
# The regex still scans the full text; non-regular/oversized files fail closed (sentinel).
|
||||
unsafe = _lifecycle_command_scan_with_data_exemption(combined)
|
||||
# The data-exemption masker tokenizes with shlex, so it is charged against the walk budget.
|
||||
if not _LifecycleScanBudget().charge_text(combined):
|
||||
unsafe = _budget_exhausted("text", 0)
|
||||
else:
|
||||
unsafe = _lifecycle_command_scan_with_data_exemption(combined)
|
||||
else:
|
||||
unsafe = contains_gateway_lifecycle_command_or_referenced_script(
|
||||
combined, cwd=_resolve_script_directory(script) if script else None
|
||||
|
||||
+398
-7
@@ -434,7 +434,9 @@ from cron.jobs import (
|
||||
_ensure_cron_dir, advance_next_runs, claim_dispatch, claim_job_for_fire, fire_claim_fence,
|
||||
clear_run_claim, get_due_jobs, heartbeat_fire_claim, heartbeat_run_claim, mark_job_run,
|
||||
save_job_output, use_cron_store)
|
||||
from cron.executions import create_execution, finish_execution, mark_execution_running
|
||||
from cron.executions import (
|
||||
_TERMINAL_STATES, create_execution, finish_execution, get_execution,
|
||||
mark_execution_handoff_pending, mark_execution_running, recover_interrupted_executions)
|
||||
|
||||
# Response marker that suppresses delivery (output is still saved locally for audit).
|
||||
SILENT_MARKER = "[SILENT]"
|
||||
@@ -453,6 +455,10 @@ _parallel_pool: Optional[concurrent.futures.ThreadPoolExecutor] = None
|
||||
_parallel_pool_max_workers: Optional[int] = None
|
||||
_running_job_ids: set = set()
|
||||
_running_fire_owners: dict[str, dict[object, tuple[Optional[str], Path]]] = {}
|
||||
# Parent gateway threads synchronously waiting on restart-safe scope workers.
|
||||
# Shutdown must not misclassify these as ownerless in-process runs: the tool
|
||||
# process sweep cannot reach the worker's transient scope.
|
||||
_restart_safe_waiter_job_ids: set[str] = set()
|
||||
_running_lock = threading.Lock()
|
||||
|
||||
# Per in-flight id: time.time() claim instant + the future owning its release (``_FUTURE_PENDING``
|
||||
@@ -781,9 +787,11 @@ def mark_running_jobs_interrupted(
|
||||
``run_one_job`` sees them.
|
||||
"""
|
||||
with _running_lock:
|
||||
restart_safe_waiters = set(_restart_safe_waiter_job_ids)
|
||||
active_fires = [
|
||||
(token, job_id, owner, profile_home)
|
||||
for job_id, executions in _running_fire_owners.items()
|
||||
if job_id not in restart_safe_waiters
|
||||
for token, (owner, profile_home) in executions.items()
|
||||
]
|
||||
if only_owners is not None:
|
||||
@@ -792,7 +800,9 @@ def mark_running_jobs_interrupted(
|
||||
if only_owners is None:
|
||||
active_fires.extend(
|
||||
(None, job_id, None, _get_hermes_home())
|
||||
for job_id in _running_job_ids - registered_ids
|
||||
for job_id in (
|
||||
_running_job_ids - registered_ids - restart_safe_waiters
|
||||
)
|
||||
)
|
||||
_interrupted_job_ids.update(
|
||||
token if token is not None else job_id
|
||||
@@ -987,6 +997,26 @@ def _reclaim_fds_best_effort() -> None:
|
||||
apply_nofile_soft_limit(None)
|
||||
|
||||
|
||||
def drain_delivery_queue(adapters, loop) -> int:
|
||||
"""Send queued worker results through this gateway's live adapters."""
|
||||
from cron.delivery_queue import _path, drain
|
||||
|
||||
# Only restart-safe workers create the queue file. Every gateway (macOS,
|
||||
# Windows, launchd, Docker) runs this housekeeping tick, so skip the sqlite
|
||||
# open/create entirely until a worker has actually queued something.
|
||||
if not _path().exists():
|
||||
return 0
|
||||
return drain(
|
||||
lambda queued_job, queued_content, queued_for_failure: _deliver_result(
|
||||
queued_job,
|
||||
queued_content,
|
||||
adapters=adapters,
|
||||
loop=loop,
|
||||
for_failure=queued_for_failure,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
_DEFAULT_SCRIPT_TIMEOUT = 3600 # seconds (1 hour)
|
||||
# Backward-compatible module override used by tests and emergency monkeypatches.
|
||||
_SCRIPT_TIMEOUT = _DEFAULT_SCRIPT_TIMEOUT
|
||||
@@ -2264,6 +2294,34 @@ def run_one_job(
|
||||
claim (callers use the store CAS) but keeps it alive. True if processed (a job failure is
|
||||
recorded via ``mark_job_run``), False only if processing raised. ``cancel_event``: optional
|
||||
transport-level cancel (dashboard drain)."""
|
||||
# Every gateway path (built-in scheduler, external providers, and direct
|
||||
# API fires) crosses this seam. Ensure the detached worker has a durable
|
||||
# attempt to adopt before any launch can occur.
|
||||
if not job.get("execution_id"):
|
||||
execution = create_execution(job["id"], source="direct")
|
||||
job["execution_id"] = execution["id"]
|
||||
|
||||
execution_id = str(job["execution_id"])
|
||||
external_owner = os.environ.get("_HERMES_CRON_EXTERNAL_WORKER") == execution_id
|
||||
if not external_owner:
|
||||
try:
|
||||
if _launch_external_cron_worker(job):
|
||||
return True
|
||||
except Exception as handoff_error:
|
||||
error = f"Restart-safe cron worker dispatch failed: {handoff_error}"
|
||||
logger.error("Job '%s': %s", job["id"], error)
|
||||
claim = job.get("fire_claim")
|
||||
owner = str(claim.get("by") or "") if isinstance(claim, dict) else ""
|
||||
try:
|
||||
mark_job_run(
|
||||
job["id"],
|
||||
False,
|
||||
error,
|
||||
**({"expected_fire_owner": owner} if owner else {}),
|
||||
)
|
||||
finally:
|
||||
finish_execution(execution_id, success=False, error=error)
|
||||
return True
|
||||
if extra_prompt is None:
|
||||
# Gateway-forwarded manual run stamps its prompt on the job via trigger_job; the fire that
|
||||
# consumes the manual occurrence picks it up here. Single-fire: mark_job_run clears it.
|
||||
@@ -2613,7 +2671,13 @@ def _run_one_job_body(
|
||||
error="Dispatch claim rejected; execution was not started.")
|
||||
return True # not an error — already handled/removed
|
||||
|
||||
mark_execution_running(execution_id)
|
||||
# Claimed durably before dispatch; becomes running only right before the actual run.
|
||||
# Detached workers transition to running while adopting; in-process paths must win the
|
||||
# claimed->running CAS here before any user script or agent side effect may begin.
|
||||
external_owner = os.environ.get("_HERMES_CRON_EXTERNAL_WORKER") == execution_id
|
||||
if not external_owner and mark_execution_running(execution_id) is None:
|
||||
logger.warning("Cron job %s lost execution ownership before start; skipping", job["id"])
|
||||
return True
|
||||
|
||||
# get_secret() fails closed outside a scope; the ticker thread has none. Delivery adapters
|
||||
# resolve credentials, so the scope must span delivery too (reset in the outer finally).
|
||||
@@ -2733,6 +2797,313 @@ def _run_one_job_body(
|
||||
reset_terminal_scope(_terminal_scope_token)
|
||||
|
||||
|
||||
def _wait_for_external_cron_worker_body(
|
||||
process: subprocess.Popen,
|
||||
*,
|
||||
execution_id: str,
|
||||
) -> bool:
|
||||
"""Preserve ``run_one_job``'s synchronous contract after handoff.
|
||||
|
||||
The worker owns the durable execution and survives this gateway process.
|
||||
The caller nevertheless waits while it remains alive so manual/background
|
||||
callers do not release their in-process guard or report stale job state.
|
||||
A gateway replacement may kill this waiter; it does not kill the scoped
|
||||
worker or change its ledger ownership.
|
||||
"""
|
||||
def _is_terminal() -> bool:
|
||||
current = get_execution(execution_id)
|
||||
return bool(current and current.get("status") in _TERMINAL_STATES)
|
||||
|
||||
# The worker commits its terminal row before its process exits, so exit is
|
||||
# the correct wakeup. Each ledger read opens a connection and re-runs
|
||||
# schema init; polling it at 50ms for an hours-long agent run is ~72k
|
||||
# opens/hour of pure contention with the worker's own writes.
|
||||
while True:
|
||||
try:
|
||||
returncode = process.wait(timeout=1.0)
|
||||
except subprocess.TimeoutExpired:
|
||||
if _is_terminal():
|
||||
return True
|
||||
continue
|
||||
# The worker can commit its terminal row and exit between the first
|
||||
# read and wait(). Re-read the exact attempt before declaring that
|
||||
# it died without terminalizing.
|
||||
if _is_terminal():
|
||||
return True
|
||||
# If the adopted worker died without terminalizing, its owner is
|
||||
# now provably gone. Recover to ``unknown`` rather than routing the
|
||||
# exception through the pre-handoff dispatch-failure path, which
|
||||
# would falsely assert that no side effect could have happened.
|
||||
recover_interrupted_executions()
|
||||
if _is_terminal():
|
||||
return True
|
||||
raise RuntimeError(
|
||||
"cron external worker exited before durable recovery could "
|
||||
f"terminalize its execution state (exit {returncode})"
|
||||
)
|
||||
|
||||
|
||||
def _wait_for_external_cron_worker(
|
||||
process: subprocess.Popen,
|
||||
*,
|
||||
execution_id: str,
|
||||
job_id: Optional[str] = None,
|
||||
handoff_files: tuple[Path, ...] = (),
|
||||
) -> bool:
|
||||
try:
|
||||
return _wait_for_external_cron_worker_body(
|
||||
process, execution_id=execution_id
|
||||
)
|
||||
finally:
|
||||
if job_id is not None:
|
||||
with _running_lock:
|
||||
_restart_safe_waiter_job_ids.discard(job_id)
|
||||
# The execution is terminal or its worker is dead: nobody will read a
|
||||
# payload or acknowledgement left behind by a late/unread handoff.
|
||||
for stale in handoff_files:
|
||||
try:
|
||||
stale.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _launch_external_cron_worker(job: dict) -> bool:
|
||||
"""Launch *job* outside a managed gateway cgroup when required.
|
||||
|
||||
Returns ``False`` when the caller is not a managed systemd gateway and the
|
||||
existing in-process path should be used. In managed topology, failure to
|
||||
establish the transient scope raises: falling back would recreate the
|
||||
restart interruption this handoff exists to prevent.
|
||||
"""
|
||||
execution_id = str(job["execution_id"])
|
||||
job_id = str(job["id"])
|
||||
handoff_dir = _get_hermes_home() / "cron" / "external-workers"
|
||||
payload_path = handoff_dir / f"{execution_id}.json"
|
||||
ack_path = handoff_dir / f"{execution_id}.ready"
|
||||
command = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"cron.scheduler",
|
||||
"--external-worker-file",
|
||||
str(payload_path),
|
||||
"--ack-file",
|
||||
str(ack_path),
|
||||
]
|
||||
|
||||
from agent.secret_scope import is_multiplex_active
|
||||
from tools.environments.local import build_subprocess_env
|
||||
from tools.process_registry import restart_safe_gateway_child_argv
|
||||
|
||||
multiplex_active = is_multiplex_active()
|
||||
scoped_command = restart_safe_gateway_child_argv(
|
||||
command,
|
||||
unit_suffix=f"cron-{job_id}-exec-{execution_id}",
|
||||
)
|
||||
if scoped_command == command:
|
||||
return False
|
||||
|
||||
if mark_execution_handoff_pending(execution_id) is None:
|
||||
raise RuntimeError(
|
||||
"cron execution claim changed before external worker handoff"
|
||||
)
|
||||
|
||||
_ensure_cron_dir(handoff_dir)
|
||||
try:
|
||||
handoff_dir.chmod(0o700)
|
||||
except OSError:
|
||||
pass
|
||||
fd = os.open(payload_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as payload_file:
|
||||
json.dump(
|
||||
{
|
||||
"job": job,
|
||||
"profile_home": str(_get_hermes_home().resolve()),
|
||||
"multiplex_active": multiplex_active,
|
||||
},
|
||||
payload_file,
|
||||
)
|
||||
payload_file.flush()
|
||||
os.fsync(payload_file.fileno())
|
||||
except BaseException:
|
||||
payload_path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
worker_env = build_subprocess_env(
|
||||
scrub_secrets=multiplex_active,
|
||||
inherit_profile_home=True,
|
||||
extra={"HERMES_HOME": str(_get_hermes_home().resolve())},
|
||||
)
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
scoped_command,
|
||||
cwd=str(Path(__file__).resolve().parent.parent),
|
||||
env=worker_env,
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
start_new_session=True,
|
||||
creationflags=windows_hide_flags(),
|
||||
)
|
||||
except BaseException:
|
||||
payload_path.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
with _running_lock:
|
||||
_restart_safe_waiter_job_ids.add(job_id)
|
||||
|
||||
deadline = time.monotonic() + 5.0
|
||||
while time.monotonic() < deadline:
|
||||
if ack_path.exists():
|
||||
try:
|
||||
acknowledgement = json.loads(ack_path.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Cron external worker %s published an unreadable acknowledgement; "
|
||||
"treating handoff as ownership-uncertain",
|
||||
execution_id,
|
||||
)
|
||||
return _wait_for_external_cron_worker(
|
||||
process,
|
||||
execution_id=execution_id,
|
||||
job_id=job_id,
|
||||
handoff_files=(payload_path,),
|
||||
)
|
||||
finally:
|
||||
ack_path.unlink(missing_ok=True)
|
||||
if (
|
||||
not isinstance(acknowledgement, dict)
|
||||
or acknowledgement.get("execution_id") != execution_id
|
||||
):
|
||||
logger.error(
|
||||
"Cron external worker acknowledgement mismatch for %s; "
|
||||
"treating handoff as ownership-uncertain",
|
||||
execution_id,
|
||||
)
|
||||
return _wait_for_external_cron_worker(
|
||||
process,
|
||||
execution_id=execution_id,
|
||||
job_id=job_id,
|
||||
handoff_files=(payload_path,),
|
||||
)
|
||||
logger.info(
|
||||
"Cron job '%s' handed to restart-safe worker pid=%s execution=%s",
|
||||
job_id,
|
||||
acknowledgement.get("pid"),
|
||||
execution_id,
|
||||
)
|
||||
return _wait_for_external_cron_worker(
|
||||
process,
|
||||
execution_id=execution_id,
|
||||
job_id=job_id,
|
||||
handoff_files=(payload_path,),
|
||||
)
|
||||
returncode = process.poll()
|
||||
if returncode is not None:
|
||||
with _running_lock:
|
||||
_restart_safe_waiter_job_ids.discard(job_id)
|
||||
payload_path.unlink(missing_ok=True)
|
||||
raise RuntimeError(
|
||||
f"cron external worker exited before ownership acknowledgement "
|
||||
f"(exit {returncode})"
|
||||
)
|
||||
time.sleep(0.05)
|
||||
|
||||
# The child may have adopted the durable row just before publishing its
|
||||
# acknowledgement. Never fall back to in-process execution on an uncertain
|
||||
# handoff: that could duplicate side effects. The execution owner/dead-owner
|
||||
# recovery ledger remains the authority.
|
||||
logger.warning(
|
||||
"Cron external worker for job '%s' did not acknowledge within 5s; "
|
||||
"leaving the durable execution claim untouched",
|
||||
job_id,
|
||||
)
|
||||
return _wait_for_external_cron_worker(
|
||||
process,
|
||||
execution_id=execution_id,
|
||||
job_id=job_id,
|
||||
handoff_files=(payload_path, ack_path),
|
||||
)
|
||||
|
||||
|
||||
def _run_external_worker_payload(payload_path: Path, ack_path: Path) -> bool:
|
||||
"""Adopt and execute one gateway-dispatched cron payload.
|
||||
|
||||
The execution row is created by the gateway before spawn, then transferred
|
||||
here before the ready acknowledgement is published. No side effect runs
|
||||
unless that durable ownership transfer succeeds.
|
||||
"""
|
||||
try:
|
||||
payload = json.loads(payload_path.read_text(encoding="utf-8"))
|
||||
job = payload["job"]
|
||||
profile_home = Path(payload["profile_home"]).resolve()
|
||||
execution_id = str(job["execution_id"])
|
||||
except Exception:
|
||||
logger.exception("Cron external worker could not load payload %s", payload_path)
|
||||
return False
|
||||
finally:
|
||||
try:
|
||||
payload_path.unlink(missing_ok=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
from agent.secret_scope import (
|
||||
build_profile_secret_scope,
|
||||
is_multiplex_active,
|
||||
reset_secret_scope,
|
||||
set_multiplex_active,
|
||||
set_secret_scope,
|
||||
)
|
||||
from cron.executions import adopt_claimed_execution
|
||||
from hermes_cli.env_loader import hydrate_profile_secret_sources
|
||||
from hermes_constants import (
|
||||
reset_hermes_home_override,
|
||||
set_hermes_home_override,
|
||||
)
|
||||
|
||||
home_token = set_hermes_home_override(profile_home)
|
||||
previous_multiplex = is_multiplex_active()
|
||||
multiplex_active = bool(payload.get("multiplex_active", False))
|
||||
set_multiplex_active(multiplex_active)
|
||||
hydrate_profile_secret_sources(profile_home)
|
||||
secret_token = set_secret_scope(build_profile_secret_scope(profile_home))
|
||||
try:
|
||||
with use_cron_store(profile_home):
|
||||
if adopt_claimed_execution(execution_id) is None:
|
||||
logger.error(
|
||||
"Cron external worker refused execution %s: durable ownership "
|
||||
"could not be established",
|
||||
execution_id,
|
||||
)
|
||||
return False
|
||||
try:
|
||||
ack_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd = os.open(ack_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as ack_file:
|
||||
json.dump({"pid": os.getpid(), "execution_id": execution_id}, ack_file)
|
||||
ack_file.flush()
|
||||
os.fsync(ack_file.fileno())
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Cron external worker could not publish ready acknowledgement for %s",
|
||||
execution_id,
|
||||
)
|
||||
return False
|
||||
old_external_execution = os.environ.get("_HERMES_CRON_EXTERNAL_WORKER")
|
||||
os.environ["_HERMES_CRON_EXTERNAL_WORKER"] = execution_id
|
||||
try:
|
||||
return run_one_job(job, adapters=None, loop=None, verbose=False)
|
||||
finally:
|
||||
if old_external_execution is None:
|
||||
os.environ.pop("_HERMES_CRON_EXTERNAL_WORKER", None)
|
||||
else:
|
||||
os.environ["_HERMES_CRON_EXTERNAL_WORKER"] = old_external_execution
|
||||
finally:
|
||||
reset_secret_scope(secret_token)
|
||||
set_multiplex_active(previous_multiplex)
|
||||
reset_hermes_home_override(home_token)
|
||||
|
||||
|
||||
def _notify_provider_jobs_changed() -> None:
|
||||
"""Best-effort: tell the active scheduler provider the job set changed. Call AFTER a successful
|
||||
store mutation so an external provider can re-provision/cancel the one-shot; no-op for the
|
||||
@@ -3174,10 +3545,6 @@ def tick(
|
||||
_release_tick_lock(lock_fd)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
tick(verbose=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Split modules — re-exported so ``scheduler.<name>`` keeps resolving (and stays the single
|
||||
# monkeypatch target).
|
||||
@@ -3221,3 +3588,27 @@ from cron.scheduler_preflight import ( # noqa: E402,F401
|
||||
_is_transient_provider_resolve_error, _preflight_check_delivery, _preflight_check_provider_key,
|
||||
_preflight_check_skills, _preflight_job_config, _primary_profile_routes_for_current_home,
|
||||
)
|
||||
|
||||
|
||||
# `python -m cron.scheduler` entry: MUST stay below the split-module re-exports so the worker /
|
||||
# tick paths see every name the module re-exports.
|
||||
if __name__ == "__main__":
|
||||
if "--external-worker-file" in sys.argv:
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(add_help=False)
|
||||
parser.add_argument("--external-worker-file", type=Path, required=True)
|
||||
parser.add_argument("--ack-file", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
# The gateway spawns this worker with stdout/stderr on DEVNULL; without
|
||||
# a handler every adoption/ack failure below would be invisible.
|
||||
try:
|
||||
from hermes_logging import setup_logging
|
||||
|
||||
setup_logging(hermes_home=_get_hermes_home(), mode="cron")
|
||||
except Exception:
|
||||
pass
|
||||
raise SystemExit(
|
||||
0 if _run_external_worker_payload(args.external_worker_file, args.ack_file) else 1
|
||||
)
|
||||
tick(verbose=True)
|
||||
|
||||
@@ -1539,6 +1539,19 @@ def _deliver_result(
|
||||
targets = _sched._resolve_delivery_targets(job, for_failure=for_failure)
|
||||
if not targets:
|
||||
return _unresolved_delivery_outcome(job, for_failure)
|
||||
|
||||
# Restart-safe workers have no live gateway adapters: hand the send back through a durable
|
||||
# queue so the current or replacement gateway performs it with relay/E2EE parity. The execution
|
||||
# id is the idempotency key (the queue never retries an uncertain claimed send). Match on THIS
|
||||
# job's own attempt: a worker's script may dispatch another job in-process (`hermes cron run`),
|
||||
# and that nested delivery must not be keyed under the outer execution id.
|
||||
external_execution = os.environ.get("_HERMES_CRON_EXTERNAL_WORKER", "")
|
||||
if (external_execution and adapters is None
|
||||
and external_execution == str(job.get("execution_id") or "")):
|
||||
from cron.delivery_queue import enqueue_and_wait
|
||||
|
||||
return enqueue_and_wait(external_execution, job, content, for_failure=for_failure)
|
||||
|
||||
from gateway.config import load_gateway_config
|
||||
|
||||
# Wrap with header/footer unless cron.wrap_response: false.
|
||||
|
||||
@@ -18,18 +18,22 @@ as a no-op compatibility alias for existing installation commands.
|
||||
> longer activate exporters. Without the new variable, Hermes does not run
|
||||
> Relay plugin discovery, configuration layering, middleware, or exporters.
|
||||
|
||||
Hermes requires NeMo Relay 0.7.1 or later within the 0.7 release line. That
|
||||
release establishes the lossless provider-codec contract used for Anthropic
|
||||
Messages, OpenAI Chat Completions, and OpenAI Responses requests.
|
||||
Hermes requires NeMo Relay 0.8.3 or later within the 0.8 release line. That
|
||||
line provides the provider-codec and canonical tool-result contracts Hermes
|
||||
uses for managed provider and tool calls.
|
||||
|
||||
## Runtime Dependency and Data Boundary
|
||||
|
||||
Hermes installs the platform-specific `nemo-relay` native wheel from the
|
||||
bounded `>=0.7.1,<0.8` dependency range. The published package is built from
|
||||
bounded `>=0.8.3,<0.9` dependency range. The published package is built from
|
||||
the [NVIDIA NeMo Relay repository](https://github.com/NVIDIA/NeMo-Relay).
|
||||
Unsupported platforms use the explicit no-op runtime described above rather
|
||||
than downloading a different implementation.
|
||||
|
||||
Operator-supplied typed native plugins must be rebuilt for Relay 0.8. `grpc-v1`
|
||||
workers must be regenerated and rebuilt when they use tool callbacks, tool
|
||||
execution intercepts, or manual tool-end APIs.
|
||||
|
||||
When Relay managed execution is active, the provider request and response pass
|
||||
through that native module in the Hermes process so configured interceptors can
|
||||
operate on the real call. This is separate from the shared-metrics data
|
||||
@@ -59,12 +63,12 @@ opt-in. Set `HERMES_NEMO_RELAY_PLUGINS_TOML` to a selected `plugins.toml` to
|
||||
activate configured middleware, exporters, or dynamic plugins. When the
|
||||
variable is unset, Hermes does not invoke Relay's plugin initializer, so Relay
|
||||
does not perform plugin configuration discovery or layering. When it is set
|
||||
and the selected file loads successfully, Relay performs its normal static
|
||||
`plugins.toml` discovery and layers the selected static configuration over the
|
||||
discovered configuration. Dynamic `[[plugins.dynamic]]` records are loaded
|
||||
from the selected file only. If the selected file cannot be loaded, Hermes
|
||||
reports the error and does not invoke Relay initialization or fall back to
|
||||
ambient discovery.
|
||||
and the selected file loads successfully, Relay discovers supported user and
|
||||
system `plugins.toml` files and layers the selected static configuration over
|
||||
them. Repository-local `.nemo-relay/plugins.toml` files are ignored. Dynamic
|
||||
`[[plugins.dynamic]]` records are loaded from the selected file only. If the
|
||||
selected file cannot be loaded, Hermes reports the error and does not invoke
|
||||
Relay initialization or fall back to ambient discovery.
|
||||
|
||||
## Session-Span Segmentation for Continuous Sessions
|
||||
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fan-out resource benchmark for hermes-agent.
|
||||
|
||||
Spawns N in-process child AIAgents via the REAL delegate_task code path
|
||||
(tools.delegate_tool.delegate_task) against a local fake OpenAI server, with
|
||||
children editing python files across W distinct git worktrees so the LSP
|
||||
(pyright) path is exercised for real. Measures, for the host process:
|
||||
|
||||
threads, RSS MB, open fds, TCP ESTAB sockets, child processes (pyright,
|
||||
kernels), state.db growth, wall time.
|
||||
|
||||
Usage:
|
||||
python evals/fanout_resource_bench.py --repo <checkout> --children 24 --worktrees 6 --label before
|
||||
|
||||
Prints one JSON line; append several and compare with --compare a.json b.json.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import http.server
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Fake OpenAI chat-completions server: each child does
|
||||
# turn 1: call write_file on <its worktree>/hermes_cli/bench_<i>.py
|
||||
# turn 2: call execute_code print(1)
|
||||
# turn 3: final text
|
||||
# --------------------------------------------------------------------------
|
||||
_REPLY_KB = [0]
|
||||
|
||||
|
||||
class _Fake(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, format, *args): # quiet
|
||||
pass
|
||||
|
||||
def do_POST(self):
|
||||
n = int(self.headers.get("Content-Length", 0))
|
||||
body = json.loads(self.rfile.read(n) or b"{}")
|
||||
msgs = body.get("messages", [])
|
||||
goal = next((m["content"] for m in msgs if m.get("role") == "user"), "")
|
||||
try:
|
||||
plan = json.loads(goal)
|
||||
except Exception:
|
||||
plan = {}
|
||||
n_tool = sum(1 for m in msgs if m.get("role") == "tool")
|
||||
if n_tool == 0 and plan.get("file"):
|
||||
tc = {"id": "c1", "type": "function", "function": {"name": "write_file", "arguments": json.dumps({"path": plan["file"], "content": "import os\nx: int = 'bad'\n"})}}
|
||||
msg = {"role": "assistant", "content": None, "tool_calls": [tc]}
|
||||
finish = "tool_calls"
|
||||
elif n_tool == 1 and plan.get("file"):
|
||||
tc = {"id": "c2", "type": "function", "function": {"name": "execute_code", "arguments": json.dumps({"code": "print(1)"})}}
|
||||
msg = {"role": "assistant", "content": None, "tool_calls": [tc]}
|
||||
finish = "tool_calls"
|
||||
else:
|
||||
msg = {"role": "assistant", "content": "done " + ("x" * (_REPLY_KB[0] * 1024))}
|
||||
finish = "stop"
|
||||
if body.get("stream") is True:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/event-stream")
|
||||
self.end_headers()
|
||||
delta = {"role": "assistant", "content": msg.get("content") or ""}
|
||||
if msg.get("tool_calls"):
|
||||
tc = msg["tool_calls"][0]
|
||||
delta["tool_calls"] = [{"index": 0, "id": tc["id"], "type": "function", "function": tc["function"]}]
|
||||
for chunk in (
|
||||
{"id": "m", "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": delta, "finish_reason": None}]},
|
||||
{"id": "m", "object": "chat.completion.chunk", "choices": [{"index": 0, "delta": {}, "finish_reason": finish}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}},
|
||||
):
|
||||
self.wfile.write(f"data: {json.dumps(chunk)}\n\n".encode())
|
||||
self.wfile.write(b"data: [DONE]\n\n")
|
||||
self.wfile.flush()
|
||||
return
|
||||
resp = {"id": "x", "object": "chat.completion", "created": 0, "model": body.get("model", "m"),
|
||||
"choices": [{"index": 0, "message": msg, "finish_reason": finish}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}}
|
||||
data = json.dumps(resp).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
|
||||
def _serve():
|
||||
srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _Fake)
|
||||
threading.Thread(target=srv.serve_forever, daemon=True).start()
|
||||
return srv
|
||||
|
||||
|
||||
def _count_live(cls_name: str) -> int:
|
||||
import gc
|
||||
return sum(1 for o in gc.get_objects() if type(o).__name__ == cls_name)
|
||||
|
||||
|
||||
def _snap(pid: int, db_path: str) -> dict:
|
||||
st = open(f"/proc/{pid}/status", encoding="utf-8").read()
|
||||
g = lambda k: int(st.split(k + ":")[1].split()[0])
|
||||
tcp = subprocess.run(f"ss -tanp 2>/dev/null | grep -c 'pid={pid},'", shell=True, capture_output=True, text=True).stdout.strip()
|
||||
kids = subprocess.run(["ps", "-o", "args=", "--ppid", str(pid)], capture_output=True, text=True).stdout
|
||||
return {
|
||||
"threads": g("Threads"), "rss_mb": g("VmRSS") // 1024, "fds": len(os.listdir(f"/proc/{pid}/fd")),
|
||||
"tcp": int(tcp or 0), "pyright": kids.count("pyright"), "kernels": kids.count("hermes_kernel_runner"),
|
||||
"db_mb": round(os.path.getsize(db_path) / 2**20, 1) if os.path.exists(db_path) else 0,
|
||||
"httpx_clients": _count_live("Client"), "transports": _count_live("HTTPTransport"), "session_dbs": _count_live("SessionDB"), "live_agents": _count_live("AIAgent"),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--repo", required=True)
|
||||
ap.add_argument("--children", type=int, default=24)
|
||||
ap.add_argument("--worktrees", type=int, default=6)
|
||||
ap.add_argument("--label", default="")
|
||||
ap.add_argument("--out", default="")
|
||||
ap.add_argument("--compare", nargs=2)
|
||||
ap.add_argument("--reply-kb", type=int, default=0, help="pad each child's final reply to N KB (transcript-size realism)")
|
||||
a = ap.parse_args()
|
||||
if a.compare:
|
||||
b, c = (json.load(open(p, encoding="utf-8")) for p in a.compare)
|
||||
print(f"| metric | {b['label']} | {c['label']} | delta |\n|---|---|---|---|")
|
||||
for k in ("threads", "rss_mb", "fds", "tcp", "pyright", "kernels", "db_mb", "httpx_clients", "transports", "session_dbs"):
|
||||
bv, cv = b["peak"][k], c["peak"][k]
|
||||
print(f"| {k} (peak) | {bv} | {cv} | {cv - bv:+} |")
|
||||
for k in ("rss_mb", "live_agents", "db_mb", "threads"):
|
||||
bv, cv = b["after"].get(k), c["after"].get(k)
|
||||
if bv is not None and cv is not None:
|
||||
print(f"| {k} (after, children done) | {bv} | {cv} | {cv - bv:+} |")
|
||||
print(f"| wall_s | {b['wall_s']} | {c['wall_s']} | {c['wall_s'] - b['wall_s']:+.1f} |")
|
||||
return
|
||||
|
||||
_REPLY_KB[0] = a.reply_kb
|
||||
home = tempfile.mkdtemp(prefix="hermes_bench_home_")
|
||||
os.environ["HERMES_HOME"] = home
|
||||
os.environ["TERMINAL_ENV"] = "local"
|
||||
os.environ.pop("OPENROUTER_API_KEY", None)
|
||||
sys.path.insert(0, a.repo)
|
||||
os.chdir(a.repo)
|
||||
pyright = shutil.which("pyright-langserver", path=os.path.expanduser("~/.hermes/lsp/bin") + os.pathsep + os.environ.get("PATH", ""))
|
||||
with open(os.path.join(home, "config.yaml"), "w", encoding="utf-8") as f:
|
||||
f.write("lsp:\n enabled: true\n wait_timeout: 5.0\n install_strategy: manual\n")
|
||||
if pyright:
|
||||
f.write(f" servers:\n pyright:\n command: [{json.dumps(pyright)}, \"--stdio\"]\n")
|
||||
f.write("delegation:\n max_concurrent_children: 64\n subagent_auto_approve: true\n")
|
||||
|
||||
# W git worktrees, each a real python project (pyproject + package) so pyright roots resolve.
|
||||
wts = []
|
||||
base = tempfile.mkdtemp(prefix="hermes_bench_wt_")
|
||||
for w in range(a.worktrees):
|
||||
d = os.path.join(base, f"wt{w}")
|
||||
os.makedirs(os.path.join(d, "hermes_cli"))
|
||||
subprocess.run(["git", "init", "-q", d], check=True)
|
||||
open(os.path.join(d, "pyproject.toml"), "w", encoding="utf-8").write("[project]\nname='b'\n")
|
||||
open(os.path.join(d, "hermes_cli", "__init__.py"), "w", encoding="utf-8").write("")
|
||||
wts.append(d)
|
||||
|
||||
srv = _serve()
|
||||
port = srv.server_address[1]
|
||||
from run_agent import AIAgent
|
||||
from tools import delegate_tool
|
||||
|
||||
from hermes_state import SessionDB
|
||||
db_path = os.path.join(home, "state.db")
|
||||
from pathlib import Path
|
||||
session_db = SessionDB(db_path=Path(db_path))
|
||||
parent = AIAgent(api_key="bench", base_url=f"http://127.0.0.1:{port}/v1", model="bench-model",
|
||||
quiet_mode=True, skip_context_files=True, skip_memory=True,
|
||||
enabled_toolsets=["delegation", "file", "code_execution"],
|
||||
session_db=session_db, session_id="bench-root")
|
||||
# Children reference parent_session_id; the parent row is normally created
|
||||
# lazily on the parent's first turn, which this harness never runs.
|
||||
parent._ensure_db_session()
|
||||
pid = os.getpid()
|
||||
before = _snap(pid, db_path)
|
||||
peak = dict(before)
|
||||
stop = threading.Event()
|
||||
|
||||
def sampler():
|
||||
while not stop.wait(0.5):
|
||||
s = _snap(pid, db_path)
|
||||
for k, v in s.items():
|
||||
peak[k] = max(peak[k], v)
|
||||
threading.Thread(target=sampler, daemon=True).start()
|
||||
|
||||
tasks = [{"goal": json.dumps({"file": os.path.join(wts[i % len(wts)], "hermes_cli", f"bench_{i}.py")}),
|
||||
"context": "bench"} for i in range(a.children)]
|
||||
t0 = time.monotonic()
|
||||
res = delegate_tool.delegate_task(tasks=tasks, parent_agent=parent, background=False)
|
||||
wall = round(time.monotonic() - t0, 1)
|
||||
if os.environ.get("BENCH_DEBUG"):
|
||||
sys.__stderr__.write(str(res)[:3000] + "\n")
|
||||
time.sleep(2.0)
|
||||
stop.set()
|
||||
import gc
|
||||
gc.collect()
|
||||
after = _snap(pid, db_path)
|
||||
try:
|
||||
parsed = json.loads(res)
|
||||
items = parsed if isinstance(parsed, list) else parsed.get("results") or parsed.get("tasks") or []
|
||||
ok = sum(1 for r in items if str(r.get("status", "")) in ("completed", "success"))
|
||||
except Exception:
|
||||
ok = None
|
||||
try:
|
||||
session_db.checkpoint() if hasattr(session_db, "checkpoint") else None
|
||||
except Exception:
|
||||
pass
|
||||
out = {"label": a.label, "children": a.children, "worktrees": a.worktrees, "ok": ok, "wall_s": wall,
|
||||
"before": before, "peak": peak, "after": after}
|
||||
sys.__stderr__.write("BENCH " + json.dumps(out) + "\n"); sys.__stderr__.flush()
|
||||
if a.out:
|
||||
open(a.out, "w", encoding="utf-8").write(json.dumps(out, indent=1))
|
||||
try:
|
||||
from agent.lsp import shutdown_service
|
||||
shutdown_service()
|
||||
from tools.code_kernel import shutdown_all_kernels
|
||||
shutdown_all_kernels()
|
||||
except Exception:
|
||||
pass
|
||||
shutil.rmtree(base, ignore_errors=True)
|
||||
os._exit(0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -423,7 +423,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
|
||||
from gateway.config import Platform, PlatformConfig
|
||||
from gateway.platforms.helpers import fence_state_after
|
||||
from gateway.session import SessionSource, build_session_key
|
||||
from gateway.session import SessionSource, TranscriptReadError, build_session_key
|
||||
from hermes_constants import get_default_hermes_root, get_hermes_dir, get_hermes_home
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -599,6 +599,11 @@ def cache_image_from_bytes(data: bytes, ext: str = ".jpg") -> str:
|
||||
return _write_cache_file(get_image_cache_dir(), "img", ext, data)
|
||||
|
||||
|
||||
async def cache_image_from_bytes_async(data: bytes, ext: str = ".jpg") -> str:
|
||||
"""Cache image bytes without blocking the caller's event loop."""
|
||||
return await asyncio.to_thread(cache_image_from_bytes, data, ext)
|
||||
|
||||
|
||||
async def _cache_media_from_url(url: str, ext: str, retries: int, *, media_type: str, accept: str,
|
||||
cache_fn, log_label: str) -> str:
|
||||
"""Shared downloader behind ``cache_*_from_url``: SSRF-checked (pre-flight + per-redirect;
|
||||
@@ -616,7 +621,7 @@ async def _cache_media_from_url(url: str, ext: str, retries: int, *, media_type:
|
||||
async with client.stream("GET", url, headers=headers) as response:
|
||||
response.raise_for_status()
|
||||
content = await _read_httpx_body_with_limit(response, media_type=media_type)
|
||||
return cache_fn(content, ext)
|
||||
return await asyncio.to_thread(cache_fn, content, ext)
|
||||
except (httpx.TimeoutException, httpx.HTTPStatusError) as exc:
|
||||
if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code < 429:
|
||||
raise
|
||||
@@ -662,6 +667,11 @@ def cache_audio_from_bytes(data: bytes, ext: str = ".ogg") -> str:
|
||||
return _write_cache_file(get_audio_cache_dir(), "audio", sniff_audio_ext(data, ext), data)
|
||||
|
||||
|
||||
async def cache_audio_from_bytes_async(data: bytes, ext: str = ".ogg") -> str:
|
||||
"""Cache audio bytes without blocking the caller's event loop."""
|
||||
return await asyncio.to_thread(cache_audio_from_bytes, data, ext)
|
||||
|
||||
|
||||
async def cache_audio_from_url(url: str, ext: str = ".ogg", retries: int = 2) -> str:
|
||||
"""Download an audio URL into the audio cache; return the absolute path."""
|
||||
return await _cache_media_from_url(
|
||||
@@ -685,6 +695,11 @@ def cache_video_from_bytes(data: bytes, ext: str = ".mp4") -> str:
|
||||
return _write_cache_file(get_video_cache_dir(), "video", ext, data)
|
||||
|
||||
|
||||
async def cache_video_from_bytes_async(data: bytes, ext: str = ".mp4") -> str:
|
||||
"""Cache video bytes without blocking the caller's event loop."""
|
||||
return await asyncio.to_thread(cache_video_from_bytes, data, ext)
|
||||
|
||||
|
||||
# Document / screenshot cache utilities (same pattern; referenced by local path).
|
||||
DOCUMENT_CACHE_DIR = get_hermes_dir("cache/documents", "document_cache")
|
||||
SCREENSHOT_CACHE_DIR = get_hermes_dir("cache/screenshots", "browser_screenshots")
|
||||
@@ -1272,6 +1287,11 @@ def cache_document_from_bytes(data: bytes, filename: str) -> str:
|
||||
return str(filepath)
|
||||
|
||||
|
||||
async def cache_document_from_bytes_async(data: bytes, filename: str) -> str:
|
||||
"""Cache document bytes without blocking the caller's event loop."""
|
||||
return await asyncio.to_thread(cache_document_from_bytes, data, filename)
|
||||
|
||||
|
||||
# Unified media caching: classify attachment bytes by ext/MIME, route to cache_*_from_bytes.
|
||||
@dataclass
|
||||
class CachedMedia:
|
||||
@@ -1333,6 +1353,23 @@ def cache_media_bytes(data: bytes, *, filename: str = "", mime_type: str = "",
|
||||
return CachedMedia(to_agent_visible_cache_path(path), out_mime, "document", display or fallback_name)
|
||||
|
||||
|
||||
async def cache_media_bytes_async(
|
||||
data: bytes,
|
||||
*,
|
||||
filename: str = "",
|
||||
mime_type: str = "",
|
||||
default_kind: Optional[str] = None,
|
||||
) -> Optional[CachedMedia]:
|
||||
"""Classify and cache attachment bytes without blocking the event loop."""
|
||||
return await asyncio.to_thread(
|
||||
cache_media_bytes,
|
||||
data,
|
||||
filename=filename,
|
||||
mime_type=mime_type,
|
||||
default_kind=default_kind,
|
||||
)
|
||||
|
||||
|
||||
class MessageType(Enum):
|
||||
"""Types of incoming messages."""
|
||||
TEXT = "text"
|
||||
@@ -1887,11 +1924,38 @@ class BasePlatformAdapter(ABC):
|
||||
def set_fatal_error_handler(self, handler: Callable[["BasePlatformAdapter"], Awaitable[None] | None]) -> None:
|
||||
self._fatal_error_handler = handler
|
||||
|
||||
#: Published when an adapter is installed and running but its receive
|
||||
#: path is not yet confirmed (e.g. Telegram polling has not proven a
|
||||
#: getUpdates round-trip). Same ``retrying`` platform_state the runner
|
||||
#: uses for queued reconnects, so readers see "not delivering" (#101391).
|
||||
DEGRADED_STATUS_MESSAGE = "connected but not yet confirmed active; recovering in background"
|
||||
|
||||
@property
|
||||
def send_path_degraded(self) -> bool:
|
||||
"""True while connect() succeeded but delivery is not confirmed.
|
||||
|
||||
Adapters with a separately-proven receive path override this; the
|
||||
default adapter is either connected or not.
|
||||
"""
|
||||
return False
|
||||
|
||||
def _mark_connected(self) -> None:
|
||||
self._running = True
|
||||
self._fatal_error_code = self._fatal_error_message = None
|
||||
self._fatal_error_retryable = True
|
||||
self._write_runtime_status_safe("connected", platform_state="connected", error_code=None, error_message=None)
|
||||
if self.send_path_degraded:
|
||||
self._mark_degraded()
|
||||
else:
|
||||
self._write_runtime_status_safe("connected", platform_state="connected", error_code=None, error_message=None)
|
||||
|
||||
def _mark_degraded(self) -> None:
|
||||
"""Publish ``retrying`` for a running adapter whose delivery path is unproven."""
|
||||
self._write_runtime_status_safe(
|
||||
"connected_degraded",
|
||||
platform_state="retrying",
|
||||
error_code=None,
|
||||
error_message=self.DEGRADED_STATUS_MESSAGE,
|
||||
)
|
||||
|
||||
def _mark_disconnected(self) -> None:
|
||||
self._running = False
|
||||
@@ -2172,6 +2236,12 @@ class BasePlatformAdapter(ABC):
|
||||
peek = getattr(store, "peek_session_id", None)
|
||||
session_id = peek(session_key) if callable(peek) else None
|
||||
transcript = store.load_transcript(session_id or session_key)
|
||||
except TranscriptReadError:
|
||||
logger.warning(
|
||||
"Transcript read failed for session %s; media dedup runs "
|
||||
"with no history this turn (#100788)", session_key,
|
||||
)
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
if not transcript:
|
||||
|
||||
@@ -20,7 +20,7 @@ from gateway.config import Platform, PlatformConfig
|
||||
from gateway.platforms._shared import get_scoped_secret as _get_scoped_secret
|
||||
from gateway.platforms.base import (
|
||||
BasePlatformAdapter, MessageEvent, MessageType, SendResult,
|
||||
cache_image_from_bytes, cache_audio_from_bytes, cache_document_from_bytes)
|
||||
cache_image_from_bytes_async, cache_audio_from_bytes_async, cache_document_from_bytes_async)
|
||||
from .media_cache import ext_for_mime
|
||||
from gateway.platforms.helpers import compile_mention_patterns, strip_markdown
|
||||
from utils import TRUTHY_STRINGS
|
||||
@@ -471,11 +471,11 @@ class BlueBubblesAdapter(BasePlatformAdapter):
|
||||
data = resp.content
|
||||
mime = (att_meta.get("mimeType") or "").lower()
|
||||
if mime.startswith("image/"):
|
||||
return cache_image_from_bytes(data, _closed_ext(mime, _BLUEBUBBLES_IMAGE_EXT_OVERRIDES, ".jpg"))
|
||||
return await cache_image_from_bytes_async(data, _closed_ext(mime, _BLUEBUBBLES_IMAGE_EXT_OVERRIDES, ".jpg"))
|
||||
if mime.startswith("audio/"):
|
||||
return cache_audio_from_bytes(data, _closed_ext(mime, _BLUEBUBBLES_AUDIO_EXT_OVERRIDES, ".mp3"))
|
||||
return await cache_audio_from_bytes_async(data, _closed_ext(mime, _BLUEBUBBLES_AUDIO_EXT_OVERRIDES, ".mp3"))
|
||||
# Videos, documents, and everything else
|
||||
return cache_document_from_bytes(data, att_meta.get("transferName", "") or f"file_{uuid.uuid4().hex[:8]}")
|
||||
return await cache_document_from_bytes_async(data, att_meta.get("transferName", "") or f"file_{uuid.uuid4().hex[:8]}")
|
||||
except Exception as exc:
|
||||
logger.warning("[bluebubbles] failed to download attachment %s: %s", _redact(att_guid), exc)
|
||||
return None
|
||||
|
||||
@@ -40,7 +40,7 @@ except ImportError:
|
||||
from gateway.config import Platform, PlatformConfig
|
||||
from gateway.platforms.base import (
|
||||
gateway_trust_env, BasePlatformAdapter, MessageEvent, MessageType, SendResult,
|
||||
_ssrf_redirect_guard, cache_document_from_bytes, cache_image_from_bytes)
|
||||
_ssrf_redirect_guard, cache_document_from_bytes_async, cache_image_from_bytes_async)
|
||||
from gateway.platforms.helpers import strip_markdown
|
||||
from gateway.platforms.media_cache import ext_for_mime
|
||||
|
||||
@@ -976,12 +976,12 @@ class QQAdapter(BasePlatformAdapter):
|
||||
if content_type.startswith("image/"):
|
||||
# Historical qqbot mapping: trust mimetypes' guess (never the shared table), fall back to .jpg.
|
||||
ext = ext_for_mime(content_type, use_defaults=False, use_mimetypes=True, fallback=".jpg") or ".jpg"
|
||||
return cache_image_from_bytes(data, ext)
|
||||
return await cache_image_from_bytes_async(data, ext)
|
||||
if content_type == "voice" or content_type.startswith("audio/"):
|
||||
# QQ voice is usually .amr/.silk — convert to .wav for STT engines.
|
||||
return await self._convert_audio_to_wav(data, url)
|
||||
filename = original_name or Path(urlparse(url).path).name or "qq_attachment"
|
||||
return cache_document_from_bytes(data, filename)
|
||||
return await cache_document_from_bytes_async(data, filename)
|
||||
|
||||
@staticmethod
|
||||
def _is_voice_content_type(content_type: str, filename: str) -> bool:
|
||||
@@ -1245,16 +1245,16 @@ class QQAdapter(BasePlatformAdapter):
|
||||
try:
|
||||
if not await convert(src_path, wav_path):
|
||||
logger.warning("[%s] audio conversion failed for %s (format=%s)", self._log_tag, source_url[:60], ext)
|
||||
return cache_document_from_bytes(audio_data, f"qq_voice{ext}")
|
||||
return await cache_document_from_bytes_async(audio_data, f"qq_voice{ext}")
|
||||
except Exception:
|
||||
return cache_document_from_bytes(audio_data, f"qq_voice{ext}")
|
||||
return await cache_document_from_bytes_async(audio_data, f"qq_voice{ext}")
|
||||
finally:
|
||||
self._unlink_quiet(src_path)
|
||||
|
||||
try:
|
||||
wav_data = Path(wav_path).read_bytes()
|
||||
os.unlink(wav_path)
|
||||
return cache_document_from_bytes(wav_data, "qq_voice.wav")
|
||||
return await cache_document_from_bytes_async(wav_data, "qq_voice.wav")
|
||||
except Exception as exc:
|
||||
logger.debug("[%s] Failed to read converted wav: %s", self._log_tag, exc)
|
||||
return None
|
||||
|
||||
@@ -25,8 +25,8 @@ import httpx
|
||||
|
||||
from gateway.config import Platform, PlatformConfig
|
||||
from gateway.platforms.base import (
|
||||
BasePlatformAdapter, MessageEvent, MessageType, ProcessingOutcome, SendResult, cache_image_from_bytes,
|
||||
cache_audio_from_bytes, cache_document_from_bytes, cache_image_from_url, utf16_len)
|
||||
BasePlatformAdapter, MessageEvent, MessageType, ProcessingOutcome, SendResult, cache_image_from_bytes_async,
|
||||
cache_audio_from_bytes_async, cache_document_from_bytes_async, cache_image_from_url, utf16_len)
|
||||
from gateway.platforms.helpers import redact_phone
|
||||
from gateway.platforms.media_cache import mime_for_ext
|
||||
from tools.audio_container import CONTAINER_TO_EXT, sniff_container
|
||||
@@ -592,9 +592,9 @@ class SignalAdapter(BasePlatformAdapter):
|
||||
# to .m4a. Without ffmpeg the raw file is cached as-is (no downstream remux fallback).
|
||||
if ext == ".aac":
|
||||
raw_data, ext = (await asyncio.to_thread(_remux_aac_to_m4a, raw_data)) or (raw_data, ext)
|
||||
cache = (cache_image_from_bytes if _is_image_ext(ext)
|
||||
else cache_audio_from_bytes if _is_audio_ext(ext) else cache_document_from_bytes)
|
||||
return cache(raw_data, ext), ext
|
||||
cache = (cache_image_from_bytes_async if _is_image_ext(ext)
|
||||
else cache_audio_from_bytes_async if _is_audio_ext(ext) else cache_document_from_bytes_async)
|
||||
return await cache(raw_data, ext), ext
|
||||
|
||||
async def _rpc(self, method: str, params: dict, rpc_id: str = None, *, log_failures: bool = True,
|
||||
raise_on_rate_limit: bool = False, timeout: float = 30.0) -> Any:
|
||||
|
||||
+22
-12
@@ -8,7 +8,7 @@ import asyncio, base64, contextlib, hashlib, json, logging, mimetypes, os, re, s
|
||||
from datetime import datetime
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||
from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -31,7 +31,7 @@ from gateway.config import Platform, PlatformConfig
|
||||
from gateway.platforms.helpers import MessageDeduplicator, greedy_pack_blocks
|
||||
from gateway.platforms.base import (
|
||||
_IMAGE_EXTS, _VIDEO_EXTS, gateway_trust_env, BasePlatformAdapter, MessageEvent, MessageType, SendResult,
|
||||
cache_audio_from_bytes, cache_document_from_bytes, cache_image_from_bytes,
|
||||
cache_audio_from_bytes_async, cache_document_from_bytes_async, cache_image_from_bytes_async,
|
||||
)
|
||||
from hermes_constants import get_hermes_home
|
||||
from utils import atomic_json_write
|
||||
@@ -175,6 +175,10 @@ class ContextTokenStore:
|
||||
def __init__(self, hermes_home: str):
|
||||
self._root = _account_dir(hermes_home)
|
||||
self._cache: Dict[str, str] = {}
|
||||
# Serializes the offloaded flushes so two concurrent set() calls
|
||||
# cannot land their writes out of order (last-writer-wins would drop
|
||||
# the newer token from disk).
|
||||
self._persist_lock = asyncio.Lock()
|
||||
|
||||
@staticmethod
|
||||
def _key(account_id: str, user_id: str) -> str:
|
||||
@@ -197,10 +201,16 @@ class ContextTokenStore:
|
||||
def get(self, account_id: str, user_id: str) -> Optional[str]:
|
||||
return self._cache.get(self._key(account_id, user_id))
|
||||
|
||||
def set(self, account_id: str, user_id: str, token: str) -> None:
|
||||
async def set(self, account_id: str, user_id: str, token: str) -> None:
|
||||
self._cache[self._key(account_id, user_id)] = token
|
||||
prefix = f"{account_id}:"
|
||||
payload = {key[len(prefix):]: value for key, value in self._cache.items() if key.startswith(prefix)}
|
||||
# atomic_json_write() fsyncs, so the flush is offloaded off the loop; the payload is snapshotted
|
||||
# here (the worker never iterates ``_cache`` mid-mutation) and the lock keeps flushes in order.
|
||||
async with self._persist_lock:
|
||||
prefix = f"{account_id}:"
|
||||
payload = {key[len(prefix):]: value for key, value in self._cache.items() if key.startswith(prefix)}
|
||||
await asyncio.to_thread(self._persist, account_id, payload)
|
||||
|
||||
def _persist(self, account_id: str, payload: Dict[str, str]) -> None:
|
||||
try:
|
||||
atomic_json_write(self._root / f"{account_id}.context-tokens.json", payload)
|
||||
except Exception as exc:
|
||||
@@ -649,11 +659,11 @@ _video_item, _voice_item = partial(_media_item, ITEM_VIDEO), partial(_media_item
|
||||
|
||||
# Inbound media dispatch: item type -> (item key, download timeout, cache fn, mime or None (= guess from
|
||||
# file_name), log label). Cache fns are lambdas so monkeypatching the module names takes effect at call time.
|
||||
_INBOUND_MEDIA: Dict[int, Tuple[str, float, Callable[[bytes, str], str], Optional[str], str]] = {
|
||||
ITEM_IMAGE: ("image_item", 30.0, lambda data, _name: cache_image_from_bytes(data, ".jpg"), "image/jpeg", "image"),
|
||||
ITEM_VIDEO: ("video_item", 120.0, lambda data, _name: cache_document_from_bytes(data, "video.mp4"), "video/mp4", "video"),
|
||||
ITEM_FILE: ("file_item", 60.0, lambda data, name: cache_document_from_bytes(data, name), None, "file"),
|
||||
ITEM_VOICE: ("voice_item", 60.0, lambda data, _name: cache_audio_from_bytes(data, ".silk"), "audio/silk", "voice"),
|
||||
_INBOUND_MEDIA: Dict[int, Tuple[str, float, Callable[[bytes, str], Awaitable[str]], Optional[str], str]] = {
|
||||
ITEM_IMAGE: ("image_item", 30.0, lambda data, _name: cache_image_from_bytes_async(data, ".jpg"), "image/jpeg", "image"),
|
||||
ITEM_VIDEO: ("video_item", 120.0, lambda data, _name: cache_document_from_bytes_async(data, "video.mp4"), "video/mp4", "video"),
|
||||
ITEM_FILE: ("file_item", 60.0, lambda data, name: cache_document_from_bytes_async(data, name), None, "file"),
|
||||
ITEM_VOICE: ("voice_item", 60.0, lambda data, _name: cache_audio_from_bytes_async(data, ".silk"), "audio/silk", "voice"),
|
||||
}
|
||||
|
||||
# Outbound local-file dispatch by extension: (extensions, sender method, path kwarg); default = send_document.
|
||||
@@ -870,7 +880,7 @@ class WeixinAdapter(BasePlatformAdapter):
|
||||
return
|
||||
context_token = str(message.get("context_token") or "").strip()
|
||||
if context_token:
|
||||
self._token_store.set(self._account_id, sender_id, context_token)
|
||||
await self._token_store.set(self._account_id, sender_id, context_token)
|
||||
if self._poll_session and self._token and not self._typing_cache.get(sender_id):
|
||||
asyncio.create_task(self._fetch_typing_ticket(self._poll_session, sender_id, context_token or None, "getConfig failed"))
|
||||
media_paths, media_types = [], [] # type: List[str], List[str]
|
||||
@@ -951,7 +961,7 @@ class WeixinAdapter(BasePlatformAdapter):
|
||||
data = await _download_and_decrypt_media(
|
||||
self._poll_session, cdn_base_url=self._cdn_base_url, encrypted_query_param=media.get("encrypt_query_param"),
|
||||
aes_key_b64=aes_key_b64, full_url=media.get("full_url"), timeout_seconds=timeout_seconds)
|
||||
return cache_fn(data, filename), mime
|
||||
return await cache_fn(data, filename), mime
|
||||
except Exception as exc:
|
||||
logger.warning("[%s] %s download failed: %s", self.name, label, exc)
|
||||
return None, mime
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user