Merge branch 'simp/r3-11' into simp/integration3
This commit is contained in:
+755
-1222
File diff suppressed because it is too large
Load Diff
+205
-380
File diff suppressed because it is too large
Load Diff
+429
-843
File diff suppressed because it is too large
Load Diff
+144
-300
@@ -1,4 +1,4 @@
|
||||
"""Config/env loaders for runtime knobs (busy modes, reasoning, service tier, timeouts, fallback) for GatewayRunner.
|
||||
"""Config/env loaders (busy modes, reasoning, service tier, timeouts, fallback) for GatewayRunner.
|
||||
|
||||
Split out of ``gateway/run.py``; bound onto ``GatewayRunner`` via the MRO.
|
||||
``gateway.run`` internals are imported lazily inside method bodies (import cycle),
|
||||
@@ -7,29 +7,26 @@ so ``patch("gateway.run.X")`` keeps intercepting them at call time.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from contextlib import suppress
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
||||
|
||||
from gateway.config import Platform
|
||||
from gateway.restart import (
|
||||
DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT,
|
||||
DEFAULT_GATEWAY_POST_INTERRUPT_GRACE_TIMEOUT,
|
||||
DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT,
|
||||
DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT,
|
||||
DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT,
|
||||
parse_cron_drain_timeout,
|
||||
parse_restart_after_turn_timeout,
|
||||
parse_restart_drain_timeout,
|
||||
DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, DEFAULT_GATEWAY_POST_INTERRUPT_GRACE_TIMEOUT,
|
||||
DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT, DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT,
|
||||
DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT, parse_cron_drain_timeout,
|
||||
parse_restart_after_turn_timeout, parse_restart_drain_timeout,
|
||||
parse_signal_interrupt_grace_timeout,
|
||||
)
|
||||
from gateway.session import SessionSource
|
||||
from gateway.session_state import SERVICE_TIER_UNSET as _SERVICE_TIER_UNSET
|
||||
from hermes_cli.config import cfg_get
|
||||
from hermes_cli.config import cfg_get, resolve_ephemeral_system_prompt_from_config
|
||||
from hermes_cli.fallback_config import get_fallback_chain
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
from utils import is_truthy_value
|
||||
|
||||
if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle)
|
||||
@@ -38,9 +35,22 @@ if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle)
|
||||
# Log-record parity with the origin module.
|
||||
logger = logging.getLogger("gateway.run")
|
||||
|
||||
_BUSY_INPUT_MODES = {"interrupt", "queue", "steer"}
|
||||
|
||||
|
||||
class GatewayConfigLoadersMixin:
|
||||
"""Config/env loaders for runtime knobs (busy modes, reasoning, service tier, timeouts, fallback) for GatewayRunner."""
|
||||
"""Config/env loaders (busy modes, reasoning, service tier, timeouts, fallback) for GatewayRunner."""
|
||||
|
||||
@staticmethod
|
||||
def _cfg_str(section: str, key: str) -> str:
|
||||
"""``<section>.<key>`` from the gateway runtime config as a stripped string ("" when unset)."""
|
||||
from gateway.run import _load_gateway_runtime_config
|
||||
return str(cfg_get(_load_gateway_runtime_config(), section, key, default="") or "").strip()
|
||||
|
||||
@classmethod
|
||||
def _env_or_cfg_str(cls, env_var: str, section: str, key: str) -> str:
|
||||
"""Non-empty stripped env var, else :meth:`_cfg_str`."""
|
||||
return os.getenv(env_var, "").strip() or cls._cfg_str(section, key)
|
||||
|
||||
@staticmethod
|
||||
def _load_prefill_messages() -> List[Dict[str, Any]]:
|
||||
@@ -53,9 +63,9 @@ class GatewayConfigLoadersMixin:
|
||||
file_path = os.getenv("HERMES_PREFILL_MESSAGES_FILE", "")
|
||||
if not file_path:
|
||||
cfg = _load_gateway_runtime_config()
|
||||
file_path = str(cfg.get("prefill_messages_file", "") or "")
|
||||
if not file_path:
|
||||
file_path = str(cfg_get(cfg, "agent", "prefill_messages_file", default="") or "")
|
||||
file_path = str(
|
||||
cfg.get("prefill_messages_file", "") or cfg_get(cfg, "agent", "prefill_messages_file", default="") or ""
|
||||
)
|
||||
if not file_path:
|
||||
return []
|
||||
path = Path(file_path).expanduser()
|
||||
@@ -77,26 +87,24 @@ class GatewayConfigLoadersMixin:
|
||||
|
||||
@staticmethod
|
||||
def _load_ephemeral_system_prompt() -> str:
|
||||
"""Load ephemeral system prompt: HERMES_EPHEMERAL_SYSTEM_PROMPT env var first, then
|
||||
``display.personality`` / ``agent.system_prompt`` in config.yaml.
|
||||
"""
|
||||
"""HERMES_EPHEMERAL_SYSTEM_PROMPT env first, then ``display.personality`` / ``agent.system_prompt``."""
|
||||
from gateway.run import _load_gateway_runtime_config
|
||||
from hermes_cli.config import resolve_ephemeral_system_prompt_from_config
|
||||
|
||||
prompt = os.getenv("HERMES_EPHEMERAL_SYSTEM_PROMPT", "")
|
||||
if prompt:
|
||||
return prompt
|
||||
cfg = _load_gateway_runtime_config()
|
||||
return resolve_ephemeral_system_prompt_from_config(cfg)
|
||||
return resolve_ephemeral_system_prompt_from_config(_load_gateway_runtime_config())
|
||||
|
||||
def _channel_override(self, platform: Platform, chat_id: str, thread_id, parent_id):
|
||||
"""``channel_overrides`` entry for this channel/thread, or None (also when no config is bound)."""
|
||||
from gateway.run import _get_channel_override
|
||||
config = getattr(self, "config", None)
|
||||
if not config:
|
||||
return None
|
||||
return _get_channel_override(config, platform, chat_id, thread_id=thread_id, parent_id=parent_id)
|
||||
|
||||
def _resolve_model_for_channel(
|
||||
self,
|
||||
platform: Platform,
|
||||
chat_id: str,
|
||||
*,
|
||||
user_config: Optional[dict] = None,
|
||||
thread_id: Optional[str] = None,
|
||||
parent_id: Optional[str] = None,
|
||||
self, platform: Platform, chat_id: str, *, user_config: Optional[dict] = None,
|
||||
thread_id: Optional[str] = None, parent_id: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Resolve model for this channel: channel_overrides else global default.
|
||||
|
||||
@@ -104,31 +112,16 @@ class GatewayConfigLoadersMixin:
|
||||
API server so the surfaces cannot diverge). No session tier here: session /model overrides
|
||||
are applied later by ``_apply_session_model_override``.
|
||||
"""
|
||||
from gateway.run import _get_channel_override, _resolve_gateway_model
|
||||
from gateway.run import _resolve_gateway_model
|
||||
from hermes_cli.model_switch import resolve_effective_model
|
||||
|
||||
override = None
|
||||
config = getattr(self, "config", None)
|
||||
if config:
|
||||
override = _get_channel_override(
|
||||
config,
|
||||
platform,
|
||||
chat_id,
|
||||
thread_id=thread_id,
|
||||
parent_id=parent_id,
|
||||
)
|
||||
return resolve_effective_model(
|
||||
None, # session tier applied downstream (_apply_session_model_override)
|
||||
override,
|
||||
self._channel_override(platform, chat_id, thread_id, parent_id),
|
||||
_resolve_gateway_model(user_config),
|
||||
)
|
||||
|
||||
def _get_system_prompt_for_channel(
|
||||
self,
|
||||
platform: Platform,
|
||||
chat_id: str,
|
||||
*,
|
||||
thread_id: Optional[str] = None,
|
||||
self, platform: Platform, chat_id: str, *, thread_id: Optional[str] = None,
|
||||
parent_id: Optional[str] = None,
|
||||
) -> str:
|
||||
"""Ephemeral system prompt for this channel/thread.
|
||||
@@ -138,40 +131,26 @@ class GatewayConfigLoadersMixin:
|
||||
profiles get their own personality/system_prompt and ``/personality`` edits apply next turn).
|
||||
Legacy ``channel_prompts`` are applied separately via ``event.channel_prompt`` in ``run_sync``.
|
||||
"""
|
||||
from gateway.run import _get_channel_override
|
||||
config = getattr(self, "config", None)
|
||||
if config:
|
||||
override = _get_channel_override(
|
||||
config,
|
||||
platform,
|
||||
chat_id,
|
||||
thread_id=thread_id,
|
||||
parent_id=parent_id,
|
||||
)
|
||||
if override and override.system_prompt:
|
||||
return (override.system_prompt or "").strip()
|
||||
override = self._channel_override(platform, chat_id, thread_id, parent_id)
|
||||
if override and override.system_prompt:
|
||||
return (override.system_prompt or "").strip()
|
||||
return self._load_ephemeral_system_prompt()
|
||||
|
||||
@staticmethod
|
||||
def _load_reasoning_config(model: str = "") -> dict | None:
|
||||
"""Load reasoning effort from config.yaml, respecting per-model overrides.
|
||||
"""Reasoning effort from config.yaml via :func:`hermes_constants.resolve_reasoning_config`.
|
||||
|
||||
Thin wrapper over :func:`hermes_constants.resolve_reasoning_config` (per-model override >
|
||||
global ``agent.reasoning_effort``; YAML False = disabled). Empty ``model`` uses ``model.default``.
|
||||
Per-model override > global ``agent.reasoning_effort``; YAML False = disabled. Empty
|
||||
``model`` uses ``model.default``.
|
||||
"""
|
||||
from gateway.run import _load_gateway_runtime_config
|
||||
from hermes_constants import resolve_reasoning_config
|
||||
cfg = _load_gateway_runtime_config()
|
||||
return resolve_reasoning_config(cfg, model)
|
||||
return resolve_reasoning_config(_load_gateway_runtime_config(), model)
|
||||
|
||||
@staticmethod
|
||||
def _parse_reasoning_command_args(raw_args: str) -> tuple[str, bool]:
|
||||
"""Parse `/reasoning` args into `(value, persist_global)`.
|
||||
|
||||
Session-scoped by default; `--global` in any position persists the change to config.yaml.
|
||||
"""
|
||||
"""Parse `/reasoning` args into `(value, persist_global)`; `--global` anywhere persists to config."""
|
||||
import shlex
|
||||
|
||||
text = str(raw_args or "").strip().replace("—", "--")
|
||||
if not text:
|
||||
return "", False
|
||||
@@ -179,42 +158,26 @@ class GatewayConfigLoadersMixin:
|
||||
tokens = shlex.split(text)
|
||||
except ValueError:
|
||||
tokens = text.split()
|
||||
|
||||
persist_global = False
|
||||
value_tokens = []
|
||||
for token in tokens:
|
||||
if token == "--global":
|
||||
persist_global = True
|
||||
else:
|
||||
value_tokens.append(token)
|
||||
return " ".join(value_tokens).strip().lower(), persist_global
|
||||
value = " ".join(token for token in tokens if token != "--global")
|
||||
return value.strip().lower(), "--global" in tokens
|
||||
|
||||
def _resolve_session_reasoning_config(
|
||||
self,
|
||||
*,
|
||||
source: Optional[SessionSource] = None,
|
||||
session_key: Optional[str] = None,
|
||||
self, *, source: Optional[SessionSource] = None, session_key: Optional[str] = None,
|
||||
model: str = "",
|
||||
) -> dict | None:
|
||||
"""Resolve reasoning effort for a session, honoring session overrides.
|
||||
"""Session ``/reasoning --session`` > per-model ``agent.reasoning_overrides`` > global.
|
||||
|
||||
Priority: session ``/reasoning --session`` > per-model ``agent.reasoning_overrides`` > global
|
||||
``agent.reasoning_effort``. ``model`` must be the session's *effective* model (session
|
||||
``/model`` override included); empty uses ``model.default``.
|
||||
``model`` must be the session's *effective* model (session ``/model`` override included);
|
||||
empty uses ``model.default``.
|
||||
"""
|
||||
resolved_session_key = self._resolve_session_key_or_none(source, session_key)
|
||||
|
||||
if resolved_session_key:
|
||||
_r_state = self._peek_session_state(resolved_session_key)
|
||||
if _r_state is not None and _r_state.conversation.reasoning_override is not None:
|
||||
return _r_state.conversation.reasoning_override
|
||||
return self._load_reasoning_config(model)
|
||||
|
||||
def _set_session_reasoning_override(
|
||||
self,
|
||||
session_key: str,
|
||||
reasoning_config: Optional[dict],
|
||||
) -> None:
|
||||
def _set_session_reasoning_override(self, session_key: str, reasoning_config: Optional[dict]) -> None:
|
||||
"""Set or clear the session-scoped reasoning override."""
|
||||
if not session_key:
|
||||
return
|
||||
@@ -224,39 +187,20 @@ class GatewayConfigLoadersMixin:
|
||||
None if reasoning_config is None else dict(reasoning_config)
|
||||
)
|
||||
|
||||
def _resolve_session_service_tier(
|
||||
self,
|
||||
source=None,
|
||||
session_key: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
"""Resolve the effective service tier for a session.
|
||||
def _resolve_session_service_tier(self, source=None, session_key: Optional[str] = None) -> Optional[str]:
|
||||
"""Effective service tier: a session-scoped /fast override beats the config default.
|
||||
|
||||
A session-scoped /fast override beats the config default; the override dict stores
|
||||
"priority" or None (explicit normal), so key presence — not truthiness — decides.
|
||||
The override stores "priority" or None (explicit normal), so presence — not truthiness — decides.
|
||||
"""
|
||||
resolved_session_key = self._resolve_session_key_or_none(source, session_key)
|
||||
|
||||
if resolved_session_key:
|
||||
_t_state = self._peek_session_state(resolved_session_key)
|
||||
if (
|
||||
_t_state is not None
|
||||
and _t_state.conversation.service_tier_override
|
||||
is not _SERVICE_TIER_UNSET
|
||||
):
|
||||
if _t_state is not None and _t_state.conversation.service_tier_override is not _SERVICE_TIER_UNSET:
|
||||
return _t_state.conversation.service_tier_override
|
||||
return self._load_service_tier()
|
||||
|
||||
def _set_session_service_tier_override(
|
||||
self,
|
||||
session_key: str,
|
||||
service_tier,
|
||||
clear: bool = False,
|
||||
) -> None:
|
||||
"""Set or clear the session-scoped /fast override.
|
||||
|
||||
``service_tier`` is "priority" or None (explicit normal). Pass
|
||||
``clear=True`` to remove the override entirely (fall back to config).
|
||||
"""
|
||||
def _set_session_service_tier_override(self, session_key: str, service_tier, clear: bool = False) -> None:
|
||||
"""Set ("priority" / None = explicit normal) or ``clear`` the session-scoped /fast override."""
|
||||
if not session_key:
|
||||
return
|
||||
# Presence-sensitive: "priority" or None (explicit normal) both count as an override; the
|
||||
@@ -265,15 +209,10 @@ class GatewayConfigLoadersMixin:
|
||||
_SERVICE_TIER_UNSET if clear else service_tier
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _load_service_tier() -> str | None:
|
||||
"""Load Priority Processing (agent.service_tier) from config.yaml: "fast"/"priority"/"on" =>
|
||||
"priority"; "normal"/"off" disable; None when unset/unsupported.
|
||||
"""
|
||||
from gateway.run import _load_gateway_runtime_config
|
||||
cfg = _load_gateway_runtime_config()
|
||||
raw = str(cfg_get(cfg, "agent", "service_tier", default="") or "").strip()
|
||||
|
||||
@classmethod
|
||||
def _load_service_tier(cls) -> str | None:
|
||||
"""``agent.service_tier``: fast/priority/on => "priority"; normal/off => None; None when unset/unknown."""
|
||||
raw = cls._cfg_str("agent", "service_tier")
|
||||
value = raw.lower()
|
||||
if not value or value in {"normal", "default", "standard", "off", "none"}:
|
||||
return None
|
||||
@@ -286,72 +225,38 @@ class GatewayConfigLoadersMixin:
|
||||
|
||||
@staticmethod
|
||||
def _load_show_reasoning() -> bool:
|
||||
"""Load show_reasoning toggle from config.yaml display section."""
|
||||
"""``display.show_reasoning`` toggle."""
|
||||
from gateway.run import _load_gateway_runtime_config
|
||||
cfg = _load_gateway_runtime_config()
|
||||
return is_truthy_value(
|
||||
cfg_get(cfg, "display", "show_reasoning"),
|
||||
default=False,
|
||||
)
|
||||
return is_truthy_value(cfg_get(_load_gateway_runtime_config(), "display", "show_reasoning"), default=False)
|
||||
|
||||
@staticmethod
|
||||
def _load_busy_input_mode() -> str:
|
||||
"""Load gateway drain-time busy-input behavior from config/env."""
|
||||
from gateway.run import _load_gateway_runtime_config
|
||||
mode = os.getenv("HERMES_GATEWAY_BUSY_INPUT_MODE", "").strip().lower()
|
||||
if not mode:
|
||||
cfg = _load_gateway_runtime_config()
|
||||
mode = str(cfg_get(cfg, "display", "busy_input_mode", default="") or "").strip().lower()
|
||||
if mode == "queue":
|
||||
return "queue"
|
||||
if mode == "steer":
|
||||
return "steer"
|
||||
return "interrupt"
|
||||
@classmethod
|
||||
def _load_busy_input_mode(cls) -> str:
|
||||
"""Gateway drain-time busy-input behavior from env/config (default ``interrupt``)."""
|
||||
mode = cls._env_or_cfg_str("HERMES_GATEWAY_BUSY_INPUT_MODE", "display", "busy_input_mode").lower()
|
||||
return mode if mode in {"queue", "steer"} else "interrupt"
|
||||
|
||||
@staticmethod
|
||||
def _load_busy_text_mode() -> str:
|
||||
"""Resolve normal busy TEXT follow-up behavior.
|
||||
@classmethod
|
||||
def _load_busy_text_mode(cls) -> str:
|
||||
"""Normal busy TEXT follow-up behavior.
|
||||
|
||||
``busy_input_mode`` is the source of truth (default ``interrupt``); legacy ``busy_text_mode``
|
||||
is honored only when explicitly set so existing queue setups keep working.
|
||||
"""
|
||||
from gateway.run import GatewayRunner, _load_gateway_runtime_config
|
||||
# Legacy explicit override wins for backward compat.
|
||||
legacy = os.getenv("HERMES_GATEWAY_BUSY_TEXT_MODE", "").strip().lower()
|
||||
if not legacy:
|
||||
cfg = _load_gateway_runtime_config()
|
||||
legacy = str(cfg_get(cfg, "display", "busy_text_mode", default="") or "").strip().lower()
|
||||
if legacy == "interrupt":
|
||||
return "interrupt"
|
||||
if legacy == "queue":
|
||||
return "queue"
|
||||
# No explicit legacy knob → follow busy_input_mode.
|
||||
input_mode = GatewayRunner._load_busy_input_mode()
|
||||
return "queue" if input_mode == "queue" else "interrupt"
|
||||
from gateway.run import GatewayRunner
|
||||
legacy = cls._env_or_cfg_str("HERMES_GATEWAY_BUSY_TEXT_MODE", "display", "busy_text_mode").lower()
|
||||
if legacy in {"interrupt", "queue"}:
|
||||
return legacy
|
||||
return "queue" if GatewayRunner._load_busy_input_mode() == "queue" else "interrupt"
|
||||
|
||||
@staticmethod
|
||||
def _busy_modes_from_config(
|
||||
config: dict,
|
||||
*,
|
||||
fallback_input: str,
|
||||
fallback_text: str,
|
||||
) -> tuple[str, str]:
|
||||
def _busy_modes_from_config(config: dict, *, fallback_input: str, fallback_text: str) -> tuple[str, str]:
|
||||
"""Resolve one profile's busy modes without consulting process env."""
|
||||
raw_input = str(
|
||||
cfg_get(config, "display", "busy_input_mode", default="") or ""
|
||||
).strip().lower()
|
||||
input_mode = (
|
||||
raw_input
|
||||
if raw_input in {"interrupt", "queue", "steer"}
|
||||
else fallback_input
|
||||
)
|
||||
|
||||
raw_text = str(
|
||||
cfg_get(config, "display", "busy_text_mode", default="") or ""
|
||||
).strip().lower()
|
||||
raw_input = str(cfg_get(config, "display", "busy_input_mode", default="") or "").strip().lower()
|
||||
input_mode = raw_input if raw_input in _BUSY_INPUT_MODES else fallback_input
|
||||
raw_text = str(cfg_get(config, "display", "busy_text_mode", default="") or "").strip().lower()
|
||||
if raw_text in {"interrupt", "queue"}:
|
||||
text_mode = raw_text
|
||||
elif raw_input in {"interrupt", "queue", "steer"}:
|
||||
elif raw_input in _BUSY_INPUT_MODES:
|
||||
text_mode = "queue" if input_mode == "queue" else "interrupt"
|
||||
else:
|
||||
text_mode = fallback_text
|
||||
@@ -360,14 +265,11 @@ class GatewayConfigLoadersMixin:
|
||||
def _snapshot_profile_busy_modes(self, profile_name: str, config: dict) -> None:
|
||||
"""Cache a routed profile's busy policy for this gateway lifetime."""
|
||||
input_mode, text_mode = self._busy_modes_from_config(
|
||||
config,
|
||||
fallback_input=getattr(self, "_busy_input_mode", "interrupt"),
|
||||
config, fallback_input=getattr(self, "_busy_input_mode", "interrupt"),
|
||||
fallback_text=getattr(self, "_busy_text_mode", "interrupt"),
|
||||
)
|
||||
input_modes = self.__dict__.setdefault("_busy_input_modes_by_profile", {})
|
||||
text_modes = self.__dict__.setdefault("_busy_text_modes_by_profile", {})
|
||||
input_modes[profile_name] = input_mode
|
||||
text_modes[profile_name] = text_mode
|
||||
self.__dict__.setdefault("_busy_input_modes_by_profile", {})[profile_name] = input_mode
|
||||
self.__dict__.setdefault("_busy_text_modes_by_profile", {})[profile_name] = text_mode
|
||||
|
||||
def _busy_profile_name_for_source(self, source: SessionSource) -> Optional[str]:
|
||||
"""Return the routed profile whose busy policy applies, if any."""
|
||||
@@ -381,46 +283,42 @@ class GatewayConfigLoadersMixin:
|
||||
name = ""
|
||||
return name or None
|
||||
|
||||
def _effective_busy_input_mode(self, source: SessionSource) -> str:
|
||||
"""Resolve busy input mode from the routed profile startup snapshot."""
|
||||
fallback = getattr(self, "_busy_input_mode", "interrupt")
|
||||
def _effective_busy_mode(self, source: SessionSource, attr: str) -> str:
|
||||
"""Busy mode from the routed profile snapshot (``attr``: ``_busy_input_mode`` / ``_busy_text_mode``)."""
|
||||
fallback = getattr(self, attr, "interrupt")
|
||||
profile_name = self._busy_profile_name_for_source(source)
|
||||
if not profile_name:
|
||||
return fallback
|
||||
modes = getattr(self, "_busy_input_modes_by_profile", None)
|
||||
modes = getattr(self, attr + "s_by_profile", None)
|
||||
return modes.get(profile_name, fallback) if isinstance(modes, dict) else fallback
|
||||
|
||||
def _effective_busy_input_mode(self, source: SessionSource) -> str:
|
||||
"""Resolve busy input mode from the routed profile startup snapshot."""
|
||||
return self._effective_busy_mode(source, "_busy_input_mode")
|
||||
|
||||
def _effective_busy_text_mode(self, source: SessionSource) -> str:
|
||||
"""Resolve legacy busy text mode from the routed profile snapshot."""
|
||||
fallback = getattr(self, "_busy_text_mode", "interrupt")
|
||||
profile_name = self._busy_profile_name_for_source(source)
|
||||
if not profile_name:
|
||||
return fallback
|
||||
modes = getattr(self, "_busy_text_modes_by_profile", None)
|
||||
return modes.get(profile_name, fallback) if isinstance(modes, dict) else fallback
|
||||
return self._effective_busy_mode(source, "_busy_text_mode")
|
||||
|
||||
@staticmethod
|
||||
def _load_restart_drain_timeout() -> float:
|
||||
"""Load graceful gateway restart/stop drain timeout in seconds."""
|
||||
from gateway.run import _load_gateway_runtime_config
|
||||
raw = os.getenv("HERMES_RESTART_DRAIN_TIMEOUT", "").strip()
|
||||
if not raw:
|
||||
cfg = _load_gateway_runtime_config()
|
||||
raw = str(cfg_get(cfg, "agent", "restart_drain_timeout", default="") or "").strip()
|
||||
def _warn_unparsable_timeout(cfg_key: str, raw: object, default: float) -> None:
|
||||
"""Warn when a supplied timeout value is not a number (the parser already fell back to ``default``)."""
|
||||
try:
|
||||
float(raw)
|
||||
except (TypeError, ValueError):
|
||||
logger.warning("Invalid %s '%s', using default %.0fs", cfg_key, raw, default)
|
||||
|
||||
@classmethod
|
||||
def _load_restart_drain_timeout(cls) -> float:
|
||||
"""Graceful gateway restart/stop drain timeout in seconds."""
|
||||
raw = cls._env_or_cfg_str("HERMES_RESTART_DRAIN_TIMEOUT", "agent", "restart_drain_timeout")
|
||||
value = parse_restart_drain_timeout(raw)
|
||||
if raw and value == DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT:
|
||||
try:
|
||||
float(raw)
|
||||
except (TypeError, ValueError):
|
||||
logger.warning(
|
||||
"Invalid restart_drain_timeout '%s', using default %.0fs",
|
||||
raw,
|
||||
DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT,
|
||||
)
|
||||
cls._warn_unparsable_timeout("restart_drain_timeout", raw, DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT)
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def _load_env_or_agent_cfg_timeout(env_var: str, cfg_key: str, parse, default: float) -> float:
|
||||
@classmethod
|
||||
def _load_env_or_agent_cfg_timeout(cls, env_var: str, cfg_key: str, parse, default: float) -> float:
|
||||
"""Env var (non-empty) else ``agent.<cfg_key>``; warn once when a supplied value fails to parse.
|
||||
|
||||
``0`` is a valid value; the parser falls back to ``default`` on garbage."""
|
||||
@@ -429,19 +327,15 @@ class GatewayConfigLoadersMixin:
|
||||
if env_raw is not None and str(env_raw).strip() != "":
|
||||
raw: object = env_raw
|
||||
else:
|
||||
cfg = _load_gateway_runtime_config()
|
||||
raw = cfg_get(cfg, "agent", cfg_key, default=None)
|
||||
raw = cfg_get(_load_gateway_runtime_config(), "agent", cfg_key, default=None)
|
||||
value = parse(raw)
|
||||
if raw is not None and str(raw).strip() != "":
|
||||
try:
|
||||
float(raw)
|
||||
except (TypeError, ValueError):
|
||||
logger.warning("Invalid %s '%s', using default %.0fs", cfg_key, raw, default)
|
||||
cls._warn_unparsable_timeout(cfg_key, raw, default)
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def _load_restart_after_turn_timeout(cls) -> float:
|
||||
"""Load in-band restart wait-for-idle timeout in seconds."""
|
||||
"""In-band restart wait-for-idle timeout in seconds."""
|
||||
return cls._load_env_or_agent_cfg_timeout(
|
||||
"HERMES_RESTART_AFTER_TURN_TIMEOUT", "restart_after_turn_timeout",
|
||||
parse_restart_after_turn_timeout, DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT,
|
||||
@@ -449,105 +343,66 @@ class GatewayConfigLoadersMixin:
|
||||
|
||||
@classmethod
|
||||
def _load_cron_drain_timeout(cls) -> float:
|
||||
"""Load the cron-only floor under the stop()/drain wait."""
|
||||
"""The cron-only floor under the stop()/drain wait."""
|
||||
return cls._load_env_or_agent_cfg_timeout(
|
||||
"HERMES_CRON_DRAIN_TIMEOUT", "cron_drain_timeout",
|
||||
parse_cron_drain_timeout, DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _load_signal_interrupt_grace_timeout() -> float:
|
||||
"""Load the unexpected-signal post-interrupt grace in seconds."""
|
||||
@classmethod
|
||||
def _load_signal_interrupt_grace_timeout(cls) -> float:
|
||||
"""``gateway.signal_interrupt_grace_timeout``: unexpected-signal post-interrupt grace in seconds."""
|
||||
from gateway.run import _load_gateway_runtime_config
|
||||
cfg = _load_gateway_runtime_config()
|
||||
raw = cfg_get(
|
||||
cfg,
|
||||
"gateway",
|
||||
"signal_interrupt_grace_timeout",
|
||||
default=None,
|
||||
)
|
||||
raw = cfg_get(_load_gateway_runtime_config(), "gateway", "signal_interrupt_grace_timeout", default=None)
|
||||
value = parse_signal_interrupt_grace_timeout(raw)
|
||||
if raw is not None and raw != "":
|
||||
try:
|
||||
float(raw)
|
||||
except (TypeError, ValueError):
|
||||
logger.warning(
|
||||
"Invalid signal_interrupt_grace_timeout '%s', using default %.0fs",
|
||||
raw,
|
||||
DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT,
|
||||
)
|
||||
cls._warn_unparsable_timeout(
|
||||
"signal_interrupt_grace_timeout", raw, DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT
|
||||
)
|
||||
return value
|
||||
|
||||
def _post_interrupt_grace_timeout(self) -> float:
|
||||
"""Return the grace before teardown after forcibly interrupting agents."""
|
||||
if (
|
||||
getattr(self, "_signal_initiated_shutdown", False)
|
||||
and not getattr(self, "_restart_requested", False)
|
||||
):
|
||||
return max(
|
||||
0.0,
|
||||
float(
|
||||
getattr(
|
||||
self,
|
||||
"_signal_interrupt_grace_timeout",
|
||||
DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT,
|
||||
)
|
||||
),
|
||||
)
|
||||
"""Grace before teardown after forcibly interrupting agents (longer on an unexpected signal)."""
|
||||
if getattr(self, "_signal_initiated_shutdown", False) and not getattr(self, "_restart_requested", False):
|
||||
grace = getattr(self, "_signal_interrupt_grace_timeout", DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT)
|
||||
return max(0.0, float(grace))
|
||||
return DEFAULT_GATEWAY_POST_INTERRUPT_GRACE_TIMEOUT
|
||||
|
||||
@staticmethod
|
||||
def _load_background_notifications_mode() -> str:
|
||||
"""Load background process notification mode from config or env var."""
|
||||
"""Background process notification mode from env/config (default ``concise``)."""
|
||||
from gateway.run import _load_gateway_runtime_config
|
||||
mode = os.getenv("HERMES_BACKGROUND_NOTIFICATIONS", "")
|
||||
if not mode:
|
||||
cfg = _load_gateway_runtime_config()
|
||||
raw = cfg_get(cfg, "display", "background_process_notifications")
|
||||
raw = cfg_get(_load_gateway_runtime_config(), "display", "background_process_notifications")
|
||||
if raw is False:
|
||||
mode = "off"
|
||||
elif raw not in {None, ""}:
|
||||
mode = str(raw)
|
||||
mode = (mode or "concise").strip().lower()
|
||||
valid = {"concise", "all", "result", "error", "off"}
|
||||
if mode not in valid:
|
||||
logger.warning(
|
||||
"Unknown background_process_notifications '%s', defaulting to 'concise'",
|
||||
mode,
|
||||
)
|
||||
if mode not in {"concise", "all", "result", "error", "off"}:
|
||||
logger.warning("Unknown background_process_notifications '%s', defaulting to 'concise'", mode)
|
||||
return "concise"
|
||||
return mode
|
||||
|
||||
@staticmethod
|
||||
def _load_provider_routing() -> dict:
|
||||
"""Load OpenRouter provider routing preferences from config.yaml."""
|
||||
"""OpenRouter provider routing preferences (canonical fail-open loader: managed overlay + ${VAR})."""
|
||||
from gateway.run import _load_gateway_runtime_config
|
||||
try:
|
||||
# Canonical gateway loader (fail-open): managed overlay + ${VAR}
|
||||
# expansion now apply to provider_routing too.
|
||||
cfg = _load_gateway_runtime_config()
|
||||
return cfg.get("provider_routing", {}) or {}
|
||||
return _load_gateway_runtime_config().get("provider_routing", {}) or {}
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def _load_fallback_model() -> list | None:
|
||||
"""Load fallback provider chain from config.yaml.
|
||||
|
||||
Merges ``fallback_providers`` (kept first) with legacy ``fallback_model`` entries.
|
||||
"""
|
||||
"""Fallback chain: ``fallback_providers`` (kept first) merged with legacy ``fallback_model``."""
|
||||
from gateway.run import _load_gateway_runtime_config
|
||||
try:
|
||||
# Canonical gateway loader (fail-open): managed overlay + ${VAR}
|
||||
# expansion now apply to the fallback chain too.
|
||||
cfg = _load_gateway_runtime_config()
|
||||
fb = get_fallback_chain(cfg)
|
||||
if fb:
|
||||
return fb
|
||||
# Canonical gateway loader (fail-open): managed overlay + ${VAR} expansion apply here too.
|
||||
return get_fallback_chain(_load_gateway_runtime_config()) or None
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
return None
|
||||
|
||||
def _refresh_fallback_model(self) -> list | None:
|
||||
"""Re-read fallback_providers from disk for the next agent create/reuse.
|
||||
@@ -567,24 +422,16 @@ class GatewayConfigLoadersMixin:
|
||||
# loader would return {} on a torn mid-edit write and WIPE the last known-good chain.
|
||||
# The overlay/expansion below fixes the managed-scope/${VAR} drift without losing that.
|
||||
cfg = read_user_config_raw(cfg_path)
|
||||
try:
|
||||
with suppress(Exception):
|
||||
from hermes_cli import managed_scope
|
||||
cfg = managed_scope.apply_managed_overlay(cfg)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
with suppress(Exception):
|
||||
from hermes_cli.config import _expand_env_vars
|
||||
expanded = _expand_env_vars(cfg)
|
||||
if isinstance(expanded, dict):
|
||||
cfg = expanded
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
# Transient failure — keep last known-good chain.
|
||||
logger.debug(
|
||||
"fallback_providers refresh: config.yaml read failed; "
|
||||
"keeping last known-good chain", exc_info=True,
|
||||
)
|
||||
logger.debug("fallback_providers refresh: config.yaml read failed; keeping last known-good chain", exc_info=True)
|
||||
return self._fallback_model
|
||||
self._fallback_model = get_fallback_chain(cfg) or None
|
||||
return self._fallback_model
|
||||
@@ -601,10 +448,7 @@ class GatewayConfigLoadersMixin:
|
||||
return
|
||||
new_chain = list(chain or [])
|
||||
rate_limited_until = getattr(agent, "_rate_limited_until", 0) or 0
|
||||
if (
|
||||
getattr(agent, "_fallback_activated", False)
|
||||
and rate_limited_until > time.monotonic()
|
||||
):
|
||||
if getattr(agent, "_fallback_activated", False) and rate_limited_until > time.monotonic():
|
||||
return
|
||||
old_chain = list(getattr(agent, "_fallback_chain", []) or [])
|
||||
agent._fallback_chain = new_chain
|
||||
|
||||
+138
-195
@@ -7,13 +7,13 @@ so ``patch("gateway.run.X")`` keeps intercepting them at call time.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from contextlib import suppress
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from gateway.platforms.base import MessageEvent, MessageType
|
||||
from typing import Any
|
||||
|
||||
if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle)
|
||||
from gateway.run import GatewayRunner, TurnRunner # noqa: F401
|
||||
@@ -25,14 +25,12 @@ logger = logging.getLogger("gateway.run")
|
||||
class GatewayGoalsMixin:
|
||||
"""Goal/heartbeat continuation, post-turn hooks and loop-wakeup watcher methods for GatewayRunner."""
|
||||
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
# /goal — persistent cross-turn goals (Ralph-style loop)
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
# ── /goal — persistent cross-turn goals (Ralph-style loop) ──────────
|
||||
def _goal_max_turns_from_config(self) -> int:
|
||||
"""Resolve the configured /goal turn budget for gateway sessions.
|
||||
"""Resolve the configured /goal turn budget.
|
||||
|
||||
GatewayRunner.config is a GatewayConfig dataclass, not the full user config mapping, so
|
||||
top-level blocks such as ``goals`` are only reachable via hermes_cli.config.load_config().
|
||||
GatewayRunner.config is a GatewayConfig dataclass, not the full user config, so the
|
||||
top-level ``goals`` block is only reachable via hermes_cli.config.load_config().
|
||||
"""
|
||||
try:
|
||||
goals_cfg = (
|
||||
@@ -51,10 +49,9 @@ class GatewayGoalsMixin:
|
||||
async def _warm_goals_session_db(self, label: str) -> None:
|
||||
"""Warm the goals SessionDB cache off-loop (best-effort).
|
||||
|
||||
A cold cache runs the state.db init on the loop thread and freezes the loop for the init
|
||||
duration. The executor hop keeps the profile home override alive under multiplex, so the
|
||||
warm cache belongs to the caller's profile. On failure the caller falls back to the
|
||||
bootstrap windows, so a dropped warm-up is a bounded stall, never a crash.
|
||||
A cold cache runs the state.db init on the loop thread and freezes the loop. The executor
|
||||
hop keeps the profile home override alive under multiplex. On failure the caller falls
|
||||
back to the bootstrap windows, so a dropped warm-up is a bounded stall, never a crash.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.goals import _get_session_db as _warm_goals_db
|
||||
@@ -66,15 +63,14 @@ class GatewayGoalsMixin:
|
||||
async def _session_entry_for_manager(self, event: "MessageEvent", label: str):
|
||||
"""Session entry for a /goal or /heartbeat manager, or None when lookup fails.
|
||||
|
||||
Warms the SessionDB cache off-loop first: a cold cache freezes the loop for the init
|
||||
duration and drops the first write while the reply claims it was set. Internal events look
|
||||
the session up WITHOUT touching activity so they never advance the idle/daily reset clock.
|
||||
Warms the SessionDB cache first (a cold cache drops the first write while the reply
|
||||
claims it was set). Internal events never touch activity, so they don't advance the
|
||||
idle/daily reset clock.
|
||||
"""
|
||||
await self._warm_goals_session_db(label)
|
||||
try:
|
||||
session_entry = await self.async_session_store.get_or_create_session(
|
||||
event.source,
|
||||
touch_activity=not bool(getattr(event, "internal", False)),
|
||||
event.source, touch_activity=not bool(getattr(event, "internal", False)),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("%s: session lookup failed: %s", label, exc)
|
||||
@@ -108,6 +104,11 @@ class GatewayGoalsMixin:
|
||||
return None, None
|
||||
return HeartbeatManager(session_id=session_entry.session_id), session_entry
|
||||
|
||||
@staticmethod
|
||||
def _synthetic_prompt_event(source: Any, text: str, *, internal: bool = False) -> MessageEvent:
|
||||
"""Build the TEXT event used to inject a goal/heartbeat/loop prompt into a session."""
|
||||
return MessageEvent(text=text, message_type=MessageType.TEXT, source=source, internal=internal)
|
||||
|
||||
def _register_heartbeat_watch(self, quick_key: str, source: Any, session_id: str) -> None:
|
||||
"""Track a session with an active heartbeat and start the poller.
|
||||
|
||||
@@ -128,6 +129,32 @@ class GatewayGoalsMixin:
|
||||
if watch:
|
||||
watch.pop(quick_key, None)
|
||||
|
||||
async def _heartbeat_poll_once(self, watch: dict) -> None:
|
||||
"""One heartbeat poll pass: enqueue every due prompt of a non-busy watched session."""
|
||||
# Warm the cache off-loop once per poll: this only covers the degraded path where the
|
||||
# /heartbeat command's own warm-up failed.
|
||||
await self._warm_goals_session_db("heartbeat poll")
|
||||
for quick_key, (source, session_id) in list(watch.items()):
|
||||
try:
|
||||
# Busy sessions coalesce their tick to the next idle poll.
|
||||
if quick_key in self._running_agents:
|
||||
continue
|
||||
from hermes_cli.heartbeat import HeartbeatManager
|
||||
|
||||
mgr = HeartbeatManager(session_id=session_id)
|
||||
if not mgr.has_heartbeat():
|
||||
watch.pop(quick_key, None)
|
||||
continue
|
||||
prompt = mgr.due_prompt()
|
||||
if not prompt:
|
||||
continue
|
||||
adapter = self._adapter_for_source(source)
|
||||
if adapter is None:
|
||||
continue
|
||||
self._enqueue_fifo(quick_key, self._synthetic_prompt_event(source, prompt), adapter)
|
||||
except Exception as exc:
|
||||
logger.debug("heartbeat poll for %s failed: %s", quick_key, exc)
|
||||
|
||||
def _start_heartbeat_poller(self) -> None:
|
||||
"""Start the single gateway-wide heartbeat poll task (idempotent)."""
|
||||
existing = getattr(self, "_heartbeat_poll_task", None)
|
||||
@@ -140,46 +167,14 @@ class GatewayGoalsMixin:
|
||||
while True:
|
||||
await asyncio.sleep(POLL_SECONDS)
|
||||
watch = getattr(self, "_heartbeat_watch", None)
|
||||
if not watch:
|
||||
continue
|
||||
# Warm the cache off-loop once per poll. A watch can only be registered through the
|
||||
# warmed /heartbeat command, so this covers only the degraded path where that warm-
|
||||
# up failed.
|
||||
await self._warm_goals_session_db("heartbeat poll")
|
||||
for quick_key, (source, session_id) in list(watch.items()):
|
||||
try:
|
||||
# Busy sessions coalesce their tick to the next idle poll.
|
||||
if quick_key in self._running_agents:
|
||||
continue
|
||||
from hermes_cli.heartbeat import HeartbeatManager
|
||||
|
||||
mgr = HeartbeatManager(session_id=session_id)
|
||||
if not mgr.has_heartbeat():
|
||||
watch.pop(quick_key, None)
|
||||
continue
|
||||
prompt = mgr.due_prompt()
|
||||
if not prompt:
|
||||
continue
|
||||
adapter = self._adapter_for_source(source)
|
||||
if adapter is None:
|
||||
continue
|
||||
hb_event = MessageEvent(
|
||||
text=prompt,
|
||||
message_type=MessageType.TEXT,
|
||||
source=source,
|
||||
message_id=None,
|
||||
channel_prompt=None,
|
||||
)
|
||||
self._enqueue_fifo(quick_key, hb_event, adapter)
|
||||
except Exception as exc:
|
||||
logger.debug("heartbeat poll for %s failed: %s", quick_key, exc)
|
||||
if watch:
|
||||
await self._heartbeat_poll_once(watch)
|
||||
|
||||
try:
|
||||
task = asyncio.create_task(_poll_loop())
|
||||
self._heartbeat_poll_task = task
|
||||
# PERMANENT once started (an infinite while-True loop, no exit condition) — same as a
|
||||
# _spawn_supervised watcher. Tag it so _scale_to_zero_has_live_background_work() doesn't
|
||||
# treat a gateway with an active heartbeat watch as busy forever.
|
||||
# PERMANENT once started (infinite loop) — same as a _spawn_supervised watcher. Tag it
|
||||
# so _scale_to_zero_has_live_background_work() doesn't treat the gateway as busy forever.
|
||||
task._hermes_supervised_watcher = True # type: ignore[attr-defined]
|
||||
_bg = getattr(self, "_background_tasks", None)
|
||||
if _bg is not None:
|
||||
@@ -194,17 +189,14 @@ class GatewayGoalsMixin:
|
||||
if not adapter:
|
||||
logger.debug("goal continuation: no adapter for %s", getattr(source, "platform", None))
|
||||
return
|
||||
|
||||
try:
|
||||
metadata = self._thread_metadata_for_source(source)
|
||||
except Exception:
|
||||
metadata = None
|
||||
|
||||
result = await adapter.send(source.chat_id, message, metadata=metadata)
|
||||
if result is not None and not getattr(result, "success", True):
|
||||
logger.warning(
|
||||
"goal continuation: status send failed: %s",
|
||||
getattr(result, "error", "unknown error"),
|
||||
"goal continuation: status send failed: %s", getattr(result, "error", "unknown error"),
|
||||
)
|
||||
|
||||
async def _defer_goal_status_notice_after_delivery(self, source: Any, message: str) -> None:
|
||||
@@ -236,11 +228,7 @@ class GatewayGoalsMixin:
|
||||
active = getattr(adapter, "_active_sessions", {}).get(session_key)
|
||||
if active is not None:
|
||||
generation = getattr(active, "_hermes_run_generation", None)
|
||||
adapter.register_post_delivery_callback(
|
||||
session_key,
|
||||
_deliver,
|
||||
generation=generation,
|
||||
)
|
||||
adapter.register_post_delivery_callback(session_key, _deliver, generation=generation)
|
||||
return
|
||||
except Exception as exc:
|
||||
logger.debug("goal continuation: post-delivery callback registration failed: %s", exc)
|
||||
@@ -248,14 +236,9 @@ class GatewayGoalsMixin:
|
||||
await _deliver()
|
||||
|
||||
async def _post_turn_goal_continuation(
|
||||
self,
|
||||
*,
|
||||
session_entry: Any,
|
||||
source: Any,
|
||||
final_response: str,
|
||||
self, *, session_entry: Any, source: Any, final_response: str,
|
||||
) -> None:
|
||||
"""Run the goal judge after a gateway turn and, if still active, enqueue a continuation
|
||||
prompt for the same session.
|
||||
"""Run the goal judge after a gateway turn and, if still active, enqueue a continuation.
|
||||
|
||||
Called at turn boundary AFTER delivery. Uses the adapter's pending-message/FIFO machinery
|
||||
so a simultaneous real user message is handled by the same queue and takes priority.
|
||||
@@ -271,9 +254,8 @@ class GatewayGoalsMixin:
|
||||
return
|
||||
|
||||
max_turns = self._goal_max_turns_from_config()
|
||||
|
||||
# Warm the SessionDB cache off-loop: a cold cache runs the state.db init on the loop thread
|
||||
# at the turn boundary; a slow init can drop the goal read and silently end the goal loop.
|
||||
# Cold cache at the turn boundary: a slow state.db init on the loop thread can drop the
|
||||
# goal read and silently end the goal loop.
|
||||
await self._warm_goals_session_db("goal continuation")
|
||||
|
||||
mgr = GoalManager(session_id=sid, default_max_turns=max_turns)
|
||||
@@ -286,23 +268,19 @@ class GatewayGoalsMixin:
|
||||
except Exception:
|
||||
_bg_procs = None
|
||||
|
||||
# evaluate_after_turn calls judge_goal(), a synchronous HTTP request to the auxiliary LLM;
|
||||
# on the event-loop thread it blocks Discord heartbeats 10-40 s and flaps connections, so it
|
||||
# is offloaded to a thread-pool executor. _run_in_executor_with_context (not bare
|
||||
# run_in_executor): the profile secret scope and aux runtime context are contextvars; a
|
||||
# default-executor hop drops them and aux credential resolution fails under multiplexing.
|
||||
# evaluate_after_turn → judge_goal() is a synchronous aux-LLM HTTP call; on the loop thread
|
||||
# it blocks Discord heartbeats 10-40 s. _run_in_executor_with_context (not bare
|
||||
# run_in_executor) carries the profile secret scope / aux runtime contextvars, without which
|
||||
# aux credential resolution fails under multiplexing.
|
||||
decision = await self._run_in_executor_with_context(
|
||||
lambda: mgr.evaluate_after_turn(
|
||||
final_response or "",
|
||||
user_initiated=True,
|
||||
background_processes=_bg_procs,
|
||||
final_response or "", user_initiated=True, background_processes=_bg_procs,
|
||||
),
|
||||
)
|
||||
msg = decision.get("message") or ""
|
||||
|
||||
# Defer the status line until after the adapter has delivered the agent's visible final
|
||||
# response. The judge runs after the response is produced but before BasePlatformAdapter
|
||||
# sends it, so sending here would show "✓ Goal achieved" before the answer itself.
|
||||
# Status line is deferred until the adapter has delivered the visible final response,
|
||||
# otherwise "✓ Goal achieved" would show before the answer itself.
|
||||
if msg and source is not None:
|
||||
await self._defer_goal_status_notice_after_delivery(source, msg)
|
||||
|
||||
@@ -313,59 +291,41 @@ class GatewayGoalsMixin:
|
||||
if not prompt or source is None:
|
||||
return
|
||||
|
||||
# Enqueue via the adapter's FIFO so a user message already in
|
||||
# flight preempts the continuation naturally.
|
||||
# Enqueue via the adapter's FIFO so a user message already in flight preempts naturally.
|
||||
try:
|
||||
adapter = self._adapter_for_source(source)
|
||||
_quick_key = self._session_key_for_source(source)
|
||||
if adapter and _quick_key:
|
||||
cont_event = MessageEvent(
|
||||
text=prompt,
|
||||
message_type=MessageType.TEXT,
|
||||
source=source,
|
||||
message_id=None,
|
||||
channel_prompt=None,
|
||||
)
|
||||
self._enqueue_fifo(_quick_key, cont_event, adapter)
|
||||
self._enqueue_fifo(_quick_key, self._synthetic_prompt_event(source, prompt), adapter)
|
||||
except Exception as exc:
|
||||
logger.debug("goal continuation: enqueue failed: %s", exc)
|
||||
|
||||
async def _run_post_turn_hooks(
|
||||
self,
|
||||
*,
|
||||
agent_result: Any,
|
||||
source: Any,
|
||||
is_internal: bool,
|
||||
event: Any = None,
|
||||
self, *, agent_result: Any, source: Any, is_internal: bool, event: Any = None,
|
||||
) -> None:
|
||||
"""Run goal and loop bookkeeping after an agent turn returns."""
|
||||
final_text = self._final_text_for_post_turn_hooks(agent_result, event)
|
||||
|
||||
try:
|
||||
session_entry = await self.async_session_store.get_or_create_session(
|
||||
source,
|
||||
touch_activity=not is_internal,
|
||||
source, touch_activity=not is_internal,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("post-turn session resolution failed: %s", exc)
|
||||
return
|
||||
|
||||
# Empty interrupted/errored responses must not drive /goal, but an
|
||||
# in-flight /loop tick still needs to be released and rescheduled.
|
||||
# Empty interrupted/errored responses must not drive /goal, but an in-flight /loop tick
|
||||
# still needs to be released and rescheduled.
|
||||
if final_text.strip():
|
||||
try:
|
||||
await self._post_turn_goal_continuation(
|
||||
session_entry=session_entry,
|
||||
source=source,
|
||||
final_response=final_text,
|
||||
session_entry=session_entry, source=source, final_response=final_text,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("goal continuation hook failed: %s", exc)
|
||||
try:
|
||||
await self._post_turn_loop_completion(
|
||||
session_entry=session_entry,
|
||||
source=source,
|
||||
final_response=final_text,
|
||||
session_entry=session_entry, source=source, final_response=final_text,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug("loop completion hook failed: %s", exc)
|
||||
@@ -374,7 +334,7 @@ class GatewayGoalsMixin:
|
||||
def _final_text_for_post_turn_hooks(agent_result, event=None) -> str:
|
||||
"""Text for /goal and /loop after a gateway turn.
|
||||
|
||||
Streamed turns return None from _handle_message_with_agent (already_sent). The delivered
|
||||
Streamed turns return None from _handle_message_with_agent (already_sent); the delivered
|
||||
reply is stashed on the event so those hooks still see it.
|
||||
"""
|
||||
text = ""
|
||||
@@ -390,11 +350,7 @@ class GatewayGoalsMixin:
|
||||
return text
|
||||
|
||||
async def _post_turn_loop_completion(
|
||||
self,
|
||||
*,
|
||||
session_entry: Any,
|
||||
source: Any,
|
||||
final_response: str,
|
||||
self, *, session_entry: Any, source: Any, final_response: str,
|
||||
) -> None:
|
||||
"""Complete a /loop wakeup tick after a gateway turn.
|
||||
|
||||
@@ -412,8 +368,7 @@ class GatewayGoalsMixin:
|
||||
if not sid:
|
||||
return
|
||||
|
||||
# Warm the SessionDB cache off-loop: a cold cache at the turn boundary stalls the loop for
|
||||
# the init duration and can drop the tick-completion write (the /goal continuation seam).
|
||||
# Cold cache at the turn boundary can drop the tick-completion write (see /goal above).
|
||||
await self._warm_goals_session_db("loop completion")
|
||||
|
||||
mgr = LoopManager(session_id=sid)
|
||||
@@ -429,6 +384,69 @@ class GatewayGoalsMixin:
|
||||
if msg and source is not None:
|
||||
await self._defer_goal_status_notice_after_delivery(source, msg)
|
||||
|
||||
async def _loop_wakeup_fire_one(self, sid: str, state: Any, now: float, warned_no_route: set) -> None:
|
||||
"""Inject one due /loop wakeup into its session, applying every deferral rule."""
|
||||
from hermes_cli.loops import LoopManager, goal_blocks_loop_tick
|
||||
|
||||
if state.awaiting_response or now < state.next_due_at:
|
||||
return
|
||||
route = state.route or {}
|
||||
platform_name = route.get("platform", "")
|
||||
chat_id = route.get("chat_id", "")
|
||||
if not platform_name or not chat_id:
|
||||
return # CLI / TUI-owned loop — their own schedulers drive it.
|
||||
adapter = next((a for p, a in self.adapters.items() if p.value == platform_name), None)
|
||||
if adapter is None:
|
||||
if sid not in warned_no_route:
|
||||
warned_no_route.add(sid)
|
||||
logger.debug(
|
||||
"loop wakeup: no adapter for platform %r (session %s)", platform_name, sid,
|
||||
)
|
||||
return
|
||||
|
||||
# Build the source + session key to check business.
|
||||
source = self._build_process_event_source({
|
||||
"session_key": "",
|
||||
"platform": platform_name,
|
||||
"chat_id": chat_id,
|
||||
"chat_type": route.get("chat_type", ""),
|
||||
"thread_id": route.get("thread_id", ""),
|
||||
"user_id": route.get("user_id", ""),
|
||||
"user_name": route.get("user_name", ""),
|
||||
})
|
||||
if source is None:
|
||||
return
|
||||
try:
|
||||
session_key = self._session_key_for_source(source)
|
||||
except Exception:
|
||||
session_key = None
|
||||
if session_key and session_key in self._running_agents:
|
||||
return # busy — stays due, next scan retries
|
||||
if goal_blocks_loop_tick(sid):
|
||||
return
|
||||
|
||||
mgr = LoopManager(session_id=sid)
|
||||
if not mgr.is_due(now):
|
||||
return
|
||||
wakeup = mgr.fire_tick()
|
||||
if not wakeup:
|
||||
return
|
||||
try:
|
||||
logger.info(
|
||||
"loop wakeup #%s — injecting for %s chat=%s thread=%s",
|
||||
mgr.state.ticks_fired if mgr.state else "?",
|
||||
platform_name, source.chat_id, source.thread_id,
|
||||
)
|
||||
await adapter.handle_message(self._synthetic_prompt_event(source, wakeup, internal=True))
|
||||
# Slash-command loops dispatch through the command path and never hit the post-turn
|
||||
# completion hook — complete the tick immediately (caps + scheduling).
|
||||
if wakeup.lstrip().startswith("/"):
|
||||
mgr.complete_tick("")
|
||||
except Exception as exc:
|
||||
logger.warning("loop wakeup injection failed for %s: %s", sid, exc)
|
||||
with suppress(Exception):
|
||||
mgr.abandon_tick()
|
||||
|
||||
async def _loop_wakeup_watcher(self, interval: float = 15.0) -> None:
|
||||
"""Fire due /loop wakeups for idle gateway sessions.
|
||||
|
||||
@@ -443,11 +461,7 @@ class GatewayGoalsMixin:
|
||||
warned_no_route: set = set()
|
||||
while self._running:
|
||||
try:
|
||||
from hermes_cli.loops import (
|
||||
LoopManager,
|
||||
goal_blocks_loop_tick,
|
||||
list_active_loops,
|
||||
)
|
||||
from hermes_cli.loops import list_active_loops
|
||||
|
||||
# Warm the cache off-loop once per scan: the scan reads every persisted loop, so a
|
||||
# cold cache would run the state.db init on the loop thread before the first read.
|
||||
@@ -455,78 +469,7 @@ class GatewayGoalsMixin:
|
||||
|
||||
now = time.time()
|
||||
for sid, state in list_active_loops():
|
||||
if state.awaiting_response or now < state.next_due_at:
|
||||
continue
|
||||
route = state.route or {}
|
||||
platform_name = route.get("platform", "")
|
||||
chat_id = route.get("chat_id", "")
|
||||
if not platform_name or not chat_id:
|
||||
# CLI / TUI-owned loop — their own schedulers drive it.
|
||||
continue
|
||||
adapter = None
|
||||
for p, a in self.adapters.items():
|
||||
if p.value == platform_name:
|
||||
adapter = a
|
||||
break
|
||||
if adapter is None:
|
||||
if sid not in warned_no_route:
|
||||
warned_no_route.add(sid)
|
||||
logger.debug(
|
||||
"loop wakeup: no adapter for platform %r (session %s)",
|
||||
platform_name, sid,
|
||||
)
|
||||
continue
|
||||
|
||||
# Build the source + session key to check business.
|
||||
evt_stub = {
|
||||
"session_key": "",
|
||||
"platform": platform_name,
|
||||
"chat_id": chat_id,
|
||||
"chat_type": route.get("chat_type", ""),
|
||||
"thread_id": route.get("thread_id", ""),
|
||||
"user_id": route.get("user_id", ""),
|
||||
"user_name": route.get("user_name", ""),
|
||||
}
|
||||
source = self._build_process_event_source(evt_stub)
|
||||
if source is None:
|
||||
continue
|
||||
try:
|
||||
session_key = self._session_key_for_source(source)
|
||||
except Exception:
|
||||
session_key = None
|
||||
if session_key and session_key in self._running_agents:
|
||||
continue # busy — stays due, next scan retries
|
||||
if goal_blocks_loop_tick(sid):
|
||||
continue
|
||||
|
||||
mgr = LoopManager(session_id=sid)
|
||||
if not mgr.is_due(now):
|
||||
continue
|
||||
wakeup = mgr.fire_tick()
|
||||
if not wakeup:
|
||||
continue
|
||||
try:
|
||||
synth_event = MessageEvent(
|
||||
text=wakeup,
|
||||
message_type=MessageType.TEXT,
|
||||
source=source,
|
||||
internal=True,
|
||||
)
|
||||
logger.info(
|
||||
"loop wakeup #%s — injecting for %s chat=%s thread=%s",
|
||||
mgr.state.ticks_fired if mgr.state else "?",
|
||||
platform_name, source.chat_id, source.thread_id,
|
||||
)
|
||||
await adapter.handle_message(synth_event)
|
||||
# Slash-command loops dispatch through the command
|
||||
# path and never hit the post-turn completion hook —
|
||||
# complete the tick immediately (caps + scheduling).
|
||||
if wakeup.lstrip().startswith("/"):
|
||||
mgr.complete_tick("")
|
||||
except Exception as exc:
|
||||
logger.warning("loop wakeup injection failed for %s: %s", sid, exc)
|
||||
with suppress(Exception):
|
||||
mgr.abandon_tick()
|
||||
await self._loop_wakeup_fire_one(sid, state, now, warned_no_route)
|
||||
except Exception as exc:
|
||||
logger.debug("loop wakeup watcher error: %s", exc)
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
+1393
-1719
File diff suppressed because it is too large
Load Diff
+782
-1217
File diff suppressed because it is too large
Load Diff
+779
-1367
File diff suppressed because it is too large
Load Diff
+738
-1167
File diff suppressed because it is too large
Load Diff
+518
-659
File diff suppressed because it is too large
Load Diff
+1982
-3698
File diff suppressed because it is too large
Load Diff
+1268
-1808
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user