Teknium
c7e2e0b779
Adds two bounded fast modes on top of the static /fast toggle, default OFF: - `auto`: every user turn opens a `agent.fast_auto_seconds` (default 60s) window; requests inside it carry the provider fast param, later tool-loop requests fall back to standard pricing. - `cold`: the same window, but only on the first turn of a session (no prior user/assistant/tool history). agent/fast_mode.py holds the whole policy: `begin_turn()` at the run_conversation ingress arms `agent._fast_until`; `effective_request_overrides()` is consumed in the ONE place request_overrides feed the transports (build_api_kwargs), so the fast param is a per-request kwarg only. System prompt, tools and messages are untouched — the prompt cache is preserved. resolve_fast_mode_overrides() is now the single gate for static and bounded modes and accepts provider/base_url: OpenRouter, Nous, Copilot, Azure, Bedrock and custom base_urls never receive service_tier/speed (#34308's route gating). Both existing callers (CLI turn route, gateway turn route) and the TUI config.set path pass the route. Surfaces: config `agent.service_tier: auto|cold` + `agent.fast_auto_seconds`, `/fast auto|cold` in CLI, gateway (picker gains both entries), TUI/desktop config.set; status shows the mode; web dashboard select lists the real values. Docs: configuration.md Fast Mode section with mode table + cost note, slash-commands, cli-config.yaml.example, locale strings for the two picker entries. Salvages #89991 (bounded fast modes) and #34308 (route gating). Fixes #64785, #74730. Co-authored-by: Eva <239388517+100yenadmin@users.noreply.github.com> Co-authored-by: kbaicai <kbaicai@qq.com>
64 lines
2.4 KiB
Python
64 lines
2.4 KiB
Python
"""Bounded fast-mode windows (``/fast auto`` and ``/fast cold``).
|
|
|
|
``agent.service_tier`` is ``None`` (normal), ``"priority"`` (static fast),
|
|
``"auto"`` or ``"cold"``. The static value is pinned into
|
|
``agent.request_overrides`` at agent build time; the two bounded modes
|
|
instead open a wall-clock window at each user-turn boundary and layer the
|
|
provider's fast override onto the request kwargs only while it is open:
|
|
|
|
- ``auto`` — every user turn opens a window of ``agent.fast_auto_seconds``.
|
|
- ``cold`` — only the first turn of a session (no prior history) opens it.
|
|
|
|
Only per-request params (``service_tier`` / ``speed``) vary between requests;
|
|
the system prompt, tools, and messages are untouched, so the prompt cache is
|
|
preserved across the window boundary.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from typing import Any
|
|
|
|
BOUNDED_MODES = frozenset({"auto", "cold"})
|
|
DEFAULT_WINDOW_SECONDS = 60
|
|
|
|
|
|
def begin_turn(agent: Any, conversation_history: Any) -> None:
|
|
"""Open (or refuse) the fast window at a user-turn boundary."""
|
|
mode = getattr(agent, "service_tier", None)
|
|
agent._fast_until = 0.0
|
|
if mode not in BOUNDED_MODES:
|
|
return
|
|
if mode == "cold" and any(
|
|
isinstance(m, dict) and m.get("role") in ("user", "assistant", "tool")
|
|
for m in (conversation_history or ())
|
|
):
|
|
return
|
|
try:
|
|
window = float(getattr(agent, "fast_auto_seconds", DEFAULT_WINDOW_SECONDS))
|
|
except (TypeError, ValueError):
|
|
window = DEFAULT_WINDOW_SECONDS
|
|
agent._fast_until = time.monotonic() + max(window, 0.0)
|
|
|
|
|
|
def effective_request_overrides(agent: Any) -> dict[str, Any]:
|
|
"""``agent.request_overrides`` plus the fast override while the window is open."""
|
|
overrides = dict(getattr(agent, "request_overrides", None) or {})
|
|
if getattr(agent, "service_tier", None) not in BOUNDED_MODES:
|
|
return overrides
|
|
if time.monotonic() >= getattr(agent, "_fast_until", 0.0):
|
|
return overrides
|
|
from hermes_cli.models import resolve_fast_mode_overrides
|
|
|
|
base_url = getattr(agent, "base_url", None)
|
|
if getattr(agent, "api_mode", None) == "anthropic_messages":
|
|
base_url = getattr(agent, "_anthropic_base_url", None) or base_url
|
|
fast = resolve_fast_mode_overrides(
|
|
getattr(agent, "model", None),
|
|
provider=getattr(agent, "provider", None),
|
|
base_url=base_url,
|
|
)
|
|
if fast:
|
|
overrides.update(fast)
|
|
return overrides
|