feat(runtime)!: complete legacy removal, local snapshot entries, and TTL cleanup

- Remove legacy provider profiles, admin-token auth, /model command,
  model picker widget, and config.yaml LLM fields (design doc section 10)
- Wire CLI/channels/cron and async sub-agents through the local snapshot
  entry; run creation rejects model config outside runtime_snapshot_id
- Add periodic run-snapshot TTL cleanup to the config service lifespan
- Isolate tests from the real config dir and activate the registry where
  run/model paths fail closed in bootstrap

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
m4
2026-07-21 18:10:23 +08:00
parent 57176b359a
commit 421a664336
82 changed files with 1563 additions and 11787 deletions
+41 -94
View File
@@ -55,12 +55,11 @@ DEFAULT_SKILL_SOURCES = ("/skills/",)
_config = None
_chat_model = None
# Track the (model, provider) binding of _chat_model so cache invalidates
# when config.model/provider change (e.g. via /model). Without this,
# _ensure_chat_model() returns the stale cached instance even after
# _ensure_config(new_cfg) has overwritten the active config — causing
# /model switch to lag one step (see issue #179).
_chat_model_key: tuple[str | None, str | None] | None = None
# Track the (provider_id, model_key, registry_revision) binding of
# _chat_model so the cache invalidates when the registry defaults change.
# The compile-time binding is only a placeholder — per-run resolution
# always comes from the run snapshot via ConfigurableModelMiddleware.
_chat_model_key: tuple[str, str, int] | None = None
# Auxiliary model for background/helper LLM calls (memory workers + main-agent
# tool selector). Cached separately from the main model; resolved through the
@@ -89,10 +88,10 @@ _EvoScientist_agent = None
def set_active_config(cfg) -> None:
"""Commit *cfg* as the active module config.
Public commit path for callers (e.g. ``/model``) that built an agent on
the pure ``create_cli_agent(config=..., chat_model=...)`` path and now
want it to become the session-wide active config. This is the write half
of ``_ensure_config(cfg)`` extracted so the pure path can defer the commit
Public commit path for callers that built an agent on the pure
``create_cli_agent(config=..., chat_model=...)`` path and now want it
to become the session-wide active config. This is the write half of
``_ensure_config(cfg)`` extracted so the pure path can defer the commit
until the agent has been built successfully.
"""
global _config
@@ -119,25 +118,11 @@ def _ensure_config(config=None):
return _config
def _build_chat_model(cfg):
"""Build a chat model from *cfg* without writing any module globals.
Pure-construction counterpart to ``_ensure_chat_model``: used by ``/model``
to verify a switch before committing, and threaded into
``create_cli_agent(chat_model=...)`` so the new agent binds the requested
model without touching the cached ``_chat_model``.
"""
from .llm import get_chat_model
return get_chat_model(model=cfg.model, provider=cfg.provider)
def _replace_chat_model(instance, key: tuple[str | None, str | None]) -> None:
def _replace_chat_model(instance, key: tuple[str, str, int]) -> None:
"""Install a new chat model and propagate the related invariants.
Single write point for ``_chat_model`` / ``_chat_model_key`` /
``_EvoScientist_agent``: both ``_ensure_chat_model`` (cache-miss
rebuild) and ``set_chat_model`` (explicit switch via ``/model``)
``_EvoScientist_agent``: ``_ensure_chat_model`` cache-miss rebuilds
funnel through here so the three globals can never drift.
"""
global _chat_model, _chat_model_key, _EvoScientist_agent
@@ -149,18 +134,24 @@ def _replace_chat_model(instance, key: tuple[str | None, str | None]) -> None:
def _ensure_chat_model():
"""Return cached chat model, rebuilding if cfg.model/provider changed.
"""Return the compile-time main chat model, built from registry defaults.
The cache key is the current config's ``(model, provider)``. If it
differs from the key that built ``_chat_model``, rebuild — this makes
``create_cli_agent(config=temp_cfg)`` bind the freshly requested model
into the new agent without requiring callers to interleave
``set_chat_model()`` calls in any particular order.
The model is resolved from the active registry's ``defaults.primary``
and cached under its ``(provider_id, model_key, registry_revision)``
key. It is only the compile-time placeholder: every run re-resolves its
model from the run snapshot via ``ConfigurableModelMiddleware``.
Raises:
ModelRegistryError: ``MODEL_REGISTRY_NOT_READY`` when the registry
is still in bootstrap (no enabled primary model configured).
"""
cfg = _ensure_config()
key = (cfg.model, cfg.provider)
from .model_registry.runtime import get_snapshot_runtime
runtime = get_snapshot_runtime()
primary_ref, _, revision = runtime.registry_defaults()
key = (primary_ref.provider_id, primary_ref.model_key, revision)
if _chat_model is None or _chat_model_key != key:
_replace_chat_model(_build_chat_model(cfg), key)
_replace_chat_model(runtime.build_default_role_model("primary"), key)
return _chat_model
@@ -170,26 +161,19 @@ def _ensure_auxiliary_chat_model():
Resolves the ``auxiliary`` role through the model registry role mapping
(design doc 6.1): ``registry.defaults.auxiliary ?? defaults.primary``,
constructed via ``build_chat_model`` from the resolved configuration.
The legacy ``cfg.auxiliary_model``/``cfg.auxiliary_provider`` free
strings are no longer consulted.
When the registry is still in bootstrap (the CLI local snapshot entry
point is wired separately) or no auxiliary default is configured — or it
matches the primary default — returns the main ``_ensure_chat_model()``
instance directly, so no second client is built. Otherwise the model is
cached under its ``(provider_id, model_key, registry_revision)`` key.
When no auxiliary default is configured — or it matches the primary
default — returns the main ``_ensure_chat_model()`` instance directly,
so no second client is built. Otherwise the model is cached under its
``(provider_id, model_key, registry_revision)`` key. A bootstrap
registry propagates ``MODEL_REGISTRY_NOT_READY`` from
``_ensure_chat_model()``.
"""
global _auxiliary_chat_model, _auxiliary_chat_model_key
from .model_registry.errors import MODEL_REGISTRY_NOT_READY, ModelRegistryError
from .model_registry.runtime import get_snapshot_runtime
runtime = get_snapshot_runtime()
try:
primary_ref, auxiliary_ref, revision = runtime.registry_defaults()
except ModelRegistryError as exc:
if exc.code == MODEL_REGISTRY_NOT_READY:
return _ensure_chat_model()
raise
primary_ref, auxiliary_ref, revision = runtime.registry_defaults()
if auxiliary_ref is None or auxiliary_ref == primary_ref:
return _ensure_chat_model()
key: tuple[str, str, int] = (
@@ -203,43 +187,6 @@ def _ensure_auxiliary_chat_model():
return _auxiliary_chat_model
def set_chat_model(model: str, provider: str | None = None):
"""Replace the cached chat model with a new one.
Called by ``/model`` to switch the LLM mid-session. No-op when the
cache already holds the requested ``(model, provider)`` — avoids
spawning a second ``get_chat_model`` instance (and its HTTP client)
under the ``/model`` flow where ``_ensure_chat_model`` has already
rebuilt ``_chat_model`` during the preceding ``_load_agent`` call.
Returns the current chat model instance.
"""
from .llm import get_chat_model
# Invalidate the auxiliary cache too: when the registry has no distinct
# auxiliary default the auxiliary model mirrors the main one, so a /model
# switch must let it re-resolve to the new main.
global _auxiliary_chat_model, _auxiliary_chat_model_key
_auxiliary_chat_model = None
_auxiliary_chat_model_key = None
key = (model, provider)
if _chat_model is None or _chat_model_key != key:
_replace_chat_model(get_chat_model(model=model, provider=provider), key)
return _chat_model
def set_chat_model_instance(instance, key: tuple[str | None, str | None]) -> None:
"""Commit an already-built chat model *instance* as the active model.
Companion to ``set_active_config`` for the pure path: installs a model that
``_build_chat_model`` already constructed (e.g. during a ``/model`` verify)
without rebuilding it, keeping ``_chat_model`` / ``_chat_model_key`` /
``_EvoScientist_agent`` in sync via ``_replace_chat_model``. Unlike
``set_chat_model``, the caller owns the ``(model, provider)`` *key*.
"""
_replace_chat_model(instance, key)
# =============================================================================
# MCP caching
# =============================================================================
@@ -781,13 +728,13 @@ def _get_default_middleware(
elif chat_model is None:
tool_selector_model = _ensure_auxiliary_chat_model()
else:
# Pure path (explicit model + config): no free-string auxiliary
# resolution — the threaded model stands in until the local snapshot
# entry point is wired (Task 7).
# Pure path (explicit model + config): the threaded model stands in
# for tool selection at build time; per-call resolution still comes
# from the run snapshot.
tool_selector_model = model
mw = [
ConfigurableModelMiddleware(role=snapshot_role),
create_message_budget_middleware(model, backend),
create_message_budget_middleware(model, backend, snapshot_role=snapshot_role),
create_context_editing_middleware(model),
ContextOverflowMapperMiddleware(),
ToolErrorHandlerMiddleware(),
@@ -948,10 +895,10 @@ def create_cli_agent(
**Pure path:** when *both* ``config`` and ``chat_model`` are explicit, this
writes none of the cached config/model module globals (``_config``,
``_chat_model``, ``_chat_model_key``, ``_EvoScientist_agent``) — the agent
is built purely from the passed-in locals. The caller commits the switch
on success via ``set_active_config`` / ``set_chat_model_instance`` (see
``/model``). Otherwise the existing module-global path runs (langgraph
dev, notebooks, and CLI startup, which pass ``config=`` only).
is built purely from the passed-in locals. The caller commits the config
on success via ``set_active_config``. Otherwise the existing
module-global path runs (langgraph dev, notebooks, and CLI startup, which
pass ``config=`` only).
Args:
workspace_dir: Per-session workspace directory. If ``None``,
-5
View File
@@ -23,11 +23,6 @@ _EXPORTS: dict[str, tuple[str, str]] = {
"save_config": (".config", "save_config"),
"get_effective_config": (".config", "get_effective_config"),
"get_config_path": (".config", "get_config_path"),
# LLM
"get_chat_model": (".llm", "get_chat_model"),
"MODELS": (".llm", "MODELS"),
"list_models": (".llm", "list_models"),
"DEFAULT_MODEL": (".llm", "DEFAULT_MODEL"),
# Prompts
"get_system_prompt": (".prompts", "get_system_prompt"),
# Tools
+4 -5
View File
@@ -150,11 +150,10 @@ class BackgroundAgentLoader(Generic[AgentT]):
def adopt(self, agent: AgentT) -> None:
"""Install an externally-built agent and supersede any in-flight load.
Used by ``/model`` (and any other caller that constructs a
replacement agent directly): bumps the generation token so a
late-arriving background load can't clobber ``self.agent`` via
the done-callback, cancels the in-flight wrapper, and seats the
new agent immediately.
Used by any caller that constructs a replacement agent directly:
bumps the generation token so a late-arriving background load can't
clobber ``self.agent`` via the done-callback, cancels the in-flight
wrapper, and seats the new agent immediately.
"""
prev = self._task
if prev is not None and not prev.done():
+17
View File
@@ -62,6 +62,23 @@ def _create_session_workspace(name: str | None = None) -> str:
return workspace_dir
def current_model_label() -> str:
"""Return the display label for the registry's primary default model.
The CLI no longer carries ``config.model``/``config.provider`` free
strings; the status bar and run metadata label the active model as
``provider_id/model_key`` from the Model Registry defaults. Returns
``"unconfigured"`` when the registry is still in bootstrap.
"""
try:
from ..model_registry.runtime import get_snapshot_runtime
primary, _, _ = get_snapshot_runtime().registry_defaults()
except Exception:
return "unconfigured"
return f"{primary.provider_id}/{primary.model_key}"
def _load_agent(
workspace_dir: str | None = None,
checkpointer=None,
+1 -1
View File
@@ -306,7 +306,7 @@ async def dispatch_channel_slash_command(
resolution, or the dispatcher's input agent when no resolver is
supplied. Callers can compare ``ctx.agent`` with
``original_agent`` to detect command-driven swaps. Used by Rich
CLI to (a) adopt an agent swap (``/model``) back into the
CLI to (a) adopt an agent swap back into the
running session and (b) refresh the status snapshot for
commands that mutate session-level state (``/new``,
``/compact``) — mirrors the REPL dispatch at
+62 -94
View File
@@ -74,16 +74,9 @@ if TYPE_CHECKING:
@app.command()
def onboard(
skip_validation: bool = typer.Option(
False, "--skip-validation", help="Skip API key validation during setup"
False, "--skip-validation", help="Skip Tavily key validation during setup"
),
# ---- Pre-fill answers (any subset; remaining prompts stay interactive)
provider: str | None = typer.Option(
None, "--provider", help="Pre-set LLM provider (e.g. anthropic, openai)"
),
model: str | None = typer.Option(None, "--model", help="Pre-set model name"),
api_key: str | None = typer.Option(
None, "--api-key", help="Pre-set API key for the chosen --provider"
),
tavily_key: str | None = typer.Option(
None, "--tavily-key", help="Pre-set Tavily API key"
),
@@ -120,16 +113,16 @@ def onboard(
):
"""Interactive setup wizard for EvoScientist.
Guides you through configuring API keys, model selection,
workspace settings, and agent parameters.
Guides you through workspace settings, channels, and agent parameters.
Models and provider credentials are configured through the Model
Registry (WebUI configuration page or the model-registry API), not
here.
Any answer can be pre-set via a flag (``--provider anthropic
--model claude-sonnet-4-6 ...``); prompts for unset answers stay
Any answer can be pre-set via a flag; prompts for unset answers stay
interactive unless ``--non-interactive`` is passed, in which case any
missing required answer aborts the wizard.
"""
from ..config.onboard.constants import (
VALID_PROVIDERS,
VALID_UI_BACKENDS,
VALID_WORKSPACE_MODES,
)
@@ -149,11 +142,6 @@ def onboard(
f"--workspace-mode must be one of {sorted(VALID_WORKSPACE_MODES)}",
param_hint="--workspace-mode",
)
if provider is not None and provider not in VALID_PROVIDERS:
raise typer.BadParameter(
f"--provider must be one of {sorted(VALID_PROVIDERS)}",
param_hint="--provider",
)
# Match the interactive prompt's range (1024 < port < 65536). Without
# this check, --port 80 or --port 99999 would land in config and break
# the langgraph dev server on startup.
@@ -169,12 +157,6 @@ def onboard(
answers["ui"] = ui
if port is not None:
answers["port"] = str(port)
if provider is not None:
answers["provider"] = provider
if model is not None:
answers["model"] = model
if api_key is not None:
answers["api_key"] = api_key
if tavily_key is not None:
answers["tavily_key"] = tavily_key
if workspace_mode is not None:
@@ -212,8 +194,6 @@ def onboard(
_CONFIGURE_SECTIONS = {
"ui": "UI backend",
"port": "LangGraph server port",
"provider": "LLM provider + auth + API key",
"model": "Model + reasoning effort",
"tavily": "Tavily search key",
"workspace": "Workspace mode",
"thinking": "Thinking panel",
@@ -263,29 +243,6 @@ def configure_port():
_configure_section("port")
@configure_app.command("provider")
def configure_provider(
skip_validation: bool = typer.Option(False, "--skip-validation"),
):
"""Re-run LLM provider, auth mode, and API key prompts.
Model selection is automatically re-run after provider — the model list
depends on the provider, and silently leaving e.g. ``model="claude-...""``
when the provider was switched to ``openai`` would break the first
request. Press Enter on the model picker to keep the current default.
"""
_run_onboard_cli(
skip_validation=skip_validation,
only_sections={"provider", "model"},
)
@configure_app.command("model")
def configure_model():
"""Re-run model selection (and reasoning effort for OpenRouter)."""
_configure_section("model")
@configure_app.command("tavily")
def configure_tavily(
skip_validation: bool = typer.Option(False, "--skip-validation"),
@@ -1022,7 +979,7 @@ def _make_serve_cmd_completed_hook(
):
"""Build the ``on_cmd_completed`` hook used by serve mode.
Adopts ``/model`` agent swaps and ``/resume`` thread/workspace
Adopts command-driven agent swaps and ``/resume`` thread/workspace
swaps back into ``runtime_state`` so the outer poll loop picks up
the new handles on subsequent messages. Also keeps
``channel_runtime`` in sync so the bus sees the new values.
@@ -1389,6 +1346,47 @@ def _serve_drain_notifications(
_notif_loop.close()
def _startup_gates() -> None:
"""Refuse agent startup on legacy artifacts or a bootstrap registry.
Design doc section 10 step 4 and section 8.1:
1. Legacy model-configuration artifacts (old ``providers.yaml``, old
``run-runtime-snapshots.sqlite3``, leftover LLM fields in
``config.yaml``) abort startup with an explicit reset guide —
nothing is read partially.
2. A bootstrap registry aborts startup with
``MODEL_REGISTRY_NOT_READY`` — there is no implicit default model
anymore; configure and enable a primary model through the Model
Registry first.
"""
from ..config.legacy_artifacts import (
LegacyArtifactsError,
assert_no_legacy_artifacts,
)
from ..model_registry.errors import MODEL_REGISTRY_NOT_READY, ModelRegistryError
try:
assert_no_legacy_artifacts()
except LegacyArtifactsError as exc:
console.print(f"[red]{exc}[/red]")
raise typer.Exit(1) from exc
try:
from ..model_registry.runtime import get_snapshot_runtime
get_snapshot_runtime().registry_defaults()
except ModelRegistryError as exc:
if exc.code != MODEL_REGISTRY_NOT_READY:
raise
console.print(
f"[red]{exc.code}: {exc}[/red]\n"
"Configure a provider/model through the Model Registry (WebUI "
"configuration page or the model-registry API), run the provider "
"test, enable the model, and set it as the primary default."
)
raise typer.Exit(1) from exc
@app.command()
def serve(
no_thinking: bool = typer.Option(
@@ -1452,20 +1450,7 @@ def serve(
if debug:
_configure_logging()
# Auto-start ccproxy if any provider uses OAuth mode
_ccproxy_proc_serve = None
if config.anthropic_auth_mode == "oauth" or config.openai_auth_mode == "oauth":
try:
from ..ccproxy_manager import maybe_start_ccproxy, stop_ccproxy
_ccproxy_proc_serve = maybe_start_ccproxy(config)
if _ccproxy_proc_serve:
import atexit
atexit.register(stop_ccproxy, _ccproxy_proc_serve)
except RuntimeError as exc:
console.print(f"[red]{exc}[/red]")
raise typer.Exit(1) from exc
_startup_gates()
if not config.channel_enabled:
console.print("[red]No channels configured.[/red]")
@@ -1565,6 +1550,9 @@ def serve(
_orig_sigterm = signal.signal(signal.SIGTERM, _handle_shutdown)
try:
from .agent import current_model_label
model_label = current_model_label()
while not shutdown_event.is_set():
try:
msg = _message_queue.get(timeout=0.5)
@@ -1577,7 +1565,7 @@ def serve(
_serve_process_message(
msg,
runtime_state=runtime_state,
model=config.model,
model=model_label,
workspace_dir=ws,
show_thinking=effective_channel_thinking,
on_cmd_completed=_serve_on_cmd_completed,
@@ -1593,7 +1581,7 @@ def serve(
if async_notifier.has_pending_notifications(runtime_state.thread_id):
_serve_drain_notifications(
runtime_state=runtime_state,
model=config.model,
model=model_label,
workspace_dir=ws,
show_thinking=effective_channel_thinking,
)
@@ -2084,11 +2072,6 @@ def _main_callback(
"--dangerous",
help="DANGEROUS: real-filesystem access (no workspace confinement); implies --auto-approve",
),
auth_mode: str | None = typer.Option(
None,
"--auth-mode",
help="Auth mode for Anthropic/OpenAI: api_key (default) or oauth (ccproxy).",
),
ui: str | None = typer.Option(
None,
"--ui",
@@ -2168,29 +2151,11 @@ def _main_callback(
cli_overrides["enable_ask_user"] = True
if dangerous:
cli_overrides["dangerous_mode"] = True
if auth_mode:
if auth_mode not in ("api_key", "oauth"):
raise typer.BadParameter("--auth-mode must be 'api_key' or 'oauth'")
cli_overrides["anthropic_auth_mode"] = auth_mode
cli_overrides["openai_auth_mode"] = auth_mode
config = get_effective_config(cli_overrides)
apply_config_to_env(config)
# Auto-start ccproxy if any provider uses OAuth mode
_ccproxy_proc = None
if config.anthropic_auth_mode == "oauth" or config.openai_auth_mode == "oauth":
try:
from ..ccproxy_manager import maybe_start_ccproxy, stop_ccproxy
_ccproxy_proc = maybe_start_ccproxy(config)
if _ccproxy_proc:
import atexit
atexit.register(stop_ccproxy, _ccproxy_proc)
except RuntimeError as exc:
console.print(f"[red]{exc}[/red]")
raise typer.Exit(1) from exc
_startup_gates()
show_thinking = config.show_thinking if not no_thinking else False
effective_channel_thinking = config.channel_send_thinking and (not no_thinking)
@@ -2356,6 +2321,9 @@ def _main_callback(
config=config,
)
try:
from .agent import current_model_label
model_label = current_model_label()
if effective_output_format == "stream-json":
# Headless JSONL path: drive the sink through the gateway
# directly. We are already inside the async single-shot
@@ -2364,7 +2332,7 @@ def _main_callback(
request = RunRequest(
message=prompt,
thread_id=tid,
metadata=build_metadata(workspace_dir, config.model),
metadata=build_metadata(workspace_dir, model_label),
target=GraphTarget(
local_graph=agent, workspace_dir=workspace_dir
),
@@ -2388,7 +2356,7 @@ def _main_callback(
thread_id=tid,
show_thinking=show_thinking,
workspace_dir=workspace_dir,
model=config.model,
model=model_label,
ui_backend=config.ui_backend,
runtime_gateways=runtime_gateways,
)
@@ -2403,6 +2371,7 @@ def _main_callback(
nest_asyncio.apply()
asyncio.get_event_loop().run_until_complete(_single_shot())
else:
from .agent import current_model_label
from .interactive import cmd_interactive
# Interactive mode (default) — checkpointer managed inside cmd_interactive
@@ -2412,8 +2381,7 @@ def _main_callback(
workspace_dir=workspace_dir,
workspace_fixed=workspace_fixed,
mode=effective_mode,
model=config.model,
provider=config.provider,
model=current_model_label(),
run_name=name,
thread_id=thread_id,
ui_backend=config.ui_backend,
+8 -10
View File
@@ -961,7 +961,7 @@ def cmd_interactive(
ctx: CommandContext, original_agent: Any, cmd: Command
) -> None:
"""Mirror the REPL adoption block at
``interactive.py:1005-1030`` so ``/model`` and similar
``interactive.py:1005-1030`` so agent-swap and similar
state-mutating commands invoked via a channel actually
rebind the running session and keep the status bar
in sync."""
@@ -970,11 +970,10 @@ def cmd_interactive(
ctx.agent is not None and ctx.agent is not original_agent
)
if agent_swapped:
from ..EvoScientist import _ensure_config
from .agent import current_model_label
agent_loader.adopt(ctx.agent)
cfg = _ensure_config()
model = cfg.model
model = current_model_label()
state["status_base_snapshot"] = make_empty_status_snapshot(
model
)
@@ -1326,7 +1325,7 @@ def cmd_interactive(
if not state["running"]:
break
# Agent swap (e.g. /model successfully built a
# Agent swap (a command successfully built a
# new agent): adopt into loader + reset status
# snapshot + sync channel runtime.
agent_swapped = (
@@ -1334,11 +1333,10 @@ def cmd_interactive(
and ctx.agent is not _agent_for_ctx
)
if agent_swapped:
from ..EvoScientist import _ensure_config
from .agent import current_model_label
agent_loader.adopt(ctx.agent)
cfg = _ensure_config()
model = cfg.model
model = current_model_label()
state["status_base_snapshot"] = (
make_empty_status_snapshot(model)
)
@@ -1361,8 +1359,8 @@ def cmd_interactive(
# Commands that mutate status fields need an
# async refresh here (/compact + /new use sync
# callbacks; /model swaps the agent). /resume
# awaits its own refresh inline inside the
# callbacks; an agent swap rebuilds the snapshot).
# /resume awaits its own refresh inline inside the
# async callback.
if agent_swapped or _cmd.name in ("/compact", "/new"):
await _refresh_status_snapshot(
-35
View File
@@ -19,7 +19,6 @@ from collections.abc import Awaitable, Callable
from typing import Any
from rich.console import Console
from rich.table import Table
from ..commands.base import CommandUI
@@ -73,40 +72,6 @@ class RichCLICommandUI(CommandUI):
# Rich console flushes synchronously; nothing to await.
return
# ── /model interactive picker fallback ──────────────────
async def wait_for_model_pick(
self,
entries: list[tuple[str, str, str]],
current_model: str | None,
current_provider: str | None,
) -> tuple[str, str] | None:
"""Print the model table and return ``None``; user re-runs with
``/model <name>`` since the CLI has no interactive picker."""
table = Table(
title="Available Models",
show_header=True,
header_style="bold cyan",
)
table.add_column("Name", style="bold")
table.add_column("Provider", style="dim")
for name, _mid, prov in entries:
marker = " *" if name == current_model and prov == current_provider else ""
table.add_row(f"{name}{marker}", prov)
self.console.print(table)
self.console.print(
"[dim]Usage: /model <name> [provider] [--save] — "
"provider is optional, auto-detected from model name[/dim]"
)
return None
def update_status_after_model_change(
self, new_model: str, new_provider: str | None = None
) -> None:
"""No-op; the CLI REPL refreshes status itself after detecting an
``ctx.agent`` change post-``cmd_manager.execute``."""
return
# ── Interactive pickers ────────────────────────────────
async def wait_for_thread_pick(
+3 -109
View File
@@ -222,13 +222,12 @@ async def _sync_tui_command_completion(
"""Adopt successful command-side state changes back into the TUI app."""
agent_swapped = ctx.agent is not None and ctx.agent is not original_agent
if agent_swapped:
from ..EvoScientist import _ensure_config
from .agent import current_model_label
app._agent_loader.adopt(ctx.agent)
cfg = _ensure_config()
update_model = getattr(app, "update_status_after_model_change", None)
if callable(update_model):
update_model(cfg.model, cfg.provider)
update_model(current_model_label())
# Rebind the runtime whenever the agent OR thread_id may have moved
# — ``/new`` and ``/resume`` rotate ``app._conversation_tid``
@@ -456,7 +455,6 @@ def run_textual_interactive(
self._picker_future: asyncio.Future | None = None
self._browser_future: asyncio.Future | None = None
self._mcp_browser_future: asyncio.Future | None = None
self._model_picker_future: asyncio.Future | None = None
self._history_suggester = HistorySuggester(DATA_DIR / "history")
self._history_index: int = -1 # -1 = not browsing history
self._history_saved_input: str = "" # saved current input before browsing
@@ -627,26 +625,6 @@ def run_textual_interactive(
return await self._wait_for_mcp_browse(browser)
async def wait_for_model_pick(
self,
entries: list[tuple[str, str, str]],
current_model: str | None,
current_provider: str | None,
) -> tuple[str, str] | None:
from .widgets.model_picker import ModelPickerWidget
container = self.query_one("#chat", VerticalScroll)
picker = ModelPickerWidget(
entries,
current_model=current_model,
current_provider=current_provider,
)
await container.mount(picker)
self._anchor_chat(container)
picker.focus()
return await self._wait_for_model_pick(picker)
def clear_chat(self) -> None:
container = self.query_one("#chat", VerticalScroll)
welcome = self.query_one("#welcome", Static)
@@ -1243,34 +1221,6 @@ def run_textual_interactive(
if self._mcp_browser_future and not self._mcp_browser_future.done():
self._mcp_browser_future.set_result(None)
async def _wait_for_model_pick(self, picker_widget) -> tuple[str, str] | None:
"""Wait for user to pick a model from ModelPickerWidget.
Returns ``(name, provider)`` or ``None`` on cancel/timeout.
"""
self._model_picker_future = asyncio.get_event_loop().create_future()
try:
return await asyncio.wait_for(self._model_picker_future, timeout=120)
except (TimeoutError, asyncio.CancelledError):
return None
finally:
self._model_picker_future = None
try:
picker_widget.remove()
except Exception:
_channel_logger.debug("model picker cleanup failed", exc_info=True)
self.query_one("#prompt", ChatTextArea).focus()
def on_model_picker_widget_picked(self, event) -> None: # type: ignore[override]
"""Handle ModelPickerWidget.Picked message."""
if self._model_picker_future and not self._model_picker_future.done():
self._model_picker_future.set_result((event.name, event.provider))
def on_model_picker_widget_cancelled(self, event) -> None: # type: ignore[override]
"""Handle ModelPickerWidget.Cancelled message."""
if self._model_picker_future and not self._model_picker_future.done():
self._model_picker_future.set_result(None)
# ── Streaming core ─────────────────────────────────────
async def _stream_with_widgets(
@@ -2489,7 +2439,6 @@ def run_textual_interactive(
if focused is not None:
from .widgets.approval_widget import ApprovalWidget
from .widgets.mcp_browser import MCPBrowserWidget
from .widgets.model_picker import ModelPickerWidget
from .widgets.skill_browser import SkillBrowserWidget
from .widgets.thread_selector import ThreadPickerWidget
@@ -2505,33 +2454,6 @@ def run_textual_interactive(
if isinstance(focused, MCPBrowserWidget):
focused.action_cancel()
return
# ModelPickerWidget: when in "Custom Ollama" input mode, its
# child Input widget owns focus, so ``focused`` isn't the
# picker itself. Walk the parent chain to find it, then let
# the widget's own action_cancel decide whether to close the
# picker (list mode) or just exit input mode.
picker: ModelPickerWidget | None = None
if isinstance(focused, ModelPickerWidget):
picker = focused
else:
node = focused.parent
while node is not None and not isinstance(node, ModelPickerWidget):
node = node.parent
picker = node
if picker is not None:
prev_mode = getattr(picker, "_mode", "list")
picker.action_cancel()
# In list mode action_cancel posted Cancelled; resolve the
# future immediately to avoid a frame of lag. In input
# mode action_cancel flipped back to list — keep picker
# open, do NOT close the future.
if prev_mode == "list":
if (
self._model_picker_future
and not self._model_picker_future.done()
):
self._model_picker_future.set_result(None)
return
if self._queued_messages:
self._queued_messages.pop()
self._render_queue_indicator()
@@ -2555,7 +2477,6 @@ def run_textual_interactive(
from .widgets.approval_widget import ApprovalWidget
from .widgets.ask_user_widget import AskUserWidget
from .widgets.mcp_browser import MCPBrowserWidget
from .widgets.model_picker import ModelPickerWidget
from .widgets.skill_browser import SkillBrowserWidget
from .widgets.thread_selector import ThreadPickerWidget
@@ -2574,20 +2495,6 @@ def run_textual_interactive(
if isinstance(focused, MCPBrowserWidget):
focused.action_move_up()
return
# ModelPickerWidget: Up from the Custom Ollama Input child
# must reach the picker (to exit input mode). See the Esc
# handler above for the parent-walk rationale.
picker_up: ModelPickerWidget | None = None
if isinstance(focused, ModelPickerWidget):
picker_up = focused
else:
node = focused.parent
while node is not None and not isinstance(node, ModelPickerWidget):
node = node.parent
picker_up = node
if picker_up is not None:
picker_up.action_move_up()
return
if self._queued_messages:
last = self._queued_messages.pop()
prompt = self.query_one("#prompt", ChatTextArea)
@@ -2623,7 +2530,6 @@ def run_textual_interactive(
from .widgets.approval_widget import ApprovalWidget
from .widgets.ask_user_widget import AskUserWidget
from .widgets.mcp_browser import MCPBrowserWidget
from .widgets.model_picker import ModelPickerWidget
from .widgets.skill_browser import SkillBrowserWidget
from .widgets.thread_selector import ThreadPickerWidget
@@ -2642,18 +2548,6 @@ def run_textual_interactive(
if isinstance(focused, MCPBrowserWidget):
focused.action_move_down()
return
# Same parent-walk rationale as action_edit_queued / cancel.
picker_down: ModelPickerWidget | None = None
if isinstance(focused, ModelPickerWidget):
picker_down = focused
else:
node = focused.parent
while node is not None and not isinstance(node, ModelPickerWidget):
node = node.parent
picker_down = node
if picker_down is not None:
picker_down.action_move_down()
return
# History browsing (down key)
if self._history_index >= 0:
@@ -3069,7 +2963,7 @@ def run_textual_interactive(
def update_status_after_model_change(
self, new_model: str, new_provider: str | None = None
) -> None:
"""Update the status bar and welcome banner after /model switches the LLM."""
"""Update the status bar and welcome banner after an agent swap."""
self._current_model = new_model
if new_provider is not None:
self._current_provider = new_provider
-390
View File
@@ -1,390 +0,0 @@
"""Inline model picker widget for /model command in TUI.
Keyboard-driven widget mounted directly into the chat container.
Models are grouped by provider with a search/filter input.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Any, ClassVar, Literal
from rich.text import Text
from textual.binding import Binding, BindingType
from textual.containers import Container
from textual.message import Message
from textual.widget import Widget
from textual.widgets import Input, Static
if TYPE_CHECKING:
from textual import events
from textual.app import ComposeResult
# Sentinel ``model_id`` used for the "Custom Ollama model..." pseudo-row.
# Selecting this row switches the widget into free-text input mode instead
# of posting ``Picked`` — the user types a model name, Enter confirms.
_CUSTOM_OLLAMA_ID = "__custom_ollama__"
def _build_items(
entries: list[tuple[str, str, str]],
current_model: str | None = None,
current_provider: str | None = None,
filter_text: str = "",
) -> list[dict]:
"""Build the flat item list rendered by ModelPickerWidget.
Returns a list of::
{"type": "header", "label": str}
{"type": "model", "name": str, "model_id": str, "provider": str, "current": bool}
"""
# Apply filter. The Custom Ollama sentinel is the user's escape hatch
# when no local models match; it must remain visible regardless of filter.
if filter_text:
ft = filter_text.lower()
entries = [
(n, mid, p)
for n, mid, p in entries
if mid == _CUSTOM_OLLAMA_ID or ft in n.lower() or ft in p.lower()
]
# Group by provider preserving order. Deduplicate the Custom Ollama
# sentinel defensively — if callers somehow pass two sentinel rows
# (state reuse, stale merges), collapse them into one to avoid
# rendering duplicate "Custom Ollama model..." rows in the picker.
groups: dict[str, list[tuple[str, str, str]]] = {}
seen_sentinel = False
for name, model_id, provider in entries:
if model_id == _CUSTOM_OLLAMA_ID:
if seen_sentinel:
continue
seen_sentinel = True
if provider not in groups:
groups[provider] = []
groups[provider].append((name, model_id, provider))
items: list[dict] = []
for provider, models in groups.items():
items.append({"type": "header", "label": provider})
for name, model_id, prov in models:
is_current = name == current_model and prov == current_provider
items.append(
{
"type": "model",
"name": name,
"model_id": model_id,
"provider": prov,
"current": is_current,
}
)
return items
class ModelPickerWidget(Widget):
"""Inline model picker -- mounts in chat, keyboard-driven.
Posts ``Picked(name, provider)`` on Enter, ``Cancelled()`` on Esc.
Type to filter models.
"""
can_focus = True
# Required so the Custom Ollama ``Input`` child can hold focus when the
# user is typing a model name.
can_focus_children = True
DEFAULT_CSS = """
ModelPickerWidget {
height: auto;
max-height: 30;
margin: 1 0;
padding: 0 1;
background: $surface;
border: solid $primary;
}
ModelPickerWidget .picker-custom-input {
height: 3;
margin: 1 0 0 0;
}
ModelPickerWidget .picker-title {
height: 1;
text-style: bold;
color: $primary;
}
ModelPickerWidget .picker-filter {
height: 1;
padding: 0 1;
color: $text;
}
ModelPickerWidget .picker-rows {
height: auto;
max-height: 22;
overflow-y: auto;
}
ModelPickerWidget .picker-header {
height: 1;
padding: 0 1;
margin-top: 1;
}
ModelPickerWidget .picker-row {
height: 1;
padding: 0 1;
}
ModelPickerWidget .picker-row-selected {
background: $primary;
text-style: bold;
}
ModelPickerWidget .picker-help {
height: 1;
color: $text-muted;
text-style: italic;
}
"""
BINDINGS: ClassVar[list[BindingType]] = [
Binding("up", "move_up", "Up", show=False),
Binding("down", "move_down", "Down", show=False),
Binding("enter", "select", "Select", show=False),
Binding("escape", "cancel", "Cancel", show=False),
Binding("backspace", "backspace", "Backspace", show=False),
]
class Picked(Message):
def __init__(self, name: str, provider: str) -> None:
super().__init__()
self.name = name
self.provider = provider
class Cancelled(Message):
"""Posted when user cancels selection."""
def __init__(
self,
entries: list[tuple[str, str, str]],
*,
current_model: str | None = None,
current_provider: str | None = None,
title: str = ">>> Select model <<<",
**kwargs: Any,
) -> None:
super().__init__(**kwargs)
self._entries = entries
self._current_model = current_model
self._current_provider = current_provider
self._title = title
self._filter_text = ""
self._items = _build_items(
entries,
current_model=current_model,
current_provider=current_provider,
)
self._selected = self._first_model_index()
self._row_widgets: list[Static] = []
self._filter_widget: Static | None = None
# "list" = arrow-key selection over models; "input" = free-text entry
# for Custom Ollama model name. Transitions: selecting the sentinel
# row enters input mode; Esc or Up arrow inside input mode returns to
# list mode without closing the picker.
self._mode: Literal["list", "input"] = "list"
self._custom_input: Input | None = None
def _first_model_index(self) -> int:
for i, item in enumerate(self._items):
if item["type"] == "model":
return i
return 0
def _move(self, direction: int) -> None:
if not self._items:
return
i = (self._selected + direction) % len(self._items)
steps = 0
while self._items[i]["type"] != "model" and steps < len(self._items):
i = (i + direction) % len(self._items)
steps += 1
if self._items[i]["type"] == "model":
self._selected = i
self._update_rows()
def _rebuild(self) -> None:
"""Rebuild items from filter and re-render."""
self._items = _build_items(
self._entries,
current_model=self._current_model,
current_provider=self._current_provider,
filter_text=self._filter_text,
)
self._selected = self._first_model_index()
# Re-mount rows
rows_container = self.query_one(".picker-rows", Container)
for w in list(rows_container.children):
w.remove()
self._row_widgets.clear()
for item in self._items:
css = "picker-header" if item["type"] == "header" else "picker-row"
widget = Static("", classes=css)
self._row_widgets.append(widget)
rows_container.mount(widget)
self._update_rows()
self._update_filter()
def compose(self) -> ComposeResult:
yield Static(self._title, classes="picker-title")
self._filter_widget = Static("", classes="picker-filter")
yield self._filter_widget
with Container(classes="picker-rows"):
for item in self._items:
css = "picker-header" if item["type"] == "header" else "picker-row"
widget = Static("", classes=css)
self._row_widgets.append(widget)
yield widget
# Hidden until the user selects "Custom Ollama model..." \u2014 then shown
# and focused for free-text entry of an Ollama model name.
self._custom_input = Input(
placeholder="Type Ollama model name (e.g. llama3.3)...",
classes="picker-custom-input",
)
self._custom_input.display = False
yield self._custom_input
yield Static(
"\u2191/\u2193 navigate \u00b7 Enter select \u00b7 Type to filter \u00b7 Esc cancel",
classes="picker-help",
)
def on_mount(self) -> None:
self._update_rows()
self._update_filter()
self.call_later(self.focus)
def _update_filter(self) -> None:
if self._filter_widget is not None:
if self._filter_text:
t = Text()
t.append(" Filter: ", style="dim")
t.append(self._filter_text, style="bold")
t.append("\u2588", style="blink")
self._filter_widget.update(t)
else:
self._filter_widget.update(
Text(" Type to filter...", style="dim italic")
)
def _update_rows(self) -> None:
for i, (item, widget) in enumerate(
zip(self._items, self._row_widgets, strict=False)
):
widget.remove_class("picker-row-selected")
if item["type"] == "header":
t = Text()
t.append("\u2500\u2500 ", style="bold cyan")
t.append(item["label"], style="bold cyan")
widget.update(t)
else:
is_selected = i == self._selected
t = Text()
cursor = "\u25b8 " if is_selected else " "
t.append(cursor, style="bold cyan" if is_selected else "dim")
t.append(item["name"], style="bold" if is_selected else "")
if item["current"]:
t.append(" *", style="bold green")
t.append(f" ({item['provider']})", style="dim italic")
widget.update(t)
if is_selected:
widget.add_class("picker-row-selected")
widget.scroll_visible()
def on_key(self, event: events.Key) -> None:
# In input mode, the Input child owns printable keys + backspace.
if self._mode == "input":
return
# Let bindings handle special keys
if event.key in ("up", "down", "enter", "escape", "backspace"):
return
# Printable characters -> filter
if event.character and event.character.isprintable():
self._filter_text += event.character
self._rebuild()
event.prevent_default()
def action_backspace(self) -> None:
if self._mode == "input":
# Input widget handles its own backspace.
return
if self._filter_text:
self._filter_text = self._filter_text[:-1]
self._rebuild()
def action_move_up(self) -> None:
if self._mode == "input":
# Up from the Input field escapes back to list selection.
self._exit_input_mode()
return
self._move(-1)
def action_move_down(self) -> None:
if self._mode == "input":
# Down in input mode is ambiguous; absorb rather than toggle.
return
self._move(1)
def action_select(self) -> None:
if self._mode == "input":
self._submit_custom_input()
return
if not self._items or self._selected >= len(self._items):
self.post_message(self.Cancelled())
return
item = self._items[self._selected]
if item["type"] != "model":
self.post_message(self.Cancelled())
return
if item["provider"] == "ollama" and item["model_id"] == _CUSTOM_OLLAMA_ID:
self._enter_input_mode()
return
self.post_message(self.Picked(item["name"], item["provider"]))
def action_cancel(self) -> None:
if self._mode == "input":
# Esc returns to list selection; does NOT close the picker.
self._exit_input_mode()
return
self.post_message(self.Cancelled())
def on_blur(self, event: events.Blur) -> None:
# When the Input child has focus we must NOT steal it back.
if self._mode == "input":
return
self.call_after_refresh(self.focus)
def on_input_submitted(self, event: Input.Submitted) -> None:
"""Safety net: Enter fired inside the Input widget rather than
bubbling to ``action_select``. Route to the same submit path."""
if event.input is self._custom_input:
event.stop()
self._submit_custom_input()
def _enter_input_mode(self) -> None:
"""Show the Custom Ollama Input and move focus into it."""
self._mode = "input"
if self._custom_input is not None:
self._custom_input.display = True
# Carry any filter text over as a nice touch — user may have
# started typing a model name thinking it would filter.
self._custom_input.value = self._filter_text
self._custom_input.focus()
def _exit_input_mode(self) -> None:
"""Hide the Input and return focus to the list."""
self._mode = "list"
if self._custom_input is not None:
self._custom_input.display = False
self._custom_input.value = ""
self.focus()
def _submit_custom_input(self) -> None:
"""Confirm the typed Ollama model name. Empty input is a no-op —
user can Esc out or keep typing."""
typed = (self._custom_input.value if self._custom_input else "").strip()
if not typed:
return
self.post_message(self.Picked(typed, "ollama"))
+1 -1
View File
@@ -12,7 +12,7 @@ class CompletionKind(StrEnum):
EMPTY = "empty"
_CATEGORY_ORDER = ["Session", "Skills", "MCP", "Channels", "Model", "General"]
_CATEGORY_ORDER = ["Session", "Skills", "MCP", "Channels", "General"]
@dataclass(frozen=True)
-6
View File
@@ -47,12 +47,6 @@ class CommandUI(Protocol):
async def wait_for_mcp_browse(
self, servers: list, installed_names: set[str], pre_filter_tag: str
) -> list | None: ...
async def wait_for_model_pick(
self,
entries: list[tuple[str, str, str]],
current_model: str | None,
current_provider: str | None,
) -> tuple[str, str] | None: ...
def clear_chat(self) -> None: ...
def request_quit(self) -> None: ...
def force_quit(self) -> None: ...
@@ -5,7 +5,6 @@ from . import (
channel,
general,
mcp,
model,
schedule,
session,
skills,
@@ -16,7 +15,6 @@ __all__ = [
"channel",
"general",
"mcp",
"model",
"schedule",
"session",
"skills",
@@ -1,204 +0,0 @@
from __future__ import annotations
from typing import ClassVar
from ..base import Argument, Command, CommandContext
from ..manager import manager
def extract_model_and_provider(args: list[str]) -> tuple[str, str]:
"""Parse model name and provider from argument list.
Args:
args: Non-empty argument list (model_name [provider]).
Returns:
``(model_name, provider)`` tuple.
Raises:
ValueError: If the model is not in the registry. Skipped when
``provider_override == "ollama"``, since Ollama models are
locally-installed and never appear in ``MODELS``.
"""
from ...llm.models import MODELS
model_name = args[0]
provider_override = args[1] if len(args) > 1 else None
# Ollama models are locally-installed — not in the registry. Pass the name
# through verbatim; get_chat_model's "Assume full model ID" fallback
# (models.py) accepts them.
if provider_override == "ollama":
return model_name, "ollama"
if model_name not in MODELS:
raise ValueError(f"Unknown model '{model_name}'")
if provider_override:
provider = provider_override
else:
_, provider = MODELS[model_name]
return model_name, provider
class ModelCommand(Command):
"""Switch the LLM model for the current session."""
name = "/model"
description = "Switch model (--save to persist)"
category = "Model"
# ``--save`` is parsed manually in ``execute`` via ``"--save" in args``;
# ``type=bool`` below is declarative metadata, not enforced by the manager.
arguments: ClassVar[list[Argument]] = [
Argument(
name="model_name",
type=str,
description="Model short name (e.g. claude-sonnet-4-6). Opens picker if omitted.",
required=False,
),
Argument(
name="--save",
type=bool,
description="Save the choice to config file",
required=False,
),
]
async def execute(self, ctx: CommandContext, args: list[str]) -> None:
from ...EvoScientist import _ensure_config
from ...llm.models import list_model_picker_entries
cfg = _ensure_config()
current_model = cfg.model
current_provider = cfg.provider
# Parse --save flag
save = "--save" in args
args = [a for a in args if a != "--save"]
if args:
try:
model_name, provider = extract_model_and_provider(args)
except ValueError:
ctx.ui.append_system(
f"Unknown model '{args[0]}'. Use /model to browse available models.",
style="red",
)
return
await self._apply_model(ctx, model_name, provider, save=save)
return
# Interactive picker
if not ctx.ui.supports_interactive:
ctx.ui.append_system(
"Usage: /model <name> [provider] [--save]",
style="yellow",
)
return
entries = await list_model_picker_entries(
getattr(cfg, "ollama_base_url", None),
include_custom_ollama=True,
)
result = await ctx.ui.wait_for_model_pick(
entries,
current_model=current_model,
current_provider=current_provider,
)
if result is None:
return
name, provider = result
# Defense-in-depth: the widget should have replaced the sentinel with
# the user-typed name. If it didn't, treat as cancel rather than try
# to switch to a literal "__custom_ollama__" model.
if provider == "ollama" and name in (
"Custom Ollama model...",
"__custom_ollama__",
):
return
await self._apply_model(ctx, name, provider, save=save)
async def _apply_model(
self,
ctx: CommandContext,
model_name: str,
provider: str,
*,
save: bool = False,
) -> None:
import copy
from ...cli.agent import _load_agent
from ...EvoScientist import (
_build_chat_model,
_ensure_config,
set_active_config,
set_chat_model_instance,
)
cfg = _ensure_config()
# Build a temporary config + its chat model and verify the agent can be
# built before committing anything. ``create_cli_agent(config=...,
# chat_model=...)`` is pure (issue #183) — it writes none of the cached
# config/model module globals — so a failure below leaves the session
# on the original model with no snapshot/restore needed.
temp_cfg = copy.copy(cfg)
temp_cfg.model = model_name
temp_cfg.provider = provider
try:
new_chat_model = _build_chat_model(temp_cfg)
new_agent = _load_agent(
workspace_dir=ctx.workspace_dir,
checkpointer=ctx.checkpointer,
config=temp_cfg,
chat_model=new_chat_model,
)
except Exception as e:
ctx.ui.append_system(f"Failed to switch model: {e}", style="red")
return
# Agent built with no global mutation — commit the switch atomically.
# These are pure assignments and cannot fail, so the session can never
# be left half-switched. Apply the switch to the LIVE ``cfg`` in place
# (the active config object) instead of rebinding ``_config`` to the
# fresh ``temp_cfg`` — callers that hold the active config by reference
# (e.g. serve's ``agent_holder["config"]`` and its workspace-changing
# ``/resume`` reload) must observe the new model/provider. The verify
# build above used the ``temp_cfg`` copy, so a failed build never reaches
# here and the live ``cfg`` stays untouched (failure still no-ops).
cfg.model = model_name
cfg.provider = provider
set_active_config(cfg)
set_chat_model_instance(new_chat_model, (model_name, provider))
ctx.agent = new_agent
# Persist to config file if --save was given
if save:
from ...config.settings import set_config_value
set_config_value("model", model_name)
set_config_value("provider", provider)
# Propagate to the channel runtime if channels are running so the
# bus picks up the new agent on the next inbound message.
if ctx.channel_runtime is not None and ctx.channel_runtime.agent is not None:
ctx.channel_runtime.agent = new_agent
# Update status bar if available
update_model_fn = getattr(ctx.ui, "update_status_after_model_change", None)
if callable(update_model_fn):
update_model_fn(model_name, provider)
saved_note = " (saved to config)" if save else ""
ctx.ui.append_system(
f"Switched to {model_name} ({provider}){saved_note}", style="green"
)
manager.register(ModelCommand())
-19
View File
@@ -8,16 +8,6 @@ The onboard module is loaded lazily because it pulls in heavy dependencies
(langchain, llm) that are not needed for normal config operations.
"""
from .provider_profiles import (
ProviderModel,
ProviderProfile,
ProviderProfileError,
ProviderProfiles,
get_builtin_provider_profile,
get_provider_profiles_path,
load_provider_profiles,
save_provider_profiles,
)
from .settings import (
EvoScientistConfig,
MemoryControls,
@@ -46,29 +36,20 @@ __all__ = [
"MemoryObservationWriter",
"MemorySkillSynthesisCadence",
"MemorySkillSynthesisMode",
# provider profiles
"ProviderModel",
"ProviderProfile",
"ProviderProfileError",
"ProviderProfiles",
"apply_config_to_env",
"get_builtin_provider_profile",
# settings
"get_config_dir",
"get_config_path",
"get_config_value",
"get_default_workspace_dir",
"get_effective_config",
"get_provider_profiles_path",
"is_config_applied_env",
"list_config",
"load_config",
"load_provider_profiles",
"reset_config",
# onboard (lazy)
"run_onboard",
"save_config",
"save_provider_profiles",
"set_config_value",
]
+132
View File
@@ -0,0 +1,132 @@
"""Startup detection of pre-Registry legacy model configuration artifacts.
Design doc section 10 step 4: this project is in development and keeps no
historical compatibility. When the config service or the CLI finds legacy
artifacts — an old ``providers.yaml``, an old
``run-runtime-snapshots.sqlite3``, or LLM fields left behind in
``config.yaml`` — it must refuse to start and log an explicit reset guide
instead of partially reading them.
"""
from __future__ import annotations
import logging
from pathlib import Path
import yaml
from .settings import get_config_dir
logger = logging.getLogger(__name__)
#: LLM configuration keys that ``config.yaml`` must no longer carry
#: (section 10 step 2). Platform configuration (workspace, MCP, ports,
#: storage, scheduling, security) stays; everything model/provider/credential
#: related lives in the Model Registry (``model-runtime.sqlite3``) only.
LEGACY_CONFIG_YAML_KEYS = frozenset(
{
"provider",
"model",
"model_catalog",
"model_fallbacks",
"auxiliary_provider",
"auxiliary_model",
"anthropic_api_key",
"anthropic_base_url",
"anthropic_auth_mode",
"openai_api_key",
"openai_auth_mode",
"nvidia_api_key",
"google_api_key",
"minimax_api_key",
"minimax_base_url",
"siliconflow_api_key",
"openrouter_api_key",
"deepseek_api_key",
"zhipu_api_key",
"volcengine_api_key",
"dashscope_api_key",
"moonshot_api_key",
"kimi_api_key",
"custom_openai_api_key",
"custom_openai_base_url",
"custom_anthropic_api_key",
"custom_anthropic_base_url",
"ollama_base_url",
"use_responses_api",
"openrouter_anthropic_prompt_cache",
}
)
_LEGACY_PROVIDERS_FILE = "providers.yaml"
_LEGACY_SNAPSHOTS_DB = "run-runtime-snapshots.sqlite3"
_RESET_GUIDANCE = """\
EvoScientist no longer reads legacy model configuration (design doc §10).
To reset the development environment:
1. Delete {config_dir}/providers.yaml (Provider Profiles are superseded
by the Model Registry).
2. Delete {config_dir}/run-runtime-snapshots.sqlite3 (the old snapshot
store; run snapshots now live in model-runtime.sqlite3).
3. Remove the leftover LLM fields listed above from {config_path} —
platform fields (workspace, MCP, ports, scheduling, security) stay.
4. Configure providers/models through the Model Registry (WebUI
configuration page or the model-registry API), run the provider test,
then enable the models you need.
Startup is refused — no legacy artifact is read, even partially.\
"""
class LegacyArtifactsError(RuntimeError):
"""Raised at startup when pre-Registry configuration artifacts remain."""
def find_legacy_artifacts(config_dir: Path | None = None) -> list[str]:
"""Return human-readable descriptions of every legacy artifact found."""
config_dir = config_dir if config_dir is not None else get_config_dir()
found: list[str] = []
providers_yaml = config_dir / _LEGACY_PROVIDERS_FILE
if providers_yaml.exists():
found.append(f"legacy Provider Profiles file: {providers_yaml}")
snapshots_db = config_dir / _LEGACY_SNAPSHOTS_DB
if snapshots_db.exists():
found.append(f"legacy run snapshot database: {snapshots_db}")
config_path = config_dir / "config.yaml"
if config_path.exists():
try:
with open(config_path, encoding="utf-8") as handle:
data = yaml.safe_load(handle) or {}
except yaml.YAMLError:
data = {}
if isinstance(data, dict):
leftover = sorted(LEGACY_CONFIG_YAML_KEYS & data.keys())
if leftover:
found.append(
f"leftover LLM fields in {config_path}: {', '.join(leftover)}"
)
return found
def assert_no_legacy_artifacts(config_dir: Path | None = None) -> None:
"""Refuse startup when any legacy model configuration artifact remains.
Logs the findings plus the explicit reset guide (section 10 step 4) and
raises :class:`LegacyArtifactsError`. Nothing is read partially: the
caller must not catch-and-continue.
"""
found = find_legacy_artifacts(config_dir)
if not found:
return
config_dir = config_dir if config_dir is not None else get_config_dir()
guidance = _RESET_GUIDANCE.format(
config_dir=config_dir,
config_path=config_dir / "config.yaml",
)
message = "Legacy model configuration artifacts detected:\n" + "\n".join(
f" - {item}" for item in found
)
logger.error("%s\n%s", message, guidance)
raise LegacyArtifactsError(f"{message}\n\n{guidance}")
+2 -3
View File
@@ -7,13 +7,12 @@ Everything else lives in submodules — import directly from them:
``STEPS``, ``render_progress``
- :mod:`EvoScientist.config.onboard.steps` — per-step functions
- :mod:`EvoScientist.config.onboard.channels` — channel selection + setup
- :mod:`EvoScientist.config.onboard.helpers` — API-key prompt, ccproxy,
- :mod:`EvoScientist.config.onboard.helpers` — API-key prompt,
npx/node, LaTeX, iMessage helpers
- :mod:`EvoScientist.config.onboard.style` — Rich styles + ``_checkbox_ask``
- :mod:`EvoScientist.config.onboard.validators` — input validators
- :mod:`EvoScientist.config.onboard.prompter` — ``NonInteractivePrompter``
(CLI-answer container) + ``select_navigation_active`` / ``GoBack`` for
keyboard nav
(CLI-answer container) + ``select_navigation_active`` for keyboard nav
- :mod:`EvoScientist.config.onboard.constants` — canonical valid-value sets
This module used to re-export every symbol from every submodule for
+3 -27
View File
@@ -2,46 +2,22 @@
The interactive ``_step_*`` functions in ``steps.py`` use these for ``Choice``
construction (or are checked against them by tests). The CLI ``onboard``
command in ``cli/commands.py`` uses them to validate ``--provider`` /
``--ui`` / ``--workspace-mode`` flag inputs.
command in ``cli/commands.py`` uses them to validate ``--ui`` /
``--workspace-mode`` flag inputs.
Single source of truth — adding a new provider here AND to the corresponding
Single source of truth — adding a new value here AND to the corresponding
``Choice(value=...)`` in ``steps.py`` is required; a drift test in
``tests/test_onboard.py`` keeps both sides in sync.
"""
from __future__ import annotations
VALID_PROVIDERS: frozenset[str] = frozenset(
{
"anthropic",
"openai",
"google-genai",
"minimax",
"zhipu",
"zhipu-code",
"volcengine",
"dashscope",
"dashscope-code",
"deepseek",
"moonshot",
"kimi-coding",
"ollama",
"nvidia",
"siliconflow",
"openrouter",
"custom-openai",
"custom-anthropic",
}
)
VALID_UI_BACKENDS: frozenset[str] = frozenset({"tui", "cli", "webui"})
VALID_WORKSPACE_MODES: frozenset[str] = frozenset({"daemon", "run"})
__all__ = [
"VALID_PROVIDERS",
"VALID_UI_BACKENDS",
"VALID_WORKSPACE_MODES",
]
+1 -198
View File
@@ -1,4 +1,4 @@
"""Cross-step helpers: API key prompt loop, ccproxy login, npx/node bootstrapping,
"""Cross-step helpers: API key prompt loop, npx/node bootstrapping,
LaTeX detection/install, iMessage setup.
"""
@@ -11,126 +11,7 @@ import sys
import questionary
from ..settings import EvoScientistConfig
from .style import QMARK, WIZARD_STYLE, console
from .validators import (
validate_anthropic_key,
validate_dashscope_code_key,
validate_dashscope_key,
validate_deepseek_key,
validate_google_key,
validate_kimi_key,
validate_minimax_key,
validate_moonshot_key,
validate_nvidia_key,
validate_openai_key,
validate_openrouter_key,
validate_siliconflow_key,
validate_volcengine_key,
validate_zhipu_key,
)
def _provider_key_info(config: EvoScientistConfig, provider: str):
"""Return (display_name, current_value, validate_fn) for a provider."""
mapping = {
"anthropic": (
"Anthropic",
config.anthropic_api_key or os.environ.get("ANTHROPIC_API_KEY", ""),
validate_anthropic_key,
),
"minimax": (
"MiniMax",
config.minimax_api_key or os.environ.get("MINIMAX_API_KEY", ""),
lambda key: validate_minimax_key(
key,
base_url=config.minimax_base_url
or os.environ.get(
"MINIMAX_BASE_URL", "https://api.minimaxi.com/anthropic"
),
),
),
"nvidia": (
"NVIDIA",
config.nvidia_api_key or os.environ.get("NVIDIA_API_KEY", ""),
validate_nvidia_key,
),
"google-genai": (
"Google",
config.google_api_key or os.environ.get("GOOGLE_API_KEY", ""),
validate_google_key,
),
"siliconflow": (
"SiliconFlow",
config.siliconflow_api_key or os.environ.get("SILICONFLOW_API_KEY", ""),
validate_siliconflow_key,
),
"openrouter": (
"OpenRouter",
config.openrouter_api_key or os.environ.get("OPENROUTER_API_KEY", ""),
validate_openrouter_key,
),
"deepseek": (
"DeepSeek",
config.deepseek_api_key or os.environ.get("DEEPSEEK_API_KEY", ""),
validate_deepseek_key,
),
"zhipu": (
"ZhipuAI",
config.zhipu_api_key or os.environ.get("ZHIPU_API_KEY", ""),
validate_zhipu_key,
),
"zhipu-code": (
"ZhipuAI CodePlan",
config.zhipu_api_key or os.environ.get("ZHIPU_API_KEY", ""),
validate_zhipu_key,
),
"volcengine": (
"Volcengine",
config.volcengine_api_key or os.environ.get("VOLCENGINE_API_KEY", ""),
validate_volcengine_key,
),
"dashscope": (
"DashScope",
config.dashscope_api_key or os.environ.get("DASHSCOPE_API_KEY", ""),
validate_dashscope_key,
),
"dashscope-code": (
"DashScope Coding Plan",
config.dashscope_api_key or os.environ.get("DASHSCOPE_API_KEY", ""),
validate_dashscope_code_key,
),
"moonshot": (
"Moonshot",
config.moonshot_api_key or os.environ.get("MOONSHOT_API_KEY", ""),
validate_moonshot_key,
),
"kimi-coding": (
"Kimi Coding Plan",
config.kimi_api_key or os.environ.get("KIMI_API_KEY", ""),
validate_kimi_key,
),
"custom-openai": (
"OpenAI-compatible",
config.custom_openai_api_key or os.environ.get("CUSTOM_OPENAI_API_KEY", ""),
None,
),
"custom-anthropic": (
"Custom Anthropic",
config.custom_anthropic_api_key
or os.environ.get("CUSTOM_ANTHROPIC_API_KEY", ""),
None,
),
"ollama": ("Ollama", "__no_key__", None),
}
return mapping.get(
provider,
(
"OpenAI",
config.openai_api_key or os.environ.get("OPENAI_API_KEY", ""),
validate_openai_key,
),
)
def _prompt_and_validate_api_key(
@@ -192,66 +73,6 @@ def _prompt_and_validate_api_key(
return new_key or None
def _prompt_ccproxy_port(config: EvoScientistConfig) -> None:
"""Prompt the user for a ccproxy port and save it to config."""
def valid_port(value: str) -> bool:
if not value: # empty = keep default
return True
try:
return 0 < int(value) < 2**16
except (ValueError, TypeError):
return False
current_port = getattr(config, "ccproxy_port", 8000)
try:
raw = questionary.text(
f"Enter port number for ccproxy to run on (Current: {current_port}, Enter to keep):",
validate=valid_port,
style=WIZARD_STYLE,
qmark=QMARK,
).ask()
if raw is None:
raise KeyboardInterrupt()
raw = raw.strip()
ccproxy_port = int(raw) if raw else current_port
except (ValueError, TypeError):
ccproxy_port = current_port
console.print(f" [dim]Using default port: {ccproxy_port}[/dim]")
config.ccproxy_port = ccproxy_port
console.print(
f" [green]✓ ccproxy will run on http://127.0.0.1:{ccproxy_port}[/green]"
)
def _run_ccproxy_login(provider: str, label: str) -> None:
"""Run ccproxy auth login for the given provider and show status."""
from ...ccproxy_manager import _ccproxy_exe, check_ccproxy_auth
console.print(" [dim]Opening browser for authentication...[/dim]")
try:
proc = subprocess.run(
[_ccproxy_exe() or "ccproxy", "auth", "login", provider],
capture_output=True,
text=True,
timeout=120,
)
for line in proc.stdout.splitlines():
if line.strip().startswith("https://"):
console.print(f" [dim]Visit: {line.strip()}[/dim]")
break
authed, msg = check_ccproxy_auth(provider)
if authed:
console.print(f" [green]✓ {label}: {msg}[/green]")
else:
console.print(f" [red]Authentication failed: {msg}[/red]")
except subprocess.TimeoutExpired:
console.print(" [red]Login timed out.[/red]")
except Exception as exc:
console.print(f" [red]Login error: {exc}[/red]")
def _check_npx() -> bool:
"""Check if npx is available on the system.
@@ -591,24 +412,6 @@ def validate_imessage() -> tuple[bool, str]:
return True, f"imsg{version_str} at {cli_path}"
def _install_ccproxy() -> bool:
"""Run pip install for ccproxy (evoscientist[oauth]).
Uses uv pip install when available (uv-managed envs don't ship pip).
Returns:
True if installation succeeded and ccproxy is available.
"""
from ...ccproxy_manager import is_ccproxy_available
from ...mcp.registry import install_library
ok = install_library("evoscientist[oauth]")
if not ok:
console.print(" [red]✗ Installation failed.[/red]")
return False
return is_ccproxy_available()
def _install_imsg() -> bool:
"""Run brew install for imsg CLI.
+3 -30
View File
@@ -5,31 +5,13 @@ from __future__ import annotations
from typing import Any
class GoBack(Exception):
"""Raised inside the provider sub-loop to rewind to provider selection."""
# Sentinel value the back-keybinding writes into the prompt result, and the
# value the trailing ``← Back`` menu item carries. Same string so the two
# code paths converge to a single ``GoBack`` raise.
BACK_SENTINEL = "__back__"
def install_navigation_keys(
question,
*,
with_back: bool = False,
sentinel: str = BACK_SENTINEL,
) -> None:
def install_navigation_keys(question) -> None:
"""Add keyboard shortcuts on a questionary select ``Question``.
Bindings (merged in front of questionary's defaults — Ctrl+C/Ctrl+D still
cancel the wizard):
- ``→`` — accept the option under the cursor and advance (mirrors Enter).
- ``Esc`` / ``←`` (only when ``with_back=True``) — exit with ``sentinel``
so the wizard can rewind. Used in the provider sub-loop's auth_mode
prompts.
"""
from prompt_toolkit.key_binding import KeyBindings, merge_key_bindings
@@ -49,13 +31,6 @@ def install_navigation_keys(
event.app.exit(result=pointed.value)
return
if with_back:
@kb.add("escape", eager=True)
@kb.add("left", eager=True)
def _back(event):
event.app.exit(result=sentinel)
question.application.key_bindings = merge_key_bindings(
[kb, question.application.key_bindings]
)
@@ -78,7 +53,7 @@ def select_navigation_active():
def _wrapped(*args, **kwargs):
q = original(*args, **kwargs)
try:
install_navigation_keys(q, with_back=False)
install_navigation_keys(q)
except Exception:
# Don't let a stray keybinding error block the wizard.
pass
@@ -92,7 +67,7 @@ def select_navigation_active():
class NonInteractivePrompter:
"""Container for CLI-supplied wizard answers (``--provider``, ``--model``…).
"""Container for CLI-supplied wizard answers (``--ui``, ``--tavily-key``…).
``strict=True`` makes missing presets fatal instead of falling back to
interactive. Wizard reads ``answers`` / ``skip_set`` / ``strict`` directly.
@@ -113,8 +88,6 @@ class NonInteractivePrompter:
__all__ = [
"BACK_SENTINEL",
"GoBack",
"NonInteractivePrompter",
"install_navigation_keys",
"select_navigation_active",
+2 -636
View File
@@ -1,8 +1,8 @@
"""Individual wizard step functions.
Each ``_step_*`` prompts the user for one logical decision and returns the
chosen value. Conditional steps (auth mode, base URL) are only called by
``run_onboard`` when the provider needs them.
chosen value. LLM provider/model/API-key configuration no longer happens in
the wizard — models are configured via the WebUI / model registry.
"""
from __future__ import annotations
@@ -14,24 +14,17 @@ import questionary
from prompt_toolkit.formatted_text import FormattedText
from questionary import Choice
from ...llm import get_models_for_provider
from ...llm.ollama_discovery import validate_ollama_connection
from ..settings import EvoScientistConfig
from .helpers import (
_auto_install_latexmk,
_check_latex_components,
_detect_tinytex_install_method,
_ensure_npx,
_install_ccproxy,
_install_tinytex,
_print_latex_status,
_prompt_and_validate_api_key,
_prompt_ccproxy_port,
_provider_key_info,
_run_ccproxy_login,
)
from .style import (
CONFIRM_STYLE,
QMARK,
WIZARD_STYLE,
_checkbox_ask,
@@ -228,633 +221,6 @@ def _step_webui_port(config: EvoScientistConfig) -> int:
return port
def _step_provider(
config: EvoScientistConfig,
*,
label: str | None = None,
default_value: str | None = None,
) -> str:
"""Step 1: Select LLM provider.
Args:
config: Current configuration.
label: Optional role label (e.g. "co-pilot") to clarify which model this
provider is for. When omitted, the generic main-model prompt is used.
default_value: Preselect this provider instead of ``config.provider``
(e.g. the auxiliary provider when configuring the co-pilot).
Returns:
Selected provider name.
"""
choices = [
# Direct providers
Choice(title="Anthropic (Claude models — API / OAuth)", value="anthropic"),
Choice(title="OpenAI (GPT models — API / OAuth)", value="openai"),
Choice(title="Google GenAI (Gemini models)", value="google-genai"),
Choice(
title="MiniMax (M2 — M3 models, up to 1M context, thinking)",
value="minimax",
),
Choice(title="ZhipuAI (智谱 — GLM models)", value="zhipu"),
Choice(
title="ZhipuAI CodePlan (智谱代码计划 — GLM models for coding)",
value="zhipu-code",
),
Choice(
title="Volcengine (火山引擎 — Doubao models)",
value="volcengine",
),
Choice(
title="DashScope (阿里云 — Qwen models)",
value="dashscope",
),
Choice(
title="DashScope Coding Plan (阿里云代码计划 — Qwen models)",
value="dashscope-code",
),
Choice(
title="DeepSeek (DeepSeek-R1, DeepSeek-V3)",
value="deepseek",
),
Choice(
title="Moonshot (月之暗面 — Moonshot models)",
value="moonshot",
),
Choice(
title="Kimi Coding Plan (Kimi 代码计划 — coding-focused)",
value="kimi-coding",
),
# Local
Choice(title="Ollama (local models)", value="ollama"),
# Third-party / aggregator
Choice(title="NVIDIA (third party — limited free requests)", value="nvidia"),
Choice(
title="SiliconFlow (aggregator — GLM, Kimi, MiniMax, etc.)",
value="siliconflow",
),
Choice(
title="OpenRouter (aggregator — Grok, Gemini, Qwen, etc.)",
value="openrouter",
),
Choice(
title="OpenAI-compatible (third-party OpenAI endpoint)",
value="custom-openai",
),
Choice(
title="Claude-compatible (third-party Anthropic endpoint)",
value="custom-anthropic",
),
]
# Set default based on current config (or an explicit override).
valid_providers = {c.value for c in choices}
preferred = default_value or config.provider
default = preferred if preferred in valid_providers else "anthropic"
provider = questionary.select(
f"Select {label} provider:" if label else "Select your LLM provider:",
choices=choices,
default=default,
style=WIZARD_STYLE,
qmark=QMARK,
use_indicator=True,
).ask()
if provider is None:
raise KeyboardInterrupt()
return provider
_MINIMAX_REGIONS: dict[str, str] = {
"global": "https://api.minimax.io/anthropic",
"cn": "https://api.minimaxi.com/anthropic",
}
def _step_minimax_region(config: EvoScientistConfig) -> str:
"""Step 2a (MiniMax): Select API region.
MiniMax has two regional endpoints — Global (api.minimax.io) and
Mainland China (api.minimaxi.com). API keys are region-bound.
Returns:
The selected base URL.
"""
current = config.minimax_base_url or os.environ.get("MINIMAX_BASE_URL", "")
if current == _MINIMAX_REGIONS["global"]:
default = "global"
else:
default = "cn"
region = questionary.select(
"Select MiniMax API region (must match where your key was created):",
choices=[
Choice(
title="Global (api.minimax.io — platform.minimax.io keys)",
value="global",
),
Choice(
title="Mainland China (api.minimaxi.com — platform.minimaxi.com keys)",
value="cn",
),
],
default=default,
style=WIZARD_STYLE,
qmark=QMARK,
use_indicator=True,
).ask()
if region is None:
raise KeyboardInterrupt()
return _MINIMAX_REGIONS[region]
def _step_oauth_auth_mode(
config: EvoScientistConfig,
*,
provider_label: str,
ccproxy_provider: str,
config_attr: str,
prompt_login_label: str,
oauth_choice_label: str | None = None,
status_label: str | None = None,
question_label: str | None = None,
) -> str:
"""Select API-key vs ccproxy OAuth authentication for a provider.
Args:
config: Current configuration.
provider_label: Provider display name for direct API-key access.
ccproxy_provider: ccproxy auth provider name.
config_attr: Config attribute storing this provider's auth mode.
prompt_login_label: Label used in "Log in to ..." prompts.
oauth_choice_label: Optional display label for the OAuth choice.
status_label: Optional display label for status messages.
question_label: Optional prompt label override.
Returns:
Selected auth mode: "api_key" or "oauth".
"""
from ...ccproxy_manager import check_ccproxy_auth, is_ccproxy_available
ccproxy_available = is_ccproxy_available()
from .prompter import BACK_SENTINEL, GoBack, install_navigation_keys
oauth_label = oauth_choice_label or f"{prompt_login_label} OAuth"
auth_status_label = status_label or oauth_label
auth_question_label = question_label or f"{provider_label} authentication mode"
choices = [
Choice(title=f"API Key (direct {provider_label} access)", value="api_key"),
Choice(
title=f"{oauth_label} (via ccproxy — no API key needed)"
+ (
""
if ccproxy_available
else " [requires: pip install evoscientist[oauth]]"
),
value="oauth",
),
questionary.Separator(),
Choice(title="← Back (re-pick provider)", value=BACK_SENTINEL),
]
current = getattr(config, config_attr)
if current not in ("api_key", "oauth"):
current = "api_key"
question = questionary.select(
f"{auth_question_label} [Esc/← to go back]:",
choices=choices,
default=current,
style=WIZARD_STYLE,
qmark=QMARK,
use_indicator=True,
)
install_navigation_keys(question, with_back=True)
auth_mode = question.ask()
if auth_mode is None:
raise KeyboardInterrupt()
if auth_mode == BACK_SENTINEL:
raise GoBack()
if auth_mode == "oauth" and not ccproxy_available:
console.print(" [yellow]✗ ccproxy not installed[/yellow]")
console.print()
install = questionary.confirm(
'Install ccproxy now? (pip install "evoscientist[oauth]")',
default=True,
style=WIZARD_STYLE,
qmark=f" {QMARK}",
).ask()
if install is None:
raise KeyboardInterrupt()
if install:
console.print()
if _install_ccproxy():
console.print(" [green]✓ ccproxy installed successfully.[/green]")
else:
console.print(" [yellow]Falling back to API key mode.[/yellow]")
return "api_key"
else:
console.print(
' [dim]Skipped. Install manually: pip install "evoscientist[oauth]"[/dim]'
)
return "api_key"
if auth_mode == "oauth":
_prompt_ccproxy_port(config)
authed, msg = check_ccproxy_auth(ccproxy_provider)
if authed:
console.print(f" [green]✓ {auth_status_label}: {msg}[/green]")
relogin = questionary.confirm(
"Re-authenticate to refresh credentials?",
default=False,
style=CONFIRM_STYLE,
qmark=QMARK,
).ask()
if relogin is None:
raise KeyboardInterrupt()
if relogin:
_run_ccproxy_login(ccproxy_provider, auth_status_label)
else:
console.print(
f" [yellow]{auth_status_label} not authenticated: {msg}[/yellow]"
)
login = questionary.confirm(
f"Log in to {prompt_login_label} now?",
default=True,
style=CONFIRM_STYLE,
qmark=QMARK,
).ask()
if login is None:
raise KeyboardInterrupt()
if login:
_run_ccproxy_login(ccproxy_provider, auth_status_label)
return auth_mode
def _step_anthropic_auth_mode(config: EvoScientistConfig) -> str:
"""Step 2a: Select Anthropic authentication mode (API key vs OAuth).
Args:
config: Current configuration.
Returns:
Selected auth mode: "api_key" or "oauth".
"""
return _step_oauth_auth_mode(
config,
provider_label="Anthropic",
ccproxy_provider="claude_api",
config_attr="anthropic_auth_mode",
prompt_login_label="Claude",
oauth_choice_label="Claude Code OAuth",
status_label="OAuth",
question_label="Authentication mode",
)
def _step_openai_auth_mode(config: EvoScientistConfig) -> str:
"""Step 2b: Select OpenAI authentication mode (API key vs Codex OAuth).
Args:
config: Current configuration.
Returns:
Selected auth mode: "api_key" or "oauth".
"""
return _step_oauth_auth_mode(
config,
provider_label="OpenAI",
ccproxy_provider="codex",
config_attr="openai_auth_mode",
prompt_login_label="Codex",
oauth_choice_label="Codex OAuth",
status_label="Codex OAuth",
question_label="OpenAI authentication mode",
)
def _step_provider_api_key(
config: EvoScientistConfig,
provider: str,
skip_validation: bool = False,
) -> str | None:
"""Step 2: Enter API key for the selected provider.
Args:
config: Current configuration.
provider: Selected provider name.
skip_validation: Skip API key validation.
Returns:
New API key or None if unchanged.
"""
key_name, current, validate_fn = _provider_key_info(config, provider)
hint = f"Current: ***{current[-4:]}" if current else "Not set"
prompt_text = f"Enter {key_name} API key ({hint}, Enter to keep):"
return _prompt_and_validate_api_key(
prompt_text,
current,
validate_fn,
skip_validation,
)
def _step_base_url(config: EvoScientistConfig, current_value: str | None = None) -> str:
"""Prompt for custom provider base URL.
Args:
config: Current configuration.
current_value: Current base URL value (if None, defaults to empty).
Returns:
Base URL string.
"""
current = current_value if current_value is not None else ""
hint = f"Current: {current}" if current else ""
default = current or ""
url = questionary.text(
f"Base URL{' (' + hint + ', Enter to keep)' if hint else ''}:",
default=default,
style=WIZARD_STYLE,
qmark=QMARK,
placeholder=FormattedText([("fg:#858585", " e.g. https://api.example.com/v1")])
if not default
else None,
).ask()
if url is None:
raise KeyboardInterrupt()
return url.strip()
def _step_ollama_base_url(config: EvoScientistConfig) -> tuple[str, list[str]]:
"""Prompt for Ollama server base URL and validate connection.
Args:
config: Current configuration.
Returns:
Tuple of (base_url, detected_model_names).
"""
current = config.ollama_base_url or os.environ.get("OLLAMA_BASE_URL", "")
default = current or "http://localhost:11434"
url = questionary.text(
f"Ollama base URL (Enter for {default}):",
default=default,
style=WIZARD_STYLE,
qmark=QMARK,
).ask()
if url is None:
raise KeyboardInterrupt()
url = url.strip()
detected_models: list[str] = []
if url:
console.print(" [dim]Checking Ollama connection...[/dim]", end="")
valid, msg, detected_models = validate_ollama_connection(url)
if valid:
console.print(f"\r [green]\u2713 {msg}[/green] ")
else:
console.print(f"\r [yellow]\u2717 {msg}[/yellow] ")
console.print(" [dim]You can start Ollama later and it will work.[/dim]")
return url, detected_models
def _step_model(
config: EvoScientistConfig,
provider: str,
*,
ollama_detected_models: list[str] | None = None,
label: str | None = None,
default_value: str | None = None,
) -> str:
"""Step 3: Select model for the provider.
Args:
config: Current configuration.
provider: Selected provider name.
ollama_detected_models: Model names detected from a live Ollama server.
label: Optional role label (e.g. "co-pilot") for the prompt. When omitted,
the generic main-model prompt is used.
default_value: Preselect this model instead of ``config.model`` (e.g. the
auxiliary model when configuring the co-pilot).
Returns:
Selected model name.
"""
model_prompt = f"Select {label} model:" if label else "Select model:"
model_default = default_value or config.model
# Ollama: show only what's actually pulled on the server
if provider == "ollama":
if ollama_detected_models:
_CUSTOM_SENTINEL = "__custom__"
choices = [
Choice(title=name, value=name) for name in ollama_detected_models
]
choices.append(Choice(title="Type a model name...", value=_CUSTOM_SENTINEL))
default = ollama_detected_models[0]
if model_default in ollama_detected_models:
default = model_default
selected = questionary.select(
model_prompt,
choices=choices,
default=default,
style=WIZARD_STYLE,
qmark=QMARK,
use_indicator=True,
).ask()
if selected is None:
raise KeyboardInterrupt()
if selected != _CUSTOM_SENTINEL:
return selected
# No detected models (server down or empty) — direct text input
if not ollama_detected_models:
console.print(
" [dim]No models detected — type the model name you plan to pull.[/dim]"
)
model = questionary.text(
"Model name:",
style=WIZARD_STYLE,
qmark=QMARK,
placeholder=FormattedText([("fg:#858585", " e.g. qwen3-coder-next")]),
).ask()
if model is None:
raise KeyboardInterrupt()
model = model.strip()
if not model:
model = "qwen3-coder-next"
console.print(f" [dim]Using default: {model}[/dim]")
return model
# Get models for the selected provider
entries = get_models_for_provider(provider)
if not entries:
# Custom / unknown provider: direct text input.
# Keep prompting until a non-empty model name is provided — saving an
# empty string here leaves the first request broken with an opaque
# "model required" error from the provider SDK.
while True:
model = questionary.text(
"Model name:",
style=WIZARD_STYLE,
qmark=QMARK,
placeholder=FormattedText([("fg:#858585", " e.g. owner/model-name")]),
default=model_default or "",
).ask()
if model is None:
raise KeyboardInterrupt()
model = model.strip()
if model:
return model
console.print(
" [yellow]Model name cannot be empty for a custom provider. "
"Press Ctrl+C to cancel.[/yellow]"
)
provider_models = [name for name, _ in entries]
# Create choices with model IDs as hints
_CUSTOM_SENTINEL = "__custom__"
choices = []
for name, model_id in entries:
choices.append(Choice(title=f"{name} ({model_id})", value=name))
choices.append(Choice(title="Type a model name...", value=_CUSTOM_SENTINEL))
# Determine default. An explicit ``default_value`` override (e.g. a saved
# co-pilot model on a re-run) that isn't a registry model is a custom name:
# preselect "Type a model name..." and prefill it. A plain ``config.model``
# that just isn't in the current provider's list (e.g. the provider was
# changed) falls back to the first model, NOT the custom entry.
custom_default = (
default_value if default_value and default_value not in provider_models else ""
)
if model_default in provider_models:
default = model_default
elif custom_default:
default = _CUSTOM_SENTINEL
else:
default = provider_models[0]
selected = questionary.select(
model_prompt,
choices=choices,
default=default,
style=WIZARD_STYLE,
qmark=QMARK,
use_indicator=True,
).ask()
if selected is None:
raise KeyboardInterrupt()
if selected != _CUSTOM_SENTINEL:
return selected
model = questionary.text(
"Model name:",
default=custom_default,
style=WIZARD_STYLE,
qmark=QMARK,
placeholder=FormattedText([("fg:#858585", " e.g. owner/model-name")]),
).ask()
if model is None:
raise KeyboardInterrupt()
model = model.strip()
if not model:
model = provider_models[0]
console.print(f" [dim]Using default: {model}[/dim]")
return model
def _step_auxiliary_enable(config: EvoScientistConfig) -> bool:
"""Step 3.25: Choose whether to assemble a co-pilot (auxiliary) model.
The co-pilot runs background/helper LLM calls — EvoMemory (memory workers)
and the main agent's tool selector — so it can be a cheaper/faster model.
Returns True when the user picks "Assemble"; the caller then runs the
provider/key/model pickers. Returns False to keep the pilot (main model)
everywhere.
"""
console.print(
" [dim]A cheaper/faster co-pilot for EvoMemory (memory workers).[/dim]"
)
choice = questionary.select(
"Co-pilot (auxiliary model):",
choices=[
Choice(
title="Skip — single pilot (main model handles everything)",
value="skip",
),
Choice(
title="Assemble a co-pilot — separate cheaper/faster model",
value="assemble",
),
],
default="assemble" if config.auxiliary_model else "skip",
style=WIZARD_STYLE,
qmark=QMARK,
use_indicator=True,
).ask()
if choice is None:
raise KeyboardInterrupt()
return choice == "assemble"
def _step_reasoning_effort(config: EvoScientistConfig) -> str:
"""Step 3.5: Configure OpenRouter reasoning effort level.
Only shown when the selected provider is OpenRouter. See:
https://openrouter.ai/docs/guides/best-practices/reasoning-tokens
Args:
config: Current configuration.
Returns:
Selected reasoning effort level, or empty string to use default.
"""
effort_choices = [
Choice(title="xhigh — ~95% of max_tokens for reasoning", value="xhigh"),
Choice(title="high — ~80% of max_tokens (recommended)", value="high"),
Choice(title="medium — ~50% of max_tokens", value="medium"),
Choice(title="low — ~20% of max_tokens", value="low"),
Choice(title="minimal — ~10% of max_tokens", value="minimal"),
Choice(title="none — disable reasoning entirely", value="none"),
]
current = config.reasoning_effort or "high"
effort = questionary.select(
"Select reasoning effort level:",
choices=effort_choices,
default=current,
style=WIZARD_STYLE,
qmark=QMARK,
use_indicator=True,
).ask()
if effort is None:
raise KeyboardInterrupt()
return effort
def _step_tavily_key(
config: EvoScientistConfig,
skip_validation: bool = False,
+1 -415
View File
@@ -1,7 +1,7 @@
"""Input validators for the onboarding wizard.
- IntegerValidator / ChoiceValidator: prompt_toolkit Validators
- validate_*_key: per-provider API key validators (live HTTP probes)
- validate_tavily_key: Tavily API key validator (live HTTP probe)
"""
from __future__ import annotations
@@ -97,415 +97,6 @@ def _classify_validation_error(error: BaseException) -> tuple[bool, str] | None:
return None
def validate_anthropic_key(api_key: str) -> tuple[bool, str]:
"""Validate an Anthropic API key by making a test request.
Args:
api_key: The API key to validate.
Returns:
Tuple of (is_valid, message).
"""
if not api_key:
return True, "Skipped (no key provided)"
try:
import anthropic
client = anthropic.Anthropic(api_key=api_key)
# Make a minimal request to validate the key
client.models.list()
return True, "Valid"
except anthropic.AuthenticationError:
return False, "Invalid API key"
except Exception as e:
classified = _classify_validation_error(e)
if classified is not None:
return classified
return False, f"Error: {e}"
def validate_openai_key(api_key: str) -> tuple[bool, str]:
"""Validate an OpenAI API key by making a test request.
Args:
api_key: The API key to validate.
Returns:
Tuple of (is_valid, message).
"""
if not api_key:
return True, "Skipped (no key provided)"
try:
import openai
client = openai.OpenAI(api_key=api_key)
# Make a minimal request to validate the key
client.models.list()
return True, "Valid"
except openai.AuthenticationError:
return False, "Invalid API key"
except Exception as e:
classified = _classify_validation_error(e)
if classified is not None:
return classified
return False, f"Error: {e}"
def validate_nvidia_key(api_key: str) -> tuple[bool, str]:
"""Validate an NVIDIA API key by making a test request.
Args:
api_key: The API key to validate.
Returns:
Tuple of (is_valid, message).
"""
if not api_key:
return True, "Skipped (no key provided)"
# NOTE: ``ChatNVIDIA(api_key=...)`` does NOT send a network request — it
# only stores the key in a client object. We must actually invoke the
# API (e.g. ``get_available_models()``) to verify the key is good.
try:
from langchain_nvidia_ai_endpoints import ChatNVIDIA
client = ChatNVIDIA(api_key=api_key, model="meta/llama-3.1-8b-instruct")
# Force a real authenticated request via model discovery.
client.get_available_models()
return True, "Valid"
except Exception as e:
classified = _classify_validation_error(e)
if classified is not None:
return classified
return False, f"Error: {e}"
def validate_google_key(api_key: str) -> tuple[bool, str]:
"""Validate a Google GenAI API key by making a test request.
Args:
api_key: The API key to validate.
Returns:
Tuple of (is_valid, message).
"""
if not api_key:
return True, "Skipped (no key provided)"
try:
from google import genai
client = genai.Client(api_key=api_key)
# Make a minimal request to validate the key
pager = client.models.list(config={"page_size": 1})
next(iter(pager)) # fetch first model only
return True, "Valid"
except StopIteration:
# Empty result but request succeeded — key is valid
return True, "Valid"
except Exception as e:
classified = _classify_validation_error(e)
if classified is not None:
return classified
# Google-specific 400 phrasing not in the shared hint list.
error_str = str(e).lower()
if "api_key_invalid" in error_str or "api key invalid" in error_str:
return False, "Invalid API key"
return False, f"Error: {e}"
def validate_minimax_key(
api_key: str,
base_url: str = "https://api.minimaxi.com/anthropic",
) -> tuple[bool, str]:
"""Validate a MiniMax API key without consuming tokens.
Sends a messages.create() with an empty model string. MiniMax checks
auth *before* validating request params, so a valid key returns 400
(bad model) while an invalid key returns 401.
Args:
api_key: The MiniMax API key to validate.
base_url: Anthropic-compatible endpoint (global or mainland China).
Returns:
Tuple of (is_valid, message).
"""
if not api_key:
return True, "Skipped (no key provided)"
try:
import anthropic
client = anthropic.Anthropic(
api_key=api_key,
base_url=base_url,
)
client.messages.create(
model="",
max_tokens=1,
messages=[{"role": "user", "content": "hi"}],
)
# Unexpected success — treat as valid
return True, "Valid"
except anthropic.AuthenticationError:
return False, "Invalid API key"
except anthropic.APIStatusError:
# Any non-auth HTTP error (400 bad model, 500 insufficient balance,
# etc.) means the key itself was accepted → treat as valid.
return True, "Valid"
except Exception as e:
classified = _classify_validation_error(e)
if classified is not None:
return classified
return False, f"Error: {e}"
def validate_siliconflow_key(api_key: str) -> tuple[bool, str]:
"""Validate a SiliconFlow API key by making a test request.
Returns:
Tuple of (is_valid, message).
"""
if not api_key:
return True, "Skipped (no key provided)"
try:
import openai
client = openai.OpenAI(
api_key=api_key, base_url="https://api.siliconflow.cn/v1"
)
client.models.list()
return True, "Valid"
except Exception as e:
classified = _classify_validation_error(e)
if classified is not None:
return classified
return False, f"Error: {e}"
def validate_openrouter_key(api_key: str) -> tuple[bool, str]:
"""Validate an OpenRouter API key via the authenticated /auth/key endpoint.
Returns:
Tuple of (is_valid, message).
"""
if not api_key:
return True, "Skipped (no key provided)"
try:
import httpx
resp = httpx.get(
"https://openrouter.ai/api/v1/auth/key",
headers={"Authorization": f"Bearer {api_key}"},
timeout=10,
)
if resp.status_code == 200:
return True, "Valid"
# Only 401/403 mean the key is actually rejected. 429 (rate-limit)
# and 5xx (OpenRouter incident) leave the key validity unknown —
# surface the real status so the user doesn't go re-roll a good key
# during an outage.
if resp.status_code in (401, 403):
return False, "Invalid API key"
return False, f"Validation inconclusive (HTTP {resp.status_code})"
except Exception as e:
classified = _classify_validation_error(e)
if classified is not None:
return classified
return False, f"Error: {e}"
def validate_deepseek_key(api_key: str) -> tuple[bool, str]:
"""Validate a DeepSeek API key by making a test request.
Returns:
Tuple of (is_valid, message).
"""
if not api_key:
return True, "Skipped (no key provided)"
try:
import openai
client = openai.OpenAI(api_key=api_key, base_url="https://api.deepseek.com")
client.models.list()
return True, "Valid"
except Exception as e:
classified = _classify_validation_error(e)
if classified is not None:
return classified
return False, f"Error: {e}"
def validate_zhipu_key(api_key: str) -> tuple[bool, str]:
"""Validate a ZhipuAI API key by making a test request.
Uses the general endpoint for validation — both zhipu and zhipu-code
share the same API key, only the base_url differs at runtime.
Returns:
Tuple of (is_valid, message).
"""
if not api_key:
return True, "Skipped (no key provided)"
try:
import openai
client = openai.OpenAI(
api_key=api_key, base_url="https://open.bigmodel.cn/api/paas/v4"
)
client.models.list()
return True, "Valid"
except Exception as e:
classified = _classify_validation_error(e)
if classified is not None:
return classified
return False, f"Error: {e}"
def validate_volcengine_key(api_key: str) -> tuple[bool, str]:
"""Validate a Volcengine API key by making a test request.
Returns:
Tuple of (is_valid, message).
"""
if not api_key:
return True, "Skipped (no key provided)"
try:
import openai
client = openai.OpenAI(
api_key=api_key,
base_url="https://ark.cn-beijing.volces.com/api/v3",
)
client.models.list()
return True, "Valid"
except Exception as e:
classified = _classify_validation_error(e)
if classified is not None:
return classified
return False, f"Error: {e}"
def validate_dashscope_key(api_key: str) -> tuple[bool, str]:
"""Validate a DashScope API key by making a test request.
Returns:
Tuple of (is_valid, message).
"""
if not api_key:
return True, "Skipped (no key provided)"
try:
import openai
client = openai.OpenAI(
api_key=api_key,
base_url="https://dashscope.aliyuncs.com/compatible-mode/v1",
)
client.models.list()
return True, "Valid"
except Exception as e:
classified = _classify_validation_error(e)
if classified is not None:
return classified
return False, f"Error: {e}"
def validate_dashscope_code_key(api_key: str) -> tuple[bool, str]:
"""Validate a DashScope Coding Plan API key (sk-sp-* subscription keys).
The coding endpoint at coding.dashscope.aliyuncs.com does not expose
/models (returns 404), so validation issues a minimal chat completion
instead of the usual models.list() probe.
Returns:
Tuple of (is_valid, message).
"""
if not api_key:
return True, "Skipped (no key provided)"
try:
import openai
client = openai.OpenAI(
api_key=api_key,
base_url="https://coding.dashscope.aliyuncs.com/v1",
)
client.chat.completions.create(
model="qwen3-coder-plus",
messages=[{"role": "user", "content": "hi"}],
max_tokens=1,
)
return True, "Valid"
except Exception as e:
classified = _classify_validation_error(e)
if classified is not None:
return classified
return False, f"Error: {e}"
def validate_moonshot_key(api_key: str) -> tuple[bool, str]:
"""Validate a Moonshot API key by making a test request.
Returns:
Tuple of (is_valid, message).
"""
if not api_key:
return True, "Skipped (no key provided)"
try:
import openai
client = openai.OpenAI(
api_key=api_key,
base_url="https://api.moonshot.cn/v1",
)
client.models.list()
return True, "Valid"
except Exception as e:
classified = _classify_validation_error(e)
if classified is not None:
return classified
return False, f"Error: {e}"
def validate_kimi_key(api_key: str) -> tuple[bool, str]:
"""Validate a Kimi Coding Plan API key by making a test request.
Uses the Anthropic-compatible endpoint at api.kimi.com/coding/.
Returns:
Tuple of (is_valid, message).
"""
if not api_key:
return True, "Skipped (no key provided)"
try:
import anthropic
client = anthropic.Anthropic(
api_key=api_key,
base_url="https://api.kimi.com/coding/",
default_headers={"User-Agent": "claude-code/0.1.0"},
)
client.models.list()
return True, "Valid"
except Exception as e:
classified = _classify_validation_error(e)
if classified is not None:
return classified
return False, f"Error: {e}"
def validate_tavily_key(api_key: str) -> tuple[bool, str]:
"""Validate a Tavily API key by making a test request.
@@ -530,8 +121,3 @@ def validate_tavily_key(api_key: str) -> tuple[bool, str]:
if classified is not None:
return classified
return False, f"Error: {e}"
# =============================================================================
# Display Helpers
# =============================================================================
+4 -401
View File
@@ -3,7 +3,6 @@
from __future__ import annotations
import copy
import os
import questionary
from rich.panel import Panel
@@ -17,18 +16,8 @@ from ..settings import (
)
from .channels import _step_channels
from .steps import (
_step_anthropic_auth_mode,
_step_auxiliary_enable,
_step_base_url,
_step_langgraph_dev_port,
_step_mcp_servers,
_step_minimax_region,
_step_model,
_step_ollama_base_url,
_step_openai_auth_mode,
_step_provider,
_step_provider_api_key,
_step_reasoning_effort,
_step_skills,
_step_tavily_key,
_step_thinking,
@@ -41,7 +30,6 @@ from .style import (
CONFIRM_STYLE,
QMARK,
_print_header,
_print_section,
_print_step_skipped,
console,
)
@@ -49,10 +37,6 @@ from .style import (
STEPS = [
"UI",
"LangGraph Port",
"Provider",
"API Key",
"Model",
"Auxiliary Model",
"Tavily Key",
"Workspace",
"Thinking",
@@ -110,32 +94,6 @@ def render_progress(current_step: int, completed: set[int]) -> Panel:
# =============================================================================
_PROVIDER_KEY_ATTR = {
"anthropic": "anthropic_api_key",
"minimax": "minimax_api_key",
"nvidia": "nvidia_api_key",
"google-genai": "google_api_key",
"siliconflow": "siliconflow_api_key",
"openrouter": "openrouter_api_key",
"deepseek": "deepseek_api_key",
"zhipu": "zhipu_api_key",
"zhipu-code": "zhipu_api_key",
"volcengine": "volcengine_api_key",
"dashscope": "dashscope_api_key",
"dashscope-code": "dashscope_api_key",
"moonshot": "moonshot_api_key",
"kimi-coding": "kimi_api_key",
"custom-openai": "custom_openai_api_key",
"custom-anthropic": "custom_anthropic_api_key",
}
_MINIMAX_GLOBAL_BASE_URL = "https://api.minimax.io/anthropic"
_CUSTOM_PROVIDER_BASE_URL = {
"custom-openai": ("custom_openai_base_url", "CUSTOM_OPENAI_BASE_URL"),
"custom-anthropic": ("custom_anthropic_base_url", "CUSTOM_ANTHROPIC_BASE_URL"),
}
def _autosave(config: EvoScientistConfig) -> None:
"""Persist current config to disk between phases.
@@ -148,208 +106,10 @@ def _autosave(config: EvoScientistConfig) -> None:
pass
def _configure_provider_base_url(
config: EvoScientistConfig,
provider: str,
*,
strict: bool,
) -> list[str]:
"""Configure provider-specific base URL/region and return Ollama models."""
if provider in _CUSTOM_PROVIDER_BASE_URL:
attr_name, env_name = _CUSTOM_PROVIDER_BASE_URL[provider]
current_base_url = getattr(config, attr_name) or os.environ.get(env_name, "")
if strict:
if not current_base_url:
raise RuntimeError(
f"--non-interactive: {provider} provider needs a base URL. "
f"Set the {env_name} env var or run without --non-interactive."
)
setattr(config, attr_name, current_base_url)
else:
setattr(
config,
attr_name,
_step_base_url(config, current_value=current_base_url),
)
elif provider == "minimax":
if strict:
config.minimax_base_url = (
config.minimax_base_url or _MINIMAX_GLOBAL_BASE_URL
)
else:
config.minimax_base_url = _step_minimax_region(config)
elif provider == "ollama":
if strict:
config.ollama_base_url = (
config.ollama_base_url
or os.environ.get("OLLAMA_BASE_URL", "")
or "http://localhost:11434"
)
else:
ollama_url, ollama_detected_models = _step_ollama_base_url(config)
config.ollama_base_url = ollama_url
return ollama_detected_models
return []
def _configure_provider_auth_mode(
config: EvoScientistConfig,
provider: str,
*,
strict: bool,
) -> None:
"""Configure Anthropic/OpenAI auth mode for the selected provider."""
if provider == "anthropic":
if strict:
config.anthropic_auth_mode = "api_key"
else:
config.anthropic_auth_mode = _step_anthropic_auth_mode(config)
elif provider == "openai":
if strict:
config.openai_auth_mode = "api_key"
else:
config.openai_auth_mode = _step_openai_auth_mode(config)
def _active_llm_providers(config: EvoScientistConfig) -> set[str]:
"""Return providers currently selected by the main and auxiliary models."""
providers = {config.provider}
if config.auxiliary_provider:
providers.add(config.auxiliary_provider)
return providers
def _reconcile_oauth_modes(config: EvoScientistConfig) -> None:
"""Clear OAuth flags for providers no selected model uses."""
active_providers = _active_llm_providers(config)
if "anthropic" not in active_providers:
config.anthropic_auth_mode = "api_key"
if "openai" not in active_providers:
config.openai_auth_mode = "api_key"
def _provider_uses_oauth(config: EvoScientistConfig, provider: str) -> bool:
return (provider == "anthropic" and config.anthropic_auth_mode == "oauth") or (
provider == "openai" and config.openai_auth_mode == "oauth"
)
def _apply_preset_provider_api_key(
config: EvoScientistConfig,
provider: str,
preset_api_key: str,
*,
skip_validation: bool,
) -> None:
"""Validate and store a CLI-supplied provider API key."""
if not skip_validation:
from .helpers import _provider_key_info
_info = _provider_key_info(config, provider)
validate_fn = _info[2] if _info else None
if validate_fn is not None:
console.print(" [dim]Validating preset API key...[/dim]", end="")
valid, msg = validate_fn(preset_api_key)
if valid:
console.print(f"\r [green]✓ {msg}[/green] ")
else:
console.print(f"\r [red]✗ {msg}[/red] ")
raise RuntimeError(
f"--api-key rejected by {provider} validator: {msg}. "
"Pass --skip-validation to override."
)
key_attr = _PROVIDER_KEY_ATTR.get(provider, "openai_api_key")
setattr(config, key_attr, preset_api_key)
console.print(
f" [green]✓ API key: ***{preset_api_key[-4:]}[/green] [dim](--api-key)[/dim]"
)
def _configure_provider_api_key(
config: EvoScientistConfig,
provider: str,
*,
skip_validation: bool,
preset_api_key: str | None = None,
require_api_key=None,
) -> None:
"""Configure provider API key unless the provider does not need one."""
if provider == "ollama" or _provider_uses_oauth(config, provider):
return
key_attr = _PROVIDER_KEY_ATTR.get(provider, "openai_api_key")
if preset_api_key is not None:
_apply_preset_provider_api_key(
config,
provider,
preset_api_key,
skip_validation=skip_validation,
)
return
if require_api_key is not None:
require_api_key()
new_key = _step_provider_api_key(config, provider, skip_validation)
if new_key is not None:
setattr(config, key_attr, new_key)
elif not getattr(config, key_attr):
_print_step_skipped("API Key", "not set")
def _provider_connection_configured(config: EvoScientistConfig, provider: str) -> bool:
"""Return True when provider-level setup can be safely reused."""
if provider == "ollama":
return bool(config.ollama_base_url)
if provider == "custom-openai" and not config.custom_openai_base_url:
return False
if provider == "custom-anthropic" and not config.custom_anthropic_base_url:
return False
if provider == "minimax" and not config.minimax_base_url:
return False
if _provider_uses_oauth(config, provider):
return True
key_attr = _PROVIDER_KEY_ATTR.get(provider, "openai_api_key")
return bool(getattr(config, key_attr))
def _configure_provider_connection(
config: EvoScientistConfig,
provider: str,
*,
strict: bool,
skip_validation: bool,
preset_api_key: str | None = None,
require_api_key=None,
) -> list[str]:
"""Configure provider base URL/region, auth mode, and API key."""
ollama_detected_models = _configure_provider_base_url(
config,
provider,
strict=strict,
)
_configure_provider_auth_mode(
config,
provider,
strict=strict,
)
_configure_provider_api_key(
config,
provider,
skip_validation=skip_validation,
preset_api_key=preset_api_key,
require_api_key=require_api_key,
)
return ollama_detected_models
# Sections offered in Keep/Modify/Reset → which step labels they enable.
_SECTION_LABELS: list[tuple[str, str]] = [
("ui", "UI backend"),
("port", "LangGraph server port"),
("provider", "LLM provider + auth + API key"),
("model", "Model + reasoning effort"),
("auxiliary_model", "Auxiliary model (optional)"),
("tavily", "Tavily search key"),
("workspace", "Workspace mode"),
("thinking", "Thinking panel"),
@@ -360,17 +120,10 @@ _SECTION_LABELS: list[tuple[str, str]] = [
]
_ALL_SECTIONS: frozenset[str] = frozenset(s for s, _ in _SECTION_LABELS)
# Each preset flag implies the section(s) it would change. ``--provider`` also
# cascades into ``model`` because the model list depends on the provider —
# silently keeping a stale model id would leave the first request broken.
# Each preset flag implies the section(s) it would change.
_FLAG_TO_SECTIONS: dict[str, frozenset[str]] = {
"ui": frozenset({"ui"}),
"port": frozenset({"port"}),
"provider": frozenset({"provider", "model"}),
# ``--api-key`` re-runs the provider section, which can change provider —
# cascade to model for the same reason ``--provider`` does.
"api_key": frozenset({"provider", "model"}),
"model": frozenset({"model"}),
"tavily_key": frozenset({"tavily"}),
"workspace_mode": frozenset({"workspace"}),
"show_thinking": frozenset({"thinking"}),
@@ -481,7 +234,7 @@ def run_onboard(
Args:
skip_validation: Skip API key validation.
prompter: Optional :class:`NonInteractivePrompter` carrying
CLI-supplied answers (``--provider``, ``--model``, …) and
CLI-supplied answers (``--ui``, ``--tavily-key``, …) and
``skip_set`` (sections to bypass). When None, all prompts
fall through to the interactive questionary form.
only_sections: If given, restrict the wizard to exactly these section
@@ -556,9 +309,9 @@ def run_onboard(
# Decide which sections this run should cover.
#
# - ``only_sections`` (programmatic, e.g. ``configure provider``):
# - ``only_sections`` (programmatic, e.g. ``configure mcp``):
# run exactly those sections, no Keep/Modify/Reset prompt.
# - Any preset flag (``--provider``/``--model``/…): treat as
# - Any preset flag (``--ui``/``--tavily-key``/…): treat as
# explicit user intent — skip Keep/Modify/Reset and run ONLY
# the sections each flag implies (see ``_FLAG_TO_SECTIONS``).
# - Strict ``--non-interactive`` with no preset flags: run all
@@ -654,156 +407,6 @@ def run_onboard(
config.langgraph_dev_port = _step_langgraph_dev_port(config)
_autosave(config)
ollama_detected_models: list[str] = []
if "provider" in sections_to_run:
from .prompter import GoBack
_print_section("EvoScientist · Pilot (Main model)")
_require("provider", "LLM provider")
# Provider sub-loop: auth_mode can raise GoBack to re-pick provider.
# We snapshot config at the top of each iteration so a GoBack can
# roll back partial writes (base_url, minimax region, ollama URL,
# provider id itself) — otherwise picking `custom-openai`, entering
# a base URL, going Back, then picking `anthropic` would leave a
# stale ``custom_openai_base_url`` in the final saved config.
while True:
loop_snapshot = copy.deepcopy(config)
preset_provider = _preset("provider")
if preset_provider is not None:
provider = preset_provider
config.provider = provider
console.print(
f" [green]✓ Provider: {provider}[/green] "
"[dim](--provider)[/dim]"
)
else:
provider = _step_provider(config)
config.provider = provider
try:
ollama_detected_models = _configure_provider_connection(
config,
provider,
strict=strict,
skip_validation=skip_validation,
preset_api_key=_preset("api_key"),
require_api_key=lambda provider=provider: _require(
"api_key", f"{provider} API key"
),
)
except GoBack:
# User picked "← Back" — restore config to its state at the
# top of this iteration (drops any base_url / region /
# provider writes), then discard ALL provider-coupled
# presets and re-prompt. Clearing only ``provider``
# leaves a stale ``--model`` / ``--api-key`` that would
# be re-applied under a different provider, producing
# an invalid pair (e.g. ``provider=openai`` +
# ``model=claude-sonnet-4-6``).
for field_name in vars(loop_snapshot):
setattr(
config, field_name, getattr(loop_snapshot, field_name)
)
if p:
for stale_key in ("provider", "model", "api_key"):
p.answers.pop(stale_key, None)
ollama_detected_models = []
console.print(" [dim]↩ Returning to provider selection.[/dim]")
continue
break # Provider setup succeeded — exit sub-loop
_reconcile_oauth_modes(config)
_autosave(config)
else:
# Provider section skipped — keep prior provider value to drive
# downstream sections that depend on it (e.g., model picker).
provider = config.provider
if "model" in sections_to_run:
_require("model", "Model")
preset_model = _preset("model")
if preset_model is not None:
config.model = preset_model
console.print(
f" [green]✓ Model: {preset_model}[/green] [dim](--model)[/dim]"
)
else:
config.model = _step_model(
config, provider, ollama_detected_models=ollama_detected_models
)
if provider == "openrouter" and _preset("model") is None:
config.reasoning_effort = _step_reasoning_effort(config)
_autosave(config)
if "auxiliary_model" in sections_to_run:
_print_section("Co-pilot (Auxiliary model)")
if strict:
# Optional; never prompt under --non-interactive. Keep
# current (default empty = use main model).
_print_step_skipped(
"Auxiliary Model",
"kept current" if config.auxiliary_model else "not set",
)
elif _step_auxiliary_enable(config):
from .prompter import GoBack
aux_ollama_detected_models: list[str] = []
while True:
loop_snapshot = copy.deepcopy(config)
aux_provider = _step_provider(
config,
label="co-pilot",
default_value=config.auxiliary_provider,
)
config.auxiliary_provider = aux_provider
if (
aux_provider == config.provider
and _provider_connection_configured(config, aux_provider)
):
if aux_provider == "ollama":
aux_ollama_detected_models = ollama_detected_models
_print_step_skipped(
"Co-pilot credentials",
"reusing main provider settings",
)
else:
try:
aux_ollama_detected_models = (
_configure_provider_connection(
config,
aux_provider,
strict=False,
skip_validation=skip_validation,
)
)
except GoBack:
for field_name in vars(loop_snapshot):
setattr(
config,
field_name,
getattr(loop_snapshot, field_name),
)
aux_ollama_detected_models = []
console.print(
" [dim]↩ Returning to co-pilot provider "
"selection.[/dim]"
)
continue
break
config.auxiliary_model = _step_model(
config,
aux_provider,
ollama_detected_models=aux_ollama_detected_models,
label="co-pilot",
default_value=config.auxiliary_model,
)
else:
# Skip: single driver — clear any prior auxiliary config.
config.auxiliary_provider = ""
config.auxiliary_model = ""
_reconcile_oauth_modes(config)
_autosave(config)
if "tavily" in sections_to_run:
preset_tavily = _preset("tavily_key")
if preset_tavily is not None:
-70
View File
@@ -1,70 +0,0 @@
"""Shared local authentication token for WebUI model administration."""
from __future__ import annotations
import os
import secrets
import threading
from pathlib import Path
from .settings import get_config_dir
_TOKEN_FILE_NAME = "provider-admin-token"
_TOKEN_LOCK = threading.RLock()
_cached_file_token: tuple[Path, str] | None = None
def get_provider_admin_token_path() -> Path:
"""Return the user-scoped provider administration token path."""
return get_config_dir() / _TOKEN_FILE_NAME
def _read_token(path: Path) -> str:
try:
return path.read_text(encoding="utf-8").strip()
except OSError:
return ""
def get_provider_admin_token() -> str:
"""Return an explicit token or create a shared local token when possible.
Separate local backend and WebUI processes share the user-scoped token file.
Remote/container deployments can set ``EVOSCIENTIST_PROVIDER_ADMIN_TOKEN``
explicitly when the processes do not share a filesystem.
"""
explicit = os.environ.get("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "").strip()
if explicit:
return explicit
path = get_provider_admin_token_path()
global _cached_file_token
with _TOKEN_LOCK:
if _cached_file_token is not None and _cached_file_token[0] == path:
return _cached_file_token[1]
token = _read_token(path)
if not token:
token = secrets.token_urlsafe(32)
try:
path.parent.mkdir(parents=True, exist_ok=True)
try:
path.parent.chmod(0o700)
except OSError:
pass
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as file:
file.write(f"{token}\n")
except FileExistsError:
token = _read_token(path)
except OSError:
return ""
if not token:
return ""
try:
path.chmod(0o600)
except OSError:
pass
_cached_file_token = (path, token)
return token
-831
View File
@@ -1,831 +0,0 @@
"""Persistent, runtime-editable LLM provider profiles.
The registry stores managed built-in connections and custom provider profiles.
``config.yaml`` remains backward compatible with the onboarding flow and acts
as a legacy fallback plus the default provider/model pointer.
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import tempfile
import threading
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit, urlunsplit
import yaml
from .settings import get_config_dir
PROVIDER_PROFILES_VERSION = 3
SUPPORTED_PROVIDER_ADAPTERS = (
"openai",
"anthropic",
"google-genai",
"grok",
"openrouter",
"nvidia",
"antigravity",
"openai-compatible",
"anthropic-compatible",
"ollama",
)
_ID_PATTERN = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$")
_BASE_URL_REQUIRED_ADAPTERS = {
"openai-compatible",
"anthropic-compatible",
"antigravity",
"ollama",
}
_DEFAULT_API_PATHS = {
"openai": "/v1",
"openai-compatible": "/v1",
"grok": "/v1",
"openrouter": "/api/v1",
"nvidia": "/v1",
"antigravity": "/v1",
"google-genai": "/v1beta",
}
_RESERVED_PROVIDER_IDS = {
"anthropic",
"openai",
"google-genai",
"minimax",
"nvidia",
"siliconflow",
"openrouter",
"zhipu",
"zhipu-code",
"volcengine",
"dashscope",
"dashscope-code",
"deepseek",
"moonshot",
"kimi-coding",
"custom-openai",
"custom-anthropic",
"ollama",
}
_STORE_LOCK = threading.RLock()
class ProviderProfileError(ValueError):
"""Raised when a provider profile document or selection is invalid."""
@dataclass(frozen=True)
class ProviderRuntime:
timeout_seconds: int = 120
max_retries: int = 2
default_temperature: float | None = None
default_top_p: float | None = None
default_reasoning_effort: str = "auto"
@dataclass(frozen=True)
class ModelRuntime:
limit_mode: str | None = None
context_window_tokens: int | None = None
max_input_tokens: int | None = None
max_output_tokens: int = 4096
min_effective_input_tokens: int = 4096
limits_status: str = "needs_confirmation"
limits_source: str = "user"
temperature: float | None = None
top_p: float | None = None
reasoning_effort: str = "auto"
capabilities: tuple[tuple[str, bool | str], ...] = (
("tools", "auto"),
("vision", "auto"),
("structured_output", "auto"),
)
@dataclass(frozen=True)
class ProviderModel:
id: str
name: str
model_id: str
enabled: bool = True
runtime: ModelRuntime = ModelRuntime()
@dataclass(frozen=True)
class ProviderProfile:
id: str
name: str
adapter: str
base_url: str
api_key: str
enabled: bool
models: tuple[ProviderModel, ...]
auth_mode: str = "api_key"
runtime: ProviderRuntime = ProviderRuntime()
@dataclass(frozen=True)
class ProviderProfiles:
version: int = PROVIDER_PROFILES_VERSION
builtins: tuple[ProviderProfile, ...] = ()
providers: tuple[ProviderProfile, ...] = ()
def normalize_provider_base_url(adapter: str, base_url: str) -> str:
"""Add the adapter's standard API path when only an origin is supplied."""
normalized = base_url.strip().rstrip("/")
default_path = _DEFAULT_API_PATHS.get(adapter)
if not normalized or not default_path:
return normalized
parsed = urlsplit(normalized)
if parsed.path not in {"", "/"} or parsed.query or parsed.fragment:
return normalized
return urlunsplit((parsed.scheme, parsed.netloc, default_path, "", ""))
def get_provider_profiles_path() -> Path:
"""Return the user-scoped provider profile path."""
return get_config_dir() / "providers.yaml"
def _required_string(
raw: dict[str, Any],
key: str,
*,
context: str,
max_length: int,
) -> str:
value = raw.get(key)
if not isinstance(value, str) or not value.strip():
raise ProviderProfileError(f"{context}.{key} is required.")
value = value.strip()
if len(value) > max_length:
raise ProviderProfileError(
f"{context}.{key} must be at most {max_length} characters."
)
return value
def _validate_id(value: str, *, context: str) -> str:
if not _ID_PATTERN.fullmatch(value):
raise ProviderProfileError(
f"{context} must start with a lowercase letter or digit and contain only "
"lowercase letters, digits, dots, underscores, or hyphens."
)
return value
def _optional_number(
raw: dict[str, Any], key: str, *, context: str, minimum: float, maximum: float
) -> float | None:
value = raw.get(key)
if value is None:
return None
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ProviderProfileError(f"{context}.{key} must be a number or null.")
numeric = float(value)
if not minimum <= numeric <= maximum:
raise ProviderProfileError(
f"{context}.{key} must be between {minimum:g} and {maximum:g}."
)
return numeric
def _optional_positive_int(raw: dict[str, Any], key: str, *, context: str) -> int | None:
value = raw.get(key)
if value is None:
return None
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
raise ProviderProfileError(f"{context}.{key} must be a positive integer or null.")
return value
def _parse_provider_runtime(raw: Any, *, context: str) -> ProviderRuntime:
if raw is None:
return ProviderRuntime()
if not isinstance(raw, dict):
raise ProviderProfileError(f"{context}.runtime must be an object.")
timeout_seconds = raw.get("timeout_seconds", 120)
max_retries = raw.get("max_retries", 2)
if isinstance(timeout_seconds, bool) or not isinstance(timeout_seconds, int):
raise ProviderProfileError(f"{context}.runtime.timeout_seconds must be an integer.")
if not 10 <= timeout_seconds <= 600:
raise ProviderProfileError(
f"{context}.runtime.timeout_seconds must be between 10 and 600."
)
if isinstance(max_retries, bool) or not isinstance(max_retries, int):
raise ProviderProfileError(f"{context}.runtime.max_retries must be an integer.")
if not 0 <= max_retries <= 5:
raise ProviderProfileError(
f"{context}.runtime.max_retries must be between 0 and 5."
)
effort = raw.get("default_reasoning_effort", "auto")
if effort not in {"auto", "low", "medium", "high"}:
raise ProviderProfileError(
f"{context}.runtime.default_reasoning_effort is invalid."
)
return ProviderRuntime(
timeout_seconds=timeout_seconds,
max_retries=max_retries,
default_temperature=_optional_number(
raw, "default_temperature", context=f"{context}.runtime", minimum=0, maximum=2
),
default_top_p=_optional_number(
raw, "default_top_p", context=f"{context}.runtime", minimum=0.000001, maximum=1
),
default_reasoning_effort=effort,
)
def resolve_model_input_limit(runtime: ModelRuntime) -> int:
"""Return the prompt limit declared by a confirmed model runtime."""
if runtime.limits_status != "confirmed" or runtime.limit_mode is None:
raise ProviderProfileError("Model limits require confirmation before use.")
if runtime.limit_mode == "combined":
if runtime.context_window_tokens is None:
raise ProviderProfileError("context_window_tokens is required for combined limits.")
return runtime.context_window_tokens - runtime.max_output_tokens
if runtime.limit_mode == "input_only":
if runtime.max_input_tokens is None:
raise ProviderProfileError("max_input_tokens is required for input_only limits.")
return runtime.max_input_tokens
raise ProviderProfileError("limit_mode must be combined or input_only.")
def _parse_model_runtime(raw: Any, *, context: str) -> ModelRuntime:
if raw is None:
return ModelRuntime()
if not isinstance(raw, dict):
raise ProviderProfileError(f"{context}.runtime must be an object.")
status = raw.get("limits_status", "needs_confirmation")
if status not in {"confirmed", "needs_confirmation"}:
raise ProviderProfileError(f"{context}.runtime.limits_status is invalid.")
source = raw.get("limits_source", "user")
if source not in {"catalog", "provider", "user"}:
raise ProviderProfileError(f"{context}.runtime.limits_source is invalid.")
limit_mode = raw.get("limit_mode")
if limit_mode is not None and limit_mode not in {"combined", "input_only"}:
raise ProviderProfileError(f"{context}.runtime.limit_mode is invalid.")
max_output = raw.get("max_output_tokens", 4096)
min_effective = raw.get("min_effective_input_tokens", 4096)
if isinstance(max_output, bool) or not isinstance(max_output, int) or max_output <= 0:
raise ProviderProfileError(f"{context}.runtime.max_output_tokens must be positive.")
if isinstance(min_effective, bool) or not isinstance(min_effective, int) or min_effective < 1024:
raise ProviderProfileError(
f"{context}.runtime.min_effective_input_tokens must be at least 1024."
)
effort = raw.get("reasoning_effort", "auto")
if effort not in {"auto", "low", "medium", "high"}:
raise ProviderProfileError(f"{context}.runtime.reasoning_effort is invalid.")
capabilities_raw = raw.get("capabilities", {})
if not isinstance(capabilities_raw, dict):
raise ProviderProfileError(f"{context}.runtime.capabilities must be an object.")
capabilities: list[tuple[str, bool | str]] = []
for key in ("tools", "vision", "structured_output"):
value = capabilities_raw.get(key, "auto")
if value not in {True, False, "auto"}:
raise ProviderProfileError(
f"{context}.runtime.capabilities.{key} must be true, false, or auto."
)
capabilities.append((key, value))
runtime = ModelRuntime(
limit_mode=limit_mode,
context_window_tokens=_optional_positive_int(
raw, "context_window_tokens", context=f"{context}.runtime"
),
max_input_tokens=_optional_positive_int(
raw, "max_input_tokens", context=f"{context}.runtime"
),
max_output_tokens=max_output,
min_effective_input_tokens=min_effective,
limits_status=status,
limits_source=source,
temperature=_optional_number(
raw, "temperature", context=f"{context}.runtime", minimum=0, maximum=2
),
top_p=_optional_number(
raw, "top_p", context=f"{context}.runtime", minimum=0.000001, maximum=1
),
reasoning_effort=effort,
capabilities=tuple(capabilities),
)
if status == "confirmed":
limit = resolve_model_input_limit(runtime)
safety = max(2048, int(limit * 0.10 + 0.999999))
if limit - safety < min_effective:
raise ProviderProfileError(
f"{context}.runtime leaves less than min_effective_input_tokens after safety reserve."
)
return runtime
def _parse_model(raw: Any, *, provider_id: str, index: int) -> ProviderModel:
context = f"providers[{provider_id}].models[{index}]"
if not isinstance(raw, dict):
raise ProviderProfileError(f"{context} must be an object.")
model_id = _validate_id(
_required_string(raw, "id", context=context, max_length=64),
context=f"{context}.id",
)
return ProviderModel(
id=model_id,
name=_required_string(raw, "name", context=context, max_length=120),
model_id=_required_string(raw, "model_id", context=context, max_length=300),
enabled=bool(raw.get("enabled", True)),
runtime=_parse_model_runtime(raw.get("runtime"), context=context),
)
def _parse_profile(raw: Any, *, index: int, builtin: bool = False) -> ProviderProfile:
collection = "builtins" if builtin else "providers"
context = f"{collection}[{index}]"
if not isinstance(raw, dict):
raise ProviderProfileError(f"{context} must be an object.")
provider_id = _validate_id(
_required_string(raw, "id", context=context, max_length=64),
context=f"{context}.id",
)
if builtin and provider_id not in _RESERVED_PROVIDER_IDS:
raise ProviderProfileError(
f"Built-in provider ID {provider_id!r} is not recognized."
)
if not builtin and provider_id in _RESERVED_PROVIDER_IDS:
raise ProviderProfileError(
f"Provider ID {provider_id!r} is reserved for a built-in provider. "
f"Use a unique ID such as {f'{provider_id}-custom'!r}."
)
adapter = _required_string(raw, "adapter", context=context, max_length=64)
if adapter not in SUPPORTED_PROVIDER_ADAPTERS:
raise ProviderProfileError(
f"{context}.adapter must be one of: "
f"{', '.join(SUPPORTED_PROVIDER_ADAPTERS)}."
)
base_url_raw = raw.get("base_url", "")
if not isinstance(base_url_raw, str):
raise ProviderProfileError(f"{context}.base_url must be a string.")
base_url = base_url_raw.strip().rstrip("/")
if len(base_url) > 2048:
raise ProviderProfileError(
f"{context}.base_url must be at most 2048 characters."
)
if adapter in _BASE_URL_REQUIRED_ADAPTERS and not base_url:
raise ProviderProfileError(
f"{context}.base_url is required for adapter {adapter}."
)
if base_url and not base_url.startswith(("http://", "https://")):
raise ProviderProfileError(f"{context}.base_url must use http:// or https://.")
base_url = normalize_provider_base_url(adapter, base_url)
api_key_raw = raw.get("api_key", "")
if not isinstance(api_key_raw, str):
raise ProviderProfileError(f"{context}.api_key must be a string.")
if len(api_key_raw) > 8192:
raise ProviderProfileError(
f"{context}.api_key must be at most 8192 characters."
)
auth_mode = raw.get("auth_mode", "api_key")
if not isinstance(auth_mode, str) or auth_mode not in {"api_key", "oauth"}:
raise ProviderProfileError(f"{context}.auth_mode must be 'api_key' or 'oauth'.")
models_raw = raw.get("models", [])
if not isinstance(models_raw, list):
raise ProviderProfileError(f"{context}.models must be a list.")
models = tuple(
_parse_model(model, provider_id=provider_id, index=model_index)
for model_index, model in enumerate(models_raw)
)
model_ids = [model.id for model in models]
if len(model_ids) != len(set(model_ids)):
raise ProviderProfileError(
f"Provider {provider_id!r} contains duplicate model IDs."
)
portable_reasoning_adapters = {
"openai",
"openai-compatible",
"grok",
"antigravity",
"openrouter",
}
if adapter not in portable_reasoning_adapters:
for model in models:
if model.runtime.reasoning_effort != "auto":
raise ProviderProfileError(
f"{context}.models[{model.id}].runtime.reasoning_effort is "
f"not supported by adapter {adapter!r}."
)
return ProviderProfile(
id=provider_id,
name=_required_string(raw, "name", context=context, max_length=120),
adapter=adapter,
base_url=base_url,
api_key=api_key_raw,
enabled=bool(raw.get("enabled", True)),
models=models,
auth_mode=auth_mode,
runtime=_parse_provider_runtime(raw.get("runtime"), context=context),
)
def _parse_document(raw: Any) -> ProviderProfiles:
if raw is None:
return ProviderProfiles()
if not isinstance(raw, dict):
raise ProviderProfileError("Provider profile document must be an object.")
version = raw.get("version")
if version != PROVIDER_PROFILES_VERSION:
raise ProviderProfileError(
"PROVIDER_PROFILE_RESET_REQUIRED: only version 3 provider profiles are supported."
)
builtins_raw = raw.get("builtins", [])
if not isinstance(builtins_raw, list):
raise ProviderProfileError("builtins must be a list.")
providers_raw = raw.get("providers", [])
if not isinstance(providers_raw, list):
raise ProviderProfileError("providers must be a list.")
providers = tuple(
_parse_profile(profile, index=index, builtin=False)
for index, profile in enumerate(providers_raw)
)
builtins = tuple(
_parse_profile(profile, index=index, builtin=True)
for index, profile in enumerate(builtins_raw)
)
provider_ids = [provider.id for provider in providers]
if len(provider_ids) != len(set(provider_ids)):
raise ProviderProfileError("Provider IDs must be unique.")
builtin_ids = [provider.id for provider in builtins]
if len(builtin_ids) != len(set(builtin_ids)):
raise ProviderProfileError("Built-in provider IDs must be unique.")
return ProviderProfiles(
version=PROVIDER_PROFILES_VERSION,
builtins=builtins,
providers=providers,
)
def _profile_to_dict(profile: ProviderProfile) -> dict[str, Any]:
return {
"id": profile.id,
"name": profile.name,
"adapter": profile.adapter,
"base_url": profile.base_url,
"api_key": profile.api_key,
"auth_mode": profile.auth_mode,
"enabled": profile.enabled,
"runtime": {
"timeout_seconds": profile.runtime.timeout_seconds,
"max_retries": profile.runtime.max_retries,
"default_temperature": profile.runtime.default_temperature,
"default_top_p": profile.runtime.default_top_p,
"default_reasoning_effort": profile.runtime.default_reasoning_effort,
},
"models": [
{
"id": model.id,
"name": model.name,
"model_id": model.model_id,
"enabled": model.enabled,
"runtime": {
"limit_mode": model.runtime.limit_mode,
"context_window_tokens": model.runtime.context_window_tokens,
"max_input_tokens": model.runtime.max_input_tokens,
"max_output_tokens": model.runtime.max_output_tokens,
"min_effective_input_tokens": model.runtime.min_effective_input_tokens,
"limits_status": model.runtime.limits_status,
"limits_source": model.runtime.limits_source,
"temperature": model.runtime.temperature,
"top_p": model.runtime.top_p,
"reasoning_effort": model.runtime.reasoning_effort,
"capabilities": dict(model.runtime.capabilities),
},
}
for model in profile.models
],
}
def _document_to_dict(document: ProviderProfiles) -> dict[str, Any]:
return {
"version": PROVIDER_PROFILES_VERSION,
"builtins": [_profile_to_dict(profile) for profile in document.builtins],
"providers": [_profile_to_dict(profile) for profile in document.providers],
}
def load_provider_profiles() -> ProviderProfiles:
"""Load provider profiles, returning an empty document when absent."""
path = get_provider_profiles_path()
with _STORE_LOCK:
if not path.exists():
return ProviderProfiles()
try:
with path.open(encoding="utf-8") as handle:
return _parse_document(yaml.safe_load(handle))
except ProviderProfileError:
raise
except Exception as exc:
raise ProviderProfileError(
f"Could not load provider profiles: {exc}"
) from exc
def save_provider_profiles(document: ProviderProfiles) -> None:
"""Atomically persist provider profiles with private permissions."""
path = get_provider_profiles_path()
payload = _document_to_dict(document)
with _STORE_LOCK:
path.parent.mkdir(parents=True, exist_ok=True)
try:
path.parent.chmod(0o700)
except OSError:
pass
fd, temporary = tempfile.mkstemp(
prefix="providers-", suffix=".yaml.tmp", dir=path.parent
)
try:
with os.fdopen(fd, "w", encoding="utf-8") as handle:
yaml.safe_dump(
payload,
handle,
default_flow_style=False,
sort_keys=False,
allow_unicode=True,
)
handle.flush()
os.fsync(handle.fileno())
try:
Path(temporary).chmod(0o600)
except OSError:
pass
os.replace(temporary, path)
try:
path.chmod(0o600)
except OSError:
pass
finally:
try:
Path(temporary).unlink(missing_ok=True)
except OSError:
pass
def replace_provider_profiles(raw: Any) -> ProviderProfiles:
"""Replace supplied profile collections while preserving the other side.
Existing API keys are retained when an existing profile sends no ``api_key``
or sends an empty value. ``clear_api_key: true`` explicitly removes it.
"""
if not isinstance(raw, dict):
raise ProviderProfileError("Request body must be an object.")
if "providers" not in raw and "builtins" not in raw:
raise ProviderProfileError("providers or builtins must be supplied.")
current = load_provider_profiles()
def merge_collection(
key: str,
supplied: Any,
existing_profiles: tuple[ProviderProfile, ...],
) -> list[dict[str, Any]]:
if supplied is None:
return [_profile_to_dict(profile) for profile in existing_profiles]
if not isinstance(supplied, list):
raise ProviderProfileError(f"{key} must be a list.")
existing = {profile.id: profile for profile in existing_profiles}
merged: list[dict[str, Any]] = []
for index, item in enumerate(supplied):
if not isinstance(item, dict):
raise ProviderProfileError(f"{key}[{index}] must be an object.")
candidate = dict(item)
provider_id = candidate.get("id")
prior = existing.get(provider_id) if isinstance(provider_id, str) else None
if candidate.pop("clear_api_key", False):
candidate["api_key"] = ""
elif not candidate.get("api_key") and prior is not None:
candidate["api_key"] = prior.api_key
else:
candidate.setdefault("api_key", "")
merged.append(candidate)
return merged
document = _parse_document(
{
"version": PROVIDER_PROFILES_VERSION,
"builtins": merge_collection(
"builtins", raw.get("builtins"), current.builtins
),
"providers": merge_collection(
"providers", raw.get("providers"), current.providers
),
}
)
save_provider_profiles(document)
return document
def resolve_provider_profile_draft(raw: Any) -> ProviderProfile:
"""Validate a WebUI draft and restore its saved API key when omitted.
Provider discovery and model tests operate on unsaved form values. The
browser never receives an existing secret, so an empty ``api_key`` keeps
the saved value for the matching provider ID. ``clear_api_key`` explicitly
opts out of that fallback.
"""
if not isinstance(raw, dict):
raise ProviderProfileError("provider must be an object.")
candidate = dict(raw)
provider_id = candidate.get("id")
is_builtin = isinstance(provider_id, str) and provider_id in _RESERVED_PROVIDER_IDS
prior = (
get_builtin_provider_profile(provider_id)
if is_builtin
else get_provider_profile(provider_id)
if isinstance(provider_id, str)
else None
)
if candidate.pop("clear_api_key", False):
candidate["api_key"] = ""
elif not candidate.get("api_key") and prior is not None:
candidate["api_key"] = prior.api_key
else:
candidate.setdefault("api_key", "")
candidate.setdefault("models", [])
return _parse_profile(candidate, index=0, builtin=is_builtin)
def _api_key_hint(api_key: str) -> str | None:
if not api_key:
return None
if len(api_key) <= 8:
return "configured"
return f"{api_key[:3]}...{api_key[-4:]}"
def _provider_profiles_public_payload(document: ProviderProfiles) -> dict[str, Any]:
def profile_payload(profile: ProviderProfile) -> dict[str, Any]:
return {
"id": profile.id,
"name": profile.name,
"adapter": profile.adapter,
"base_url": profile.base_url,
"auth_mode": profile.auth_mode,
"enabled": profile.enabled,
"api_key_configured": bool(profile.api_key),
"api_key_hint": _api_key_hint(profile.api_key),
"runtime": {
"timeout_seconds": profile.runtime.timeout_seconds,
"max_retries": profile.runtime.max_retries,
"default_temperature": profile.runtime.default_temperature,
"default_top_p": profile.runtime.default_top_p,
"default_reasoning_effort": profile.runtime.default_reasoning_effort,
},
"models": [
{
"id": model.id,
"name": model.name,
"model_id": model.model_id,
"enabled": model.enabled,
"runtime": {
"limit_mode": model.runtime.limit_mode,
"context_window_tokens": model.runtime.context_window_tokens,
"max_input_tokens": model.runtime.max_input_tokens,
"max_output_tokens": model.runtime.max_output_tokens,
"min_effective_input_tokens": model.runtime.min_effective_input_tokens,
"limits_status": model.runtime.limits_status,
"limits_source": model.runtime.limits_source,
"temperature": model.runtime.temperature,
"top_p": model.runtime.top_p,
"reasoning_effort": model.runtime.reasoning_effort,
"capabilities": dict(model.runtime.capabilities),
},
}
for model in profile.models
],
}
return {
"version": document.version,
"builtins": [profile_payload(profile) for profile in document.builtins],
"providers": [profile_payload(profile) for profile in document.providers],
}
def provider_profiles_public(
document: ProviderProfiles | None = None,
) -> dict[str, Any]:
"""Return a browser-safe representation with all secrets removed."""
document = document or load_provider_profiles()
public = _provider_profiles_public_payload(document)
public["reserved_provider_ids"] = sorted(_RESERVED_PROVIDER_IDS)
public["revision"] = provider_profiles_public_revision(document)
return public
def provider_profiles_public_revision(
document: ProviderProfiles | None = None,
) -> str:
"""Return a browser-safe digest that excludes API key material."""
document = document or load_provider_profiles()
encoded = json.dumps(
_provider_profiles_public_payload(document),
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()[:16]
def provider_profiles_revision(
document: ProviderProfiles | None = None,
) -> str:
"""Return a private revision that changes for connection secrets too."""
document = document or load_provider_profiles()
encoded = json.dumps(
_document_to_dict(document), sort_keys=True, separators=(",", ":")
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()[:16]
def get_provider_profile(provider_id: str) -> ProviderProfile | None:
for profile in load_provider_profiles().providers:
if profile.id == provider_id:
return profile
return None
def get_builtin_provider_profile(provider_id: str) -> ProviderProfile | None:
for profile in load_provider_profiles().builtins:
if profile.id == provider_id:
return profile
return None
def get_provider_profile_revision(provider_id: str) -> str | None:
profile = get_builtin_provider_profile(provider_id) or get_provider_profile(
provider_id
)
if profile is None:
return None
encoded = json.dumps(
_profile_to_dict(profile), sort_keys=True, separators=(",", ":")
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()[:16]
def resolve_provider_model(
provider_id: str, model_name: str
) -> tuple[ProviderProfile, ProviderModel] | None:
"""Resolve a configured ``(provider_id, model alias)`` pair."""
profile = get_provider_profile(provider_id)
if profile is None:
return None
if not profile.enabled:
raise ProviderProfileError(f"Provider {provider_id!r} is disabled.")
for model in profile.models:
if model.id == model_name:
if not model.enabled:
raise ProviderProfileError(
f"Model {model_name!r} in provider {provider_id!r} is disabled."
)
if model.runtime.limits_status != "confirmed":
raise ProviderProfileError(
f"RUNTIME_LIMITS_CONFIRMATION_REQUIRED: model {model_name!r} needs confirmed limits."
)
return profile, model
raise ProviderProfileError(
f"Model {model_name!r} is not configured for provider {provider_id!r}."
)
def list_configured_model_entries() -> list[tuple[str, str, str]]:
"""Return enabled dynamic models as ``(alias, upstream id, provider id)``."""
entries: list[tuple[str, str, str]] = []
for profile in load_provider_profiles().providers:
if not profile.enabled:
continue
for model in profile.models:
if model.enabled:
entries.append((model.id, model.model_id, profile.id))
return entries
def list_configured_builtin_model_entries() -> list[tuple[str, str, str]]:
"""Return enabled managed built-in models."""
entries: list[tuple[str, str, str]] = []
for profile in load_provider_profiles().builtins:
if not profile.enabled:
continue
for model in profile.models:
if model.enabled:
entries.append((model.id, model.model_id, profile.id))
return entries
+15 -112
View File
@@ -146,60 +146,21 @@ def get_default_workspace_dir() -> Path:
class EvoScientistConfig:
"""EvoScientist configuration settings.
LLM provider / model / API key configuration lives in the model
registry (model-runtime.sqlite3), not here — this dataclass only holds
platform settings (workspace, UI, channels, memory, HITL, …).
Attributes:
anthropic_api_key: Anthropic API key for Claude models.
openai_api_key: OpenAI API key for GPT models.
nvidia_api_key: NVIDIA API key for NVIDIA models.
google_api_key: Google API key for Gemini models.
tavily_api_key: Tavily API key for web search.
provider: Default LLM provider ('anthropic', 'openai', 'google-genai', or 'nvidia').
model: Default model name (short name or full ID).
auxiliary_provider: Provider for auxiliary_model (empty = use main provider).
auxiliary_model: Model for memory workers + tool selector + scheduler (empty = use main model).
default_mode: Default workspace mode ('daemon' or 'run').
default_workdir: Default workspace directory (empty = use
~/.evoscientist/workspace).
show_thinking: Whether to show thinking panels in CLI.
"""
# API Keys
anthropic_api_key: str = ""
anthropic_base_url: str = ""
anthropic_auth_mode: str = "api_key" # "api_key" | "oauth"
openai_api_key: str = ""
openai_auth_mode: str = "api_key" # "api_key" | "oauth"
nvidia_api_key: str = ""
google_api_key: str = ""
minimax_api_key: str = ""
minimax_base_url: str = ""
siliconflow_api_key: str = ""
openrouter_api_key: str = ""
deepseek_api_key: str = ""
zhipu_api_key: str = ""
volcengine_api_key: str = ""
dashscope_api_key: str = ""
moonshot_api_key: str = ""
kimi_api_key: str = ""
custom_openai_api_key: str = ""
custom_openai_base_url: str = ""
custom_anthropic_api_key: str = ""
custom_anthropic_base_url: str = ""
ollama_base_url: str = ""
# API Keys (non-LLM tools only)
tavily_api_key: str = ""
# LLM Settings
provider: str = "anthropic"
model: str = "claude-sonnet-4-6"
# Legacy built-in model allowlist. New WebUI saves store built-in and
# custom model catalogs in providers.yaml; this remains as a read-only
# compatibility fallback until the first Registry save.
model_catalog: list[dict[str, Any]] | None = None
model_fallbacks: str = "" # "model:provider,model:provider" fallback chain
# Optional auxiliary model for background/helper LLM calls (memory workers +
# tool selector). Empty = fall back to the main model/provider.
auxiliary_provider: str = "" # empty = use main provider
auxiliary_model: str = "" # empty = use main model
# Async Sub-agent Settings
# When True (default), the EvoSci CLI auto-starts a langgraph dev subprocess
# so any sub-agent flagged ``async: true`` in subagents/<name>.yaml runs
@@ -302,9 +263,6 @@ class EvoScientistConfig:
ui_backend: Literal["cli", "tui", "webui"] = "tui"
log_level: str = "warning"
reasoning_effort: str = "high"
# Anthropic prompt caching for OpenRouter anthropic/* models. Opt out if
# cache-write costs outweigh the benefit for a workflow.
openrouter_anthropic_prompt_cache: bool = True
# Channel Settings
channel_enabled: str = "" # "imessage" | "telegram" | "discord" | "slack" | "wechat" | "dingtalk" | "feishu" | "email" | "qq" | "signal" | "" (comma-separated for multiple)
@@ -460,9 +418,6 @@ class EvoScientistConfig:
# DM access control policy
dm_policy: str = "allowlist"
# OpenAI API mode - "" = auto, "true" = force Responses, "false" = force Completions
use_responses_api: str = ""
# ccproxy
ccproxy_port: int = 8000
@@ -497,7 +452,9 @@ class EvoScientistConfig:
"workspace_isolation must be one of legacy, optional, required"
)
if self.scope_registry_topology != "single-host":
raise ValueError("v1 workspace isolation only supports single-host topology")
raise ValueError(
"v1 workspace isolation only supports single-host topology"
)
if self.strict_executor != "oci":
raise ValueError("strict_executor must be oci")
if self.strict_code_interpreter not in {"disabled", "scoped"}:
@@ -513,7 +470,9 @@ class EvoScientistConfig:
or isinstance(self.workspace_trash_retention_days, bool)
or self.workspace_trash_retention_days <= 0
):
raise ValueError("workspace_trash_retention_days must be a positive integer")
raise ValueError(
"workspace_trash_retention_days must be a positive integer"
)
if self.workspace_isolation == "required" and self.dangerous_mode:
raise ValueError(
"dangerous_mode is incompatible with required workspace isolation"
@@ -802,40 +761,12 @@ def list_config() -> dict[str, Any]:
# Environment variable mappings
_ENV_MAPPINGS = {
"anthropic_api_key": "ANTHROPIC_API_KEY",
"anthropic_base_url": "ANTHROPIC_BASE_URL",
"anthropic_auth_mode": "EVOSCIENTIST_ANTHROPIC_AUTH_MODE",
"openai_api_key": "OPENAI_API_KEY",
"openai_auth_mode": "EVOSCIENTIST_OPENAI_AUTH_MODE",
"nvidia_api_key": "NVIDIA_API_KEY",
"google_api_key": "GOOGLE_API_KEY",
"minimax_api_key": "MINIMAX_API_KEY",
"minimax_base_url": "MINIMAX_BASE_URL",
"siliconflow_api_key": "SILICONFLOW_API_KEY",
"openrouter_api_key": "OPENROUTER_API_KEY",
"deepseek_api_key": "DEEPSEEK_API_KEY",
"zhipu_api_key": "ZHIPU_API_KEY",
"volcengine_api_key": "VOLCENGINE_API_KEY",
"dashscope_api_key": "DASHSCOPE_API_KEY",
"moonshot_api_key": "MOONSHOT_API_KEY",
"kimi_api_key": "KIMI_API_KEY",
"custom_openai_api_key": "CUSTOM_OPENAI_API_KEY",
"custom_openai_base_url": "CUSTOM_OPENAI_BASE_URL",
"custom_anthropic_api_key": "CUSTOM_ANTHROPIC_API_KEY",
"custom_anthropic_base_url": "CUSTOM_ANTHROPIC_BASE_URL",
"ollama_base_url": "OLLAMA_BASE_URL",
"tavily_api_key": "TAVILY_API_KEY",
"default_mode": "EVOSCIENTIST_DEFAULT_MODE",
"default_workdir": "EVOSCIENTIST_WORKSPACE_DIR",
"ui_backend": "EVOSCIENTIST_UI_BACKEND",
"log_level": "EVOSCIENTIST_LOG_LEVEL",
"model_fallbacks": "EVOSCIENTIST_MODEL_FALLBACKS",
"auxiliary_provider": "EVOSCIENTIST_AUXILIARY_PROVIDER",
"auxiliary_model": "EVOSCIENTIST_AUXILIARY_MODEL",
"reasoning_effort": "EVOSCIENTIST_REASONING_EFFORT",
"openrouter_anthropic_prompt_cache": (
"EVOSCIENTIST_OPENROUTER_ANTHROPIC_PROMPT_CACHE"
),
"dangerous_mode": "EVOSCIENTIST_DANGEROUS_MODE",
"workspace_isolation": "EVOSCIENTIST_WORKSPACE_ISOLATION",
"scope_registry_topology": "EVOSCIENTIST_SCOPE_REGISTRY_TOPOLOGY",
@@ -846,7 +777,6 @@ _ENV_MAPPINGS = {
"workspace_trash_retention_days": "EVOSCIENTIST_WORKSPACE_TRASH_RETENTION_DAYS",
"channel_debug_tracing": "EVOSCIENTIST_CHANNEL_DEBUG_TRACING",
"ccproxy_port": "EVOSCIENTIST_CCPROXY_PORT",
"use_responses_api": "EVOSCIENTIST_USE_RESPONSES_API",
"checkpoint_keep_per_thread": "EVOSCIENTIST_CHECKPOINT_KEEP_PER_THREAD",
"enable_async_subagents": "EVOSCIENTIST_ENABLE_ASYNC_SUBAGENTS",
"langgraph_dev_port": "EVOSCIENTIST_LANGGRAPH_DEV_PORT",
@@ -918,42 +848,19 @@ def get_effective_config(
def apply_config_to_env(config: EvoScientistConfig) -> None:
"""Apply config API keys to environment variables if not already set.
"""Apply config values to environment variables if not already set.
This allows the config file to provide API keys that downstream
libraries (like langchain-anthropic) can pick up.
LLM provider keys are no longer injected here — they live in the model
registry (model-runtime.sqlite3) and are applied by the model runtime.
Only non-LLM tool keys and platform round-trips remain.
Args:
config: Configuration to apply.
"""
_apply_config_env("ANTHROPIC_API_KEY", config.anthropic_api_key)
_apply_config_env("ANTHROPIC_BASE_URL", config.anthropic_base_url)
_apply_config_env("OPENAI_API_KEY", config.openai_api_key)
_apply_config_env("NVIDIA_API_KEY", config.nvidia_api_key)
_apply_config_env("GOOGLE_API_KEY", config.google_api_key)
_apply_config_env("MINIMAX_API_KEY", config.minimax_api_key)
_apply_config_env("MINIMAX_BASE_URL", config.minimax_base_url)
_apply_config_env("SILICONFLOW_API_KEY", config.siliconflow_api_key)
_apply_config_env("OPENROUTER_API_KEY", config.openrouter_api_key)
_apply_config_env("DEEPSEEK_API_KEY", config.deepseek_api_key)
_apply_config_env("ZHIPU_API_KEY", config.zhipu_api_key)
_apply_config_env("VOLCENGINE_API_KEY", config.volcengine_api_key)
_apply_config_env("DASHSCOPE_API_KEY", config.dashscope_api_key)
_apply_config_env("MOONSHOT_API_KEY", config.moonshot_api_key)
_apply_config_env("KIMI_API_KEY", config.kimi_api_key)
_apply_config_env("CUSTOM_OPENAI_API_KEY", config.custom_openai_api_key)
_apply_config_env("CUSTOM_OPENAI_BASE_URL", config.custom_openai_base_url)
_apply_config_env("CUSTOM_ANTHROPIC_API_KEY", config.custom_anthropic_api_key)
_apply_config_env("CUSTOM_ANTHROPIC_BASE_URL", config.custom_anthropic_base_url)
_apply_config_env("OLLAMA_BASE_URL", config.ollama_base_url)
if config.tavily_api_key and not os.environ.get("TAVILY_API_KEY"):
os.environ["TAVILY_API_KEY"] = config.tavily_api_key
if config.reasoning_effort and not os.environ.get("EVOSCIENTIST_REASONING_EFFORT"):
os.environ["EVOSCIENTIST_REASONING_EFFORT"] = config.reasoning_effort
if not config.openrouter_anthropic_prompt_cache and not os.environ.get(
"EVOSCIENTIST_OPENROUTER_ANTHROPIC_PROMPT_CACHE"
):
os.environ["EVOSCIENTIST_OPENROUTER_ANTHROPIC_PROMPT_CACHE"] = "false"
# Round-trip dangerous_mode to env so it survives a fresh get_effective_config()
# (warning banner, run_in_background) and is inherited by the langgraph dev
# subprocess — otherwise a --dangerous CLI flag (not persisted to file/env)
@@ -964,7 +871,3 @@ def apply_config_to_env(config: EvoScientistConfig) -> None:
os.environ["EVOSCIENTIST_DANGEROUS_MODE"] = "true"
else:
os.environ.pop("EVOSCIENTIST_DANGEROUS_MODE", None)
if config.use_responses_api and not os.environ.get(
"EVOSCIENTIST_USE_RESPONSES_API"
):
os.environ["EVOSCIENTIST_USE_RESPONSES_API"] = config.use_responses_api
+7 -2
View File
@@ -254,9 +254,14 @@ def deploy(
"trust.[/bold red]"
)
# 6. ccproxy lifecycle (only if any provider uses OAuth)
# 6. ccproxy lifecycle (only if any provider uses OAuth). The legacy
# config.yaml auth-mode fields are gone with the unified model registry;
# getattr degrades to "api_key" so the OAuth branch simply stays off here.
_ccproxy_proc = None
if config.anthropic_auth_mode == "oauth" or config.openai_auth_mode == "oauth":
if (
getattr(config, "anthropic_auth_mode", "api_key") == "oauth"
or getattr(config, "openai_auth_mode", "api_key") == "oauth"
):
try:
from ..ccproxy_manager import maybe_start_ccproxy, stop_ccproxy
-12
View File
@@ -63,7 +63,6 @@ def run_webui(config: Any, workspace_dir: str | None = None) -> None:
tears down both subprocesses. Never returns a value.
"""
from ..config import apply_config_to_env, get_default_workspace_dir
from ..config.provider_admin import get_provider_admin_token
from ..langgraph_dev.manager import (
_DEFAULT_PORT,
RUNTIME,
@@ -149,16 +148,6 @@ def run_webui(config: Any, workspace_dir: str | None = None) -> None:
usage_env = {}
os.environ.update(usage_env)
# The provider editor uses a server-to-server management token. Resolve it
# through the shared user-level token file so a separately started WebUI,
# a fresh backend, and a reused backend all authenticate with the same
# value. The browser never receives it; the Next.js proxy adds it.
provider_admin_token = get_provider_admin_token()
if not provider_admin_token:
console.print("[red]Could not create the provider management token.[/red]")
raise typer.Exit(1)
os.environ["EVOSCIENTIST_PROVIDER_ADMIN_TOKEN"] = provider_admin_token
# 4. Backend (langgraph dev): reuse an EvoSci server already on the port,
# else start a fresh deploy-mode one (full MCP + async). Refuse a foreign
# occupant — that's a configuration error, not something to silently share.
@@ -237,7 +226,6 @@ def run_webui(config: Any, workspace_dir: str | None = None) -> None:
**usage_env,
"EVOSCIENTIST_LANGGRAPH_DEV_PORT": str(backend_port),
"EVOSCIENTIST_BACKEND_URL": f"http://127.0.0.1:{backend_port}",
"EVOSCIENTIST_PROVIDER_ADMIN_TOKEN": provider_admin_token,
"EVOSCIENTIST_BACKEND_SERVICE_TOKEN": scope_service_token,
"EVOSCIENTIST_WORKSPACE_DIR": ws,
"PORT": str(webui_port),
+20
View File
@@ -147,14 +147,34 @@ class LocalGraphGateway:
target: GraphTarget,
request: RunRequest,
) -> AsyncIterator[GraphEvent]:
from langgraph.types import Command
from ..stream.events import stream_agent_events
# Local snapshot entry (design doc 8.1): freeze the registry
# defaults into a per-turn run snapshot and carry only its ID in
# configurable. Resumed runs (HITL Command(resume=...)) keep the
# snapshot the original turn froze — a resume must not re-freeze
# newer defaults. A bootstrap registry raises
# MODEL_REGISTRY_NOT_READY here instead of falling back to any
# implicit default model.
runtime_snapshot_id: str | None = None
if not (isinstance(request.message, Command) and request.message.resume):
from ..model_registry.runtime import get_snapshot_runtime
runtime_snapshot_id = (
get_snapshot_runtime()
.create_local_snapshot(request.thread_id)
.snapshot_id
)
inner = stream_agent_events(
local_graph,
request.message,
request.thread_id,
metadata=request.metadata,
media=request.media,
runtime_snapshot_id=runtime_snapshot_id,
)
try:
async for event in inner:
+11
View File
@@ -568,6 +568,17 @@ class LangGraphServerGateway:
request.message,
media=request.media,
)
# Local snapshot entry (design doc 8.1): the langgraph server shares
# the same model-runtime store, so the snapshot frozen here is the
# one the server-side run resolves. Resume turns returned above and
# keep the snapshot the original turn froze.
from ..model_registry.runtime import get_snapshot_runtime
config["configurable"]["runtime_snapshot_id"] = (
get_snapshot_runtime()
.create_local_snapshot(request.thread_id)
.snapshot_id
)
await stream.run.start(
input=run_input,
config=config,
+61 -786
View File
@@ -14,22 +14,20 @@ Starlette; adding FastAPI would pull in pydantic v1-vs-v2 reconciliation
the deploy doesn't need. The one route here has no input model, just a
JSON body, so the lower-level surface is sufficient.
Lightweight by design — module-level imports stick to ``config``,
``llm.models`` (registry only; no chat-model construction), and
Starlette itself. Nothing on this surface should pull the agent into
memory.
Lightweight by design — module-level imports stick to
``model_registry.http_api``, ``sessions``, and Starlette itself. Nothing
on this surface should pull the agent into memory.
"""
from __future__ import annotations
import asyncio
import hashlib
import contextlib
import json
import logging
import os
import secrets
import sqlite3
import threading
from pathlib import Path
from typing import Any
@@ -42,39 +40,7 @@ from starlette.requests import Request
from starlette.responses import JSONResponse
from starlette.routing import Route
from EvoScientist.config import (
get_effective_config,
is_config_applied_env,
load_config,
save_config,
)
from EvoScientist.config.provider_admin import get_provider_admin_token
from EvoScientist.config.provider_profiles import (
ProviderModel,
ProviderProfileError,
list_configured_builtin_model_entries,
list_configured_model_entries,
load_provider_profiles,
provider_profiles_public,
provider_profiles_revision,
replace_provider_profiles,
resolve_provider_profile_draft,
)
from EvoScientist.llm.models import (
list_builtin_model_catalog_entries,
list_model_picker_entries,
list_models_by_provider,
normalize_builtin_model_catalog,
)
from EvoScientist.llm.provider_operations import (
ProviderOperationError,
build_builtin_provider_profile,
builtin_provider_adapter,
builtin_provider_ids,
builtin_provider_legacy_api_key,
discover_provider_models,
test_provider_model,
)
from EvoScientist.config.legacy_artifacts import assert_no_legacy_artifacts
from EvoScientist.model_registry.http_api import model_registry_routes
from EvoScientist.sessions import (
MAIN_THREAD_FILTER_PARAMS,
@@ -85,730 +51,6 @@ from EvoScientist.sessions import (
)
_logger = logging.getLogger(__name__)
_config_file_lock = threading.RLock()
_LLM_CONFIG_VALUE_FIELDS = (
"provider",
"model",
"model_fallbacks",
"auxiliary_provider",
"auxiliary_model",
"anthropic_base_url",
"anthropic_auth_mode",
"openai_auth_mode",
"minimax_base_url",
"custom_openai_base_url",
"custom_anthropic_base_url",
"ollama_base_url",
)
_LLM_CONFIG_SECRET_FIELDS = (
"anthropic_api_key",
"openai_api_key",
"nvidia_api_key",
"google_api_key",
"minimax_api_key",
"siliconflow_api_key",
"openrouter_api_key",
"deepseek_api_key",
"zhipu_api_key",
"volcengine_api_key",
"dashscope_api_key",
"moonshot_api_key",
"kimi_api_key",
"custom_openai_api_key",
"custom_anthropic_api_key",
)
_LLM_CONFIG_ENV_VARS = {
"anthropic_api_key": "ANTHROPIC_API_KEY",
"anthropic_base_url": "ANTHROPIC_BASE_URL",
"anthropic_auth_mode": "EVOSCIENTIST_ANTHROPIC_AUTH_MODE",
"openai_api_key": "OPENAI_API_KEY",
"openai_auth_mode": "EVOSCIENTIST_OPENAI_AUTH_MODE",
"nvidia_api_key": "NVIDIA_API_KEY",
"google_api_key": "GOOGLE_API_KEY",
"minimax_api_key": "MINIMAX_API_KEY",
"minimax_base_url": "MINIMAX_BASE_URL",
"siliconflow_api_key": "SILICONFLOW_API_KEY",
"openrouter_api_key": "OPENROUTER_API_KEY",
"deepseek_api_key": "DEEPSEEK_API_KEY",
"zhipu_api_key": "ZHIPU_API_KEY",
"volcengine_api_key": "VOLCENGINE_API_KEY",
"dashscope_api_key": "DASHSCOPE_API_KEY",
"moonshot_api_key": "MOONSHOT_API_KEY",
"kimi_api_key": "KIMI_API_KEY",
"custom_openai_api_key": "CUSTOM_OPENAI_API_KEY",
"custom_openai_base_url": "CUSTOM_OPENAI_BASE_URL",
"custom_anthropic_api_key": "CUSTOM_ANTHROPIC_API_KEY",
"custom_anthropic_base_url": "CUSTOM_ANTHROPIC_BASE_URL",
"ollama_base_url": "OLLAMA_BASE_URL",
"model_fallbacks": "EVOSCIENTIST_MODEL_FALLBACKS",
"auxiliary_provider": "EVOSCIENTIST_AUXILIARY_PROVIDER",
"auxiliary_model": "EVOSCIENTIST_AUXILIARY_MODEL",
}
_LLM_CONFIG_BASE_URL_FIELDS = {
"anthropic_base_url",
"minimax_base_url",
"custom_openai_base_url",
"custom_anthropic_base_url",
"ollama_base_url",
}
_LLM_CONFIG_AUTH_MODE_FIELDS = {"anthropic_auth_mode", "openai_auth_mode"}
_LLM_CONFIG_IMMEDIATE_FIELDS = {
"provider",
"model",
"model_catalog",
"builtin_providers",
}
class _ConfigRevisionConflict(ValueError):
"""Raised when config.yaml changed after the WebUI loaded it."""
def _llm_config_revision(config: Any) -> str:
values = {
field: getattr(config, field)
for field in (*_LLM_CONFIG_VALUE_FIELDS, *_LLM_CONFIG_SECRET_FIELDS)
}
values["model_catalog"] = getattr(config, "model_catalog", None)
values["provider_registry_revision"] = provider_profiles_revision()
encoded = json.dumps(values, sort_keys=True, separators=(",", ":")).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()[:16]
def _model_catalog_profile_models(
config: Any, provider_id: str
) -> tuple[ProviderModel, ...]:
try:
catalog = normalize_builtin_model_catalog(config.model_catalog) or []
except ValueError:
return ()
return tuple(
ProviderModel(
id=entry["id"],
name=entry["name"],
model_id=entry["model_id"],
enabled=entry["enabled"],
)
for entry in catalog
if entry["provider"] == provider_id
)
def _builtin_profiles_public(config: Any, document: Any) -> list[dict[str, Any]]:
managed_by_id = {profile.id: profile for profile in document.builtins}
result: list[dict[str, Any]] = []
for provider_id in builtin_provider_ids():
managed = managed_by_id.get(provider_id)
effective = build_builtin_provider_profile(
config,
provider_id,
saved_profile=managed,
load_saved=False,
)
models = (
managed.models
if managed is not None
else _model_catalog_profile_models(config, provider_id)
)
result.append(
{
"id": provider_id,
"name": managed.name if managed is not None else provider_id,
"adapter": builtin_provider_adapter(provider_id),
"base_url": effective.base_url,
"auth_mode": effective.auth_mode,
"enabled": managed.enabled if managed is not None else True,
"managed": managed is not None,
"api_key_configured": bool(effective.api_key),
"api_key_hint": (
f"...{effective.api_key[-4:]}" if effective.api_key else None
),
"models": [
{
"id": model.id,
"name": model.name,
"model_id": model.model_id,
"enabled": model.enabled,
}
for model in models
],
}
)
return result
def _llm_config_public(config: Any) -> dict[str, Any]:
values = {field: getattr(config, field) for field in _LLM_CONFIG_VALUE_FIELDS}
secret_status = {}
for field in _LLM_CONFIG_SECRET_FIELDS:
value = getattr(config, field)
secret_status[field] = {
"configured": bool(value),
"hint": f"...{value[-4:]}" if value else None,
}
env_overrides = {
field: env_name
for field, env_name in _LLM_CONFIG_ENV_VARS.items()
if os.environ.get(env_name) and not is_config_applied_env(env_name)
}
catalog_error = None
provider_document = load_provider_profiles()
managed_builtins = provider_document.builtins
if managed_builtins:
model_catalog = [
{
"provider": profile.id,
"id": model.id,
"name": model.name,
"model_id": model.model_id,
"enabled": model.enabled,
}
for profile in managed_builtins
for model in profile.models
]
else:
try:
model_catalog = normalize_builtin_model_catalog(config.model_catalog)
except ValueError as exc:
model_catalog = []
catalog_error = str(exc)
return {
"revision": _llm_config_revision(config),
"values": values,
"secrets": secret_status,
"env_overrides": env_overrides,
"model_catalog": model_catalog,
"model_catalog_error": catalog_error,
"builtin_providers": _builtin_profiles_public(config, provider_document),
"builtin_model_candidates": [
{
"id": name,
"name": name,
"model_id": model_id,
"provider": provider,
"enabled": True,
}
for name, model_id, provider in list_models_by_provider()
],
}
def _read_llm_config() -> dict[str, Any]:
with _config_file_lock:
return _llm_config_public(load_config())
def _validated_llm_config_value(field: str, value: Any) -> str:
if not isinstance(value, str):
raise ValueError(f"{field} must be a string.")
value = value.strip()
max_length = 4096 if field == "model_fallbacks" else 2048
if field in {"provider", "auxiliary_provider"}:
max_length = 64
elif field in {"model", "auxiliary_model"}:
max_length = 300
elif field in _LLM_CONFIG_AUTH_MODE_FIELDS:
max_length = 16
if len(value) > max_length:
raise ValueError(f"{field} must be at most {max_length} characters.")
if field in _LLM_CONFIG_AUTH_MODE_FIELDS and value not in {"api_key", "oauth"}:
raise ValueError(f"{field} must be 'api_key' or 'oauth'.")
if (
field in _LLM_CONFIG_BASE_URL_FIELDS
and value
and not value.startswith(("http://", "https://"))
):
raise ValueError(f"{field} must use http:// or https://.")
return value
def _patch_llm_config(payload: Any) -> dict[str, Any]:
if not isinstance(payload, dict):
raise ValueError("Request body must be an object.")
revision = payload.get("revision")
if not isinstance(revision, str) or not revision:
raise ValueError("revision is required.")
values = payload.get("values", {})
secrets_payload = payload.get("secrets", {})
clear_secrets = payload.get("clear_secrets", [])
builtin_profiles_supplied = "builtin_providers" in payload
builtin_profiles_payload = payload.get("builtin_providers")
if builtin_profiles_supplied and not isinstance(builtin_profiles_payload, list):
raise ValueError("builtin_providers must be a list.")
catalog_supplied = "model_catalog" in payload and not builtin_profiles_supplied
normalized_catalog = (
normalize_builtin_model_catalog(payload.get("model_catalog"))
if catalog_supplied
else None
)
if not isinstance(values, dict):
raise ValueError("values must be an object.")
if not isinstance(secrets_payload, dict):
raise ValueError("secrets must be an object.")
if not isinstance(clear_secrets, list) or not all(
isinstance(field, str) for field in clear_secrets
):
raise ValueError("clear_secrets must be a list of strings.")
unknown_values = set(values) - set(_LLM_CONFIG_VALUE_FIELDS)
unknown_secrets = set(secrets_payload) - set(_LLM_CONFIG_SECRET_FIELDS)
unknown_clears = set(clear_secrets) - set(_LLM_CONFIG_SECRET_FIELDS)
unknown = sorted(unknown_values | unknown_secrets | unknown_clears)
if unknown:
raise ValueError(f"Unsupported config fields: {', '.join(unknown)}.")
conflicting_secrets = sorted(set(secrets_payload) & set(clear_secrets))
if conflicting_secrets:
raise ValueError(
"Secrets cannot be replaced and cleared together: "
f"{', '.join(conflicting_secrets)}."
)
with _config_file_lock:
config = load_config()
if revision != _llm_config_revision(config):
raise _ConfigRevisionConflict(
"config.yaml changed after it was loaded. Reload and try again."
)
changed_fields: list[str] = []
for field, raw_value in values.items():
value = _validated_llm_config_value(field, raw_value)
if getattr(config, field) != value:
setattr(config, field, value)
changed_fields.append(field)
for field, raw_value in secrets_payload.items():
if not isinstance(raw_value, str):
raise ValueError(f"{field} must be a string.")
value = raw_value.strip()
if len(value) > 8192:
raise ValueError(f"{field} must be at most 8192 characters.")
if value and getattr(config, field) != value:
setattr(config, field, value)
changed_fields.append(field)
for field in clear_secrets:
if getattr(config, field):
setattr(config, field, "")
changed_fields.append(field)
if catalog_supplied and config.model_catalog != normalized_catalog:
config.model_catalog = normalized_catalog
changed_fields.append("model_catalog")
if builtin_profiles_supplied:
current_document = load_provider_profiles()
current_by_id = {
profile.id: profile for profile in current_document.builtins
}
prepared: list[dict[str, Any]] = []
for index, raw_profile in enumerate(builtin_profiles_payload):
if not isinstance(raw_profile, dict):
raise ValueError(f"builtin_providers[{index}] must be an object.")
if raw_profile.get("managed") is False:
continue
candidate = dict(raw_profile)
provider_id = candidate.get("id")
if not isinstance(provider_id, str) or not provider_id.strip():
raise ValueError(f"builtin_providers[{index}].id is required.")
provider_id = provider_id.strip()
candidate["id"] = provider_id
candidate.setdefault("name", provider_id)
candidate["adapter"] = builtin_provider_adapter(provider_id)
candidate.pop("managed", None)
candidate.pop("api_key_configured", None)
candidate.pop("api_key_hint", None)
if (
not candidate.get("api_key")
and not candidate.get("clear_api_key")
and provider_id not in current_by_id
):
candidate["api_key"] = builtin_provider_legacy_api_key(
config, provider_id
)
prepared.append(candidate)
if not current_document.builtins:
prepared_by_id = {candidate["id"]: candidate for candidate in prepared}
try:
legacy_catalog = (
normalize_builtin_model_catalog(config.model_catalog) or []
)
except ValueError:
legacy_catalog = []
migration_ids = {entry["provider"] for entry in legacy_catalog}
if config.provider in builtin_provider_ids():
migration_ids.add(config.provider)
for provider_id in migration_ids:
if provider_id in prepared_by_id:
continue
effective = build_builtin_provider_profile(
config,
provider_id,
load_saved=False,
use_environment=False,
)
models = [
{
"id": entry["id"],
"name": entry["name"],
"model_id": entry["model_id"],
"enabled": entry["enabled"],
}
for entry in legacy_catalog
if entry["provider"] == provider_id
]
if not models and provider_id == config.provider:
known = next(
(
(name, model_id)
for name, model_id, candidate_provider in list_models_by_provider()
if candidate_provider == provider_id
and name == config.model
),
None,
)
models = [
{
"id": config.model,
"name": config.model,
"model_id": known[1] if known else config.model,
"enabled": True,
}
]
prepared.append(
{
"id": provider_id,
"name": provider_id,
"adapter": builtin_provider_adapter(provider_id),
"base_url": effective.base_url,
"api_key": builtin_provider_legacy_api_key(
config, provider_id
),
"auth_mode": effective.auth_mode,
"enabled": True,
"models": models,
}
)
prepared_entries = {
(model.get("id"), candidate.get("id"))
for candidate in prepared
if candidate.get("enabled", True)
for model in candidate.get("models", [])
if isinstance(model, dict) and model.get("enabled", True)
}
prepared_entries.update(
(model.id, profile.id)
for profile in current_document.providers
if profile.enabled
for model in profile.models
if model.enabled
)
if (config.model, config.provider) not in prepared_entries:
raise ValueError(
f"Default model {config.model!r} is not enabled for provider "
f"{config.provider!r}."
)
before = current_document.builtins
updated_document = replace_provider_profiles({"builtins": prepared})
if updated_document.builtins != before:
changed_fields.append("builtin_providers")
for profile in updated_document.builtins:
if profile.id == "anthropic" and (
config.anthropic_auth_mode != profile.auth_mode
):
config.anthropic_auth_mode = profile.auth_mode
changed_fields.append("anthropic_auth_mode")
if profile.id == "openai" and (
config.openai_auth_mode != profile.auth_mode
):
config.openai_auth_mode = profile.auth_mode
changed_fields.append("openai_auth_mode")
managed_builtin_entries = list_configured_builtin_model_entries()
if managed_builtin_entries or config.model_catalog is not None:
available = set(managed_builtin_entries)
if not managed_builtin_entries:
available.update(
list_builtin_model_catalog_entries(config.model_catalog) or []
)
available.update(list_configured_model_entries())
selected = (config.model, config.provider)
if not any(
name == selected[0] and provider == selected[1]
for name, _, provider in available
):
raise ValueError(
f"Default model {config.model!r} is not enabled for provider "
f"{config.provider!r}."
)
if any(field != "builtin_providers" for field in changed_fields):
save_config(config)
response = _llm_config_public(config)
response["changed_fields"] = sorted(set(changed_fields))
response["restart_required"] = any(
field not in _LLM_CONFIG_IMMEDIATE_FIELDS for field in changed_fields
)
return response
def _provider_admin_error(request: Request) -> JSONResponse | None:
expected = get_provider_admin_token()
if not expected:
return JSONResponse(
{"error": "Provider management is not enabled for this deployment."},
status_code=503,
)
supplied = request.headers.get("x-evoscientist-admin-token", "")
if not supplied or not secrets.compare_digest(supplied, expected):
return JSONResponse({"error": "Forbidden."}, status_code=403)
return None
async def provider_profiles_endpoint(request: Request) -> JSONResponse:
"""Read or replace runtime-editable provider profiles.
API keys are accepted on writes but never returned. Existing keys are kept
when the client sends an empty value, allowing the WebUI to edit metadata
without receiving the secret first.
"""
auth_error = _provider_admin_error(request)
if auth_error is not None:
return auth_error
try:
if request.method == "GET":
document = await asyncio.to_thread(load_provider_profiles)
else:
payload = await request.json()
document = await asyncio.to_thread(replace_provider_profiles, payload)
return JSONResponse(provider_profiles_public(document))
except ProviderProfileError as exc:
return JSONResponse({"error": str(exc)}, status_code=400)
except (json.JSONDecodeError, UnicodeDecodeError):
return JSONResponse(
{"error": "Request body must be valid JSON."}, status_code=400
)
except Exception:
_logger.exception("Provider profile request failed")
return JSONResponse(
{"error": "Provider profile request failed."}, status_code=500
)
async def llm_config_endpoint(request: Request) -> JSONResponse:
"""Read or patch the LLM-related fields persisted in config.yaml."""
auth_error = _provider_admin_error(request)
if auth_error is not None:
return auth_error
try:
if request.method == "GET":
response = await asyncio.to_thread(_read_llm_config)
response["changed_fields"] = []
response["restart_required"] = False
return JSONResponse(response)
payload = await request.json()
if request.method == "POST":
if not isinstance(payload, dict):
raise ValueError("Request body must be an object.")
action = payload.get("action")
provider_draft = payload.get("provider")
if not isinstance(provider_draft, dict):
raise ValueError("provider must be an object.")
provider_id = provider_draft.get("id")
if not isinstance(provider_id, str) or not provider_id.strip():
raise ValueError("provider.id is required.")
cfg = await asyncio.to_thread(get_effective_config)
base_url = provider_draft.get("base_url")
api_key = provider_draft.get("api_key")
if base_url is not None and not isinstance(base_url, str):
raise ValueError("provider.base_url must be a string.")
if api_key is not None and not isinstance(api_key, str):
raise ValueError("provider.api_key must be a string.")
profile = build_builtin_provider_profile(
cfg,
provider_id.strip(),
base_url_override=base_url,
api_key_override=api_key,
clear_api_key=provider_draft.get("clear_api_key") is True,
)
if action == "list_models":
models = await discover_provider_models(profile)
return JSONResponse(
{
"models": [
{"model_id": model.model_id, "name": model.name}
for model in models
]
}
)
if action == "test_model":
model = _provider_model_from_payload(payload.get("model"))
result = await test_provider_model(profile, model)
return JSONResponse(
{
"ok": True,
"latency_ms": result.latency_ms,
"response": result.response,
}
)
raise ValueError("action must be list_models or test_model.")
return JSONResponse(await asyncio.to_thread(_patch_llm_config, payload))
except _ConfigRevisionConflict as exc:
return JSONResponse({"error": str(exc)}, status_code=409)
except (json.JSONDecodeError, UnicodeDecodeError):
return JSONResponse(
{"error": "Request body must be valid JSON."}, status_code=400
)
except ValueError as exc:
return JSONResponse({"error": str(exc)}, status_code=400)
except ProviderOperationError as exc:
return JSONResponse({"error": str(exc)}, status_code=502)
except Exception:
_logger.exception("LLM config request failed")
return JSONResponse({"error": "LLM config request failed."}, status_code=500)
def _persist_default_model(model: str, provider: str) -> None:
"""Update the persisted default pair without disturbing other settings."""
with _config_file_lock:
config = load_config()
config.model = model
config.provider = provider
save_config(config)
async def default_model_endpoint(request: Request) -> JSONResponse:
"""Validate and persist the deployment's default model selection."""
auth_error = _provider_admin_error(request)
if auth_error is not None:
return auth_error
try:
payload = await request.json()
if not isinstance(payload, dict):
raise ValueError("Request body must be an object.")
model = payload.get("model")
provider = payload.get("provider")
if not isinstance(model, str) or not model.strip():
raise ValueError("model is required.")
if not isinstance(provider, str) or not provider.strip():
raise ValueError("provider is required.")
model = model.strip()
provider = provider.strip()
if len(model) > 300:
raise ValueError("model must be at most 300 characters.")
if len(provider) > 64:
raise ValueError("provider must be at most 64 characters.")
cfg = await asyncio.to_thread(get_effective_config)
entries = await list_model_picker_entries(
getattr(cfg, "ollama_base_url", None),
include_custom_ollama=False,
model_catalog=getattr(cfg, "model_catalog", None),
)
if not any(
name == model and entry_provider == provider
for name, _, entry_provider in entries
):
raise ValueError(
f"Model {model!r} is not configured for provider {provider!r}."
)
await asyncio.to_thread(_persist_default_model, model, provider)
return JSONResponse({"default": {"name": model, "provider": provider}})
except (json.JSONDecodeError, UnicodeDecodeError):
return JSONResponse(
{"error": "Request body must be valid JSON."}, status_code=400
)
except (ProviderProfileError, ValueError) as exc:
return JSONResponse({"error": str(exc)}, status_code=400)
except Exception:
_logger.exception("Default model request failed")
return JSONResponse({"error": "Default model request failed."}, status_code=500)
def _provider_model_from_payload(raw: Any) -> ProviderModel:
if not isinstance(raw, dict):
raise ProviderProfileError("model must be an object.")
model_id = raw.get("id")
name = raw.get("name")
upstream_id = raw.get("model_id")
if not isinstance(model_id, str) or not model_id.strip():
raise ProviderProfileError("model.id is required.")
if not isinstance(name, str) or not name.strip():
raise ProviderProfileError("model.name is required.")
if not isinstance(upstream_id, str) or not upstream_id.strip():
raise ProviderProfileError("model.model_id is required.")
if len(model_id.strip()) > 64:
raise ProviderProfileError("model.id must be at most 64 characters.")
if len(name.strip()) > 120:
raise ProviderProfileError("model.name must be at most 120 characters.")
if len(upstream_id.strip()) > 300:
raise ProviderProfileError("model.model_id must be at most 300 characters.")
return ProviderModel(
id=model_id.strip(),
name=name.strip(),
model_id=upstream_id.strip(),
enabled=True,
)
async def provider_actions_endpoint(request: Request) -> JSONResponse:
"""Discover provider models or run a real connectivity test."""
auth_error = _provider_admin_error(request)
if auth_error is not None:
return auth_error
try:
payload = await request.json()
if not isinstance(payload, dict):
raise ProviderProfileError("Request body must be an object.")
action = payload.get("action")
profile = await asyncio.to_thread(
resolve_provider_profile_draft, payload.get("provider")
)
if action == "list_models":
models = await discover_provider_models(profile)
return JSONResponse(
{
"models": [
{"model_id": model.model_id, "name": model.name}
for model in models
]
}
)
if action == "test_model":
model = _provider_model_from_payload(payload.get("model"))
result = await test_provider_model(profile, model)
return JSONResponse(
{
"ok": True,
"latency_ms": result.latency_ms,
"response": result.response,
}
)
raise ProviderProfileError("action must be list_models or test_model.")
except (json.JSONDecodeError, UnicodeDecodeError):
return JSONResponse(
{"error": "Request body must be valid JSON."}, status_code=400
)
except ProviderProfileError as exc:
return JSONResponse({"error": str(exc)}, status_code=400)
except ProviderOperationError as exc:
return JSONResponse({"error": str(exc)}, status_code=502)
except Exception:
_logger.exception("Provider action request failed")
return JSONResponse(
{"error": "Provider action request failed."}, status_code=500
)
def _message_type(message: Any) -> str | None:
@@ -1573,33 +815,66 @@ async def bind_workspace_turn(request: Request) -> JSONResponse:
return JSONResponse(_run_reservation_payload(run))
def _refuse_startup_on_legacy_artifacts() -> None:
"""Config-service startup gate (design doc section 10 step 4).
Raising here aborts the Starlette lifespan startup, so the config
service refuses to serve until the legacy artifacts are reset.
"""
assert_no_legacy_artifacts()
_SNAPSHOT_CLEANUP_INTERVAL_SECONDS = 60
async def _snapshot_cleanup_loop(stop: asyncio.Event) -> None:
"""Expire prepared snapshots past TTL and bound snapshots past retention.
The TTL/retention rules (design doc 5.2/8.2) only take effect if a
cleanup task transitions rows to ``expired``; without one, stale
prepared rows would hold their UNIQUE slot forever.
"""
while not stop.is_set():
try:
await asyncio.wait_for(stop.wait(), timeout=_SNAPSHOT_CLEANUP_INTERVAL_SECONDS)
except asyncio.TimeoutError:
pass
if stop.is_set():
return
try:
from EvoScientist.model_registry.runtime import get_snapshot_runtime
expired = await asyncio.to_thread(
get_snapshot_runtime().snapshots.cleanup_expired
)
if expired:
_logger.info("Expired %d run snapshot(s): %s", len(expired), expired)
except Exception:
_logger.exception("Run snapshot cleanup pass failed")
@contextlib.asynccontextmanager
async def _lifespan(_app: Starlette):
_refuse_startup_on_legacy_artifacts()
stop = asyncio.Event()
cleanup_task = asyncio.create_task(_snapshot_cleanup_loop(stop))
try:
yield
finally:
stop.set()
await cleanup_task
app = Starlette(
lifespan=_lifespan,
routes=[
# The unified model registry API (design doc section 9) owns
# /api/models and /api/runtime-snapshots; the legacy handlers for
# those paths were removed here and the remaining legacy provider
# routes are deleted in Task 7.
# /api/models and /api/runtime-snapshots. The legacy provider
# routes (/api/provider-profiles, /api/provider-actions,
# /api/config, /api/default-model) and the admin-token check were
# removed with the unified model configuration refactor.
*model_registry_routes(),
Route(
"/api/provider-profiles",
provider_profiles_endpoint,
methods=["GET", "PUT"],
),
Route(
"/api/config",
llm_config_endpoint,
methods=["GET", "PATCH", "POST"],
),
Route(
"/api/default-model",
default_model_endpoint,
methods=["PUT"],
),
Route(
"/api/provider-actions",
provider_actions_endpoint,
methods=["POST"],
),
Route(
"/api/threads/{thread_id}/final-answer",
get_final_answer,
+14 -16
View File
@@ -1,33 +1,31 @@
"""LLM module for EvoScientist.
Provides a unified interface for creating chat model instances
with support for multiple providers.
The static model catalog and free-string model factory were removed in the
unified model-configuration refactor: all chat-model construction now goes
through :mod:`EvoScientist.model_registry` (SnapshotRuntime +
``build_chat_model`` + ``ResolvedModelConfig``).
``models`` is attached lazily via :mod:`lazy_loader` (SPEC-1 / PEP 562) so
that importing ``EvoScientist.llm`` (or any of its submodules, like
``context_window``) does not eagerly drag in ``langchain.chat_models`` and
its transitive ``langchain_anthropic``/``langchain_openai`` stack — that's
roughly 1 s of wall time on every CLI invocation.
What remains here:
- ``context_window`` — context-window resolution helpers consumed by the
middleware layer (e.g. ``context_editing``);
- ``patches`` — LangChain provider monkey-patches, attached lazily via
:mod:`lazy_loader` (SPEC-1 / PEP 562) so that importing
``EvoScientist.llm`` does not eagerly drag in ``langchain.chat_models``
and its transitive ``langchain_anthropic``/``langchain_openai`` stack —
that's roughly 1 s of wall time on every CLI invocation.
"""
import lazy_loader as _lazy
__getattr__, __dir__, __all__ = _lazy.attach(
__name__,
submodules=["context_window", "models", "patches"],
submodules=["context_window", "patches"],
submod_attrs={
"context_window": [
"DEFAULT_CONTEXT_WINDOW_FALLBACK",
"get_context_window",
"resolve_context_window",
],
"models": [
"DEFAULT_MODEL",
"MODELS",
"get_chat_model",
"get_model_info",
"get_models_for_provider",
"list_models",
],
},
)
File diff suppressed because it is too large Load Diff
+1 -1
View File
@@ -1,7 +1,7 @@
"""Ollama server probing — shared by onboard wizard and /model picker.
Ollama models are whatever the user has ``ollama pull``ed locally; they
cannot be enumerated in ``_MODEL_ENTRIES``. Both the setup wizard and the
cannot be enumerated in a static model catalog. Both the setup wizard and the
interactive model picker need to hit ``GET {base_url}/api/tags`` to see
what is actually installed.
+24 -1
View File
@@ -882,7 +882,9 @@ def _patch_deepseek_reasoning_passback(model: Any) -> None:
# resolves its chat model per run from ``configurable.runtime_snapshot_id``
# via ``ConfigurableModelMiddleware`` (design doc 8.2/8.3). Injecting
# ``model``/``model_provider`` here would be rejected with
# ``MODEL_CONFIG_OUTSIDE_SNAPSHOT``.
# ``MODEL_CONFIG_OUTSIDE_SNAPSHOT``. Instead, a fresh local run snapshot
# bound to the child thread is created here (section 8.1: sub-agents are a
# local entry point sharing the same SnapshotService and snapshot table).
#
# Mechanism: wrap deepagents' ``_build_start_tool`` and ``_build_update_tool``
# factories. Each wrapped factory calls the original with a proxied client
@@ -986,6 +988,27 @@ def _merge_runs_config_kwargs(kwargs: dict) -> dict:
if not isinstance(existing_configurable, dict):
existing_configurable = {}
merged_configurable = {**existing_configurable, **inherited_scope}
# Async sub-agent snapshot entry (design doc 8.1): the deployed child
# graph runs on its own thread, so it cannot reuse the parent's
# snapshot binding. Freeze a fresh local snapshot bound to the child
# thread; the child's ConfigurableModelMiddleware resolves it per call.
# A bootstrap registry raises MODEL_REGISTRY_NOT_READY here, surfacing
# as a structured start/update_async_task tool error on the parent.
child_thread_id = kwargs.get("thread_id")
if (
isinstance(child_thread_id, str)
and child_thread_id
and "runtime_snapshot_id" not in merged_configurable
):
from EvoScientist.model_registry.runtime import get_snapshot_runtime
merged_configurable["runtime_snapshot_id"] = (
get_snapshot_runtime()
.create_local_snapshot(child_thread_id)
.snapshot_id
)
kwargs = dict(kwargs)
if merged_configurable or "config" in kwargs:
kwargs["config"] = {**existing, "configurable": merged_configurable}
-519
View File
@@ -1,519 +0,0 @@
"""Provider model discovery and lightweight connectivity tests."""
from __future__ import annotations
import asyncio
import os
import time
from dataclasses import dataclass
from typing import Any
import httpx
from ..config.provider_profiles import (
ProviderModel,
ProviderProfile,
get_builtin_provider_profile,
normalize_provider_base_url,
)
from ..config.settings import is_config_applied_env
_DEFAULT_BASE_URLS = {
"openai": "https://api.openai.com/v1",
"anthropic": "https://api.anthropic.com",
"google-genai": "https://generativelanguage.googleapis.com/v1beta",
"grok": "https://api.x.ai/v1",
"openrouter": "https://openrouter.ai/api/v1",
"nvidia": "https://integrate.api.nvidia.com/v1",
"ollama": "http://127.0.0.1:11434",
}
_API_KEY_REQUIRED_ADAPTERS = {
"openai",
"anthropic",
"google-genai",
"grok",
"openrouter",
"nvidia",
}
_MAX_DISCOVERED_MODELS = 2_000
_MAX_PAGES = 100
_MAX_REQUEST_ATTEMPTS = 2
@dataclass(frozen=True)
class BuiltinProviderConnection:
adapter: str
api_key_field: str | None
base_url_field: str | None = None
default_base_url: str = ""
_BUILTIN_PROVIDER_CONNECTIONS = {
"anthropic": BuiltinProviderConnection(
"anthropic", "anthropic_api_key", "anthropic_base_url"
),
"openai": BuiltinProviderConnection("openai", "openai_api_key"),
"google-genai": BuiltinProviderConnection("google-genai", "google_api_key"),
"minimax": BuiltinProviderConnection(
"anthropic-compatible",
"minimax_api_key",
"minimax_base_url",
"https://api.minimaxi.com/anthropic",
),
"nvidia": BuiltinProviderConnection("nvidia", "nvidia_api_key"),
"siliconflow": BuiltinProviderConnection(
"openai-compatible",
"siliconflow_api_key",
default_base_url="https://api.siliconflow.cn/v1",
),
"openrouter": BuiltinProviderConnection("openrouter", "openrouter_api_key"),
"zhipu": BuiltinProviderConnection(
"openai-compatible",
"zhipu_api_key",
default_base_url="https://open.bigmodel.cn/api/paas/v4",
),
"zhipu-code": BuiltinProviderConnection(
"openai-compatible",
"zhipu_api_key",
default_base_url="https://open.bigmodel.cn/api/coding/paas/v4",
),
"volcengine": BuiltinProviderConnection(
"openai-compatible",
"volcengine_api_key",
default_base_url="https://ark.cn-beijing.volces.com/api/v3",
),
"dashscope": BuiltinProviderConnection(
"openai-compatible",
"dashscope_api_key",
default_base_url="https://dashscope.aliyuncs.com/compatible-mode/v1",
),
"dashscope-code": BuiltinProviderConnection(
"openai-compatible",
"dashscope_api_key",
default_base_url="https://coding.dashscope.aliyuncs.com/v1",
),
"deepseek": BuiltinProviderConnection(
"openai-compatible",
"deepseek_api_key",
default_base_url="https://api.deepseek.com",
),
"moonshot": BuiltinProviderConnection(
"openai-compatible",
"moonshot_api_key",
default_base_url="https://api.moonshot.cn/v1",
),
"kimi-coding": BuiltinProviderConnection(
"anthropic-compatible",
"kimi_api_key",
default_base_url="https://api.kimi.com/coding/",
),
"custom-openai": BuiltinProviderConnection(
"openai-compatible",
"custom_openai_api_key",
"custom_openai_base_url",
),
"custom-anthropic": BuiltinProviderConnection(
"anthropic-compatible",
"custom_anthropic_api_key",
"custom_anthropic_base_url",
),
"ollama": BuiltinProviderConnection("ollama", None, "ollama_base_url"),
}
_BUILTIN_API_KEY_ENVS = {
"anthropic": "ANTHROPIC_API_KEY",
"openai": "OPENAI_API_KEY",
"google-genai": "GOOGLE_API_KEY",
"minimax": "MINIMAX_API_KEY",
"nvidia": "NVIDIA_API_KEY",
"siliconflow": "SILICONFLOW_API_KEY",
"openrouter": "OPENROUTER_API_KEY",
"zhipu": "ZHIPU_API_KEY",
"zhipu-code": "ZHIPU_API_KEY",
"volcengine": "VOLCENGINE_API_KEY",
"dashscope": "DASHSCOPE_API_KEY",
"dashscope-code": "DASHSCOPE_API_KEY",
"deepseek": "DEEPSEEK_API_KEY",
"moonshot": "MOONSHOT_API_KEY",
"kimi-coding": "KIMI_API_KEY",
"custom-openai": "CUSTOM_OPENAI_API_KEY",
"custom-anthropic": "CUSTOM_ANTHROPIC_API_KEY",
}
_BUILTIN_BASE_URL_ENVS = {
"anthropic": "ANTHROPIC_BASE_URL",
"openai": "OPENAI_BASE_URL",
"minimax": "MINIMAX_BASE_URL",
"custom-openai": "CUSTOM_OPENAI_BASE_URL",
"custom-anthropic": "CUSTOM_ANTHROPIC_BASE_URL",
"ollama": "OLLAMA_BASE_URL",
}
class ProviderOperationError(RuntimeError):
"""Raised when provider discovery or testing cannot complete."""
def build_builtin_provider_profile(
config: Any,
provider_id: str,
*,
saved_profile: ProviderProfile | None = None,
load_saved: bool = True,
use_environment: bool = True,
base_url_override: str | None = None,
api_key_override: str | None = None,
clear_api_key: bool = False,
) -> ProviderProfile:
"""Resolve a built-in profile with env > registry > legacy config priority."""
spec = _BUILTIN_PROVIDER_CONNECTIONS.get(provider_id)
if spec is None:
raise ProviderOperationError(f"Unknown built-in provider {provider_id!r}.")
saved = get_builtin_provider_profile(provider_id) if load_saved else saved_profile
base_url = saved.base_url if saved is not None else spec.default_base_url
if saved is None and spec.base_url_field:
configured_base_url = getattr(config, spec.base_url_field, "")
if isinstance(configured_base_url, str) and configured_base_url.strip():
base_url = configured_base_url.strip()
base_url_env = _BUILTIN_BASE_URL_ENVS.get(provider_id)
if (
base_url_env
and use_environment
and os.environ.get(base_url_env, "").strip()
and (saved is None or not is_config_applied_env(base_url_env))
):
base_url = os.environ[base_url_env].strip()
if base_url_override is not None:
base_url = base_url_override.strip() or spec.default_base_url
api_key = saved.api_key if saved is not None else ""
if saved is None and spec.api_key_field:
configured_api_key = getattr(config, spec.api_key_field, "")
if isinstance(configured_api_key, str):
api_key = configured_api_key.strip()
api_key_env = _BUILTIN_API_KEY_ENVS.get(provider_id)
if (
api_key_env
and use_environment
and os.environ.get(api_key_env, "").strip()
and (saved is None or not is_config_applied_env(api_key_env))
):
api_key = os.environ[api_key_env].strip()
if clear_api_key:
api_key = ""
elif api_key_override is not None and api_key_override.strip():
api_key = api_key_override.strip()
return ProviderProfile(
id=provider_id,
name=provider_id,
adapter=spec.adapter,
base_url=base_url,
api_key=api_key,
enabled=saved.enabled if saved is not None else True,
models=saved.models if saved is not None else (),
auth_mode=(
saved.auth_mode
if saved is not None
else getattr(config, f"{provider_id}_auth_mode", "api_key")
if provider_id in {"anthropic", "openai"}
else "api_key"
),
)
def builtin_provider_ids() -> tuple[str, ...]:
return tuple(_BUILTIN_PROVIDER_CONNECTIONS)
def builtin_provider_adapter(provider_id: str) -> str:
spec = _BUILTIN_PROVIDER_CONNECTIONS.get(provider_id)
if spec is None:
raise ProviderOperationError(f"Unknown built-in provider {provider_id!r}.")
return spec.adapter
def builtin_provider_legacy_api_key(config: Any, provider_id: str) -> str:
"""Return only the config.yaml key, excluding environment overrides."""
spec = _BUILTIN_PROVIDER_CONNECTIONS.get(provider_id)
if spec is None or spec.api_key_field is None:
return ""
value = getattr(config, spec.api_key_field, "")
return value.strip() if isinstance(value, str) else ""
@dataclass(frozen=True)
class DiscoveredProviderModel:
model_id: str
name: str
@dataclass(frozen=True)
class ProviderModelTestResult:
latency_ms: int
response: str
def _base_url(profile: ProviderProfile) -> str:
base_url = profile.base_url or _DEFAULT_BASE_URLS.get(profile.adapter, "")
if not base_url:
raise ProviderOperationError(
f"Base URL is required for adapter {profile.adapter}."
)
return normalize_provider_base_url(profile.adapter, base_url)
def _models_url(profile: ProviderProfile) -> str:
base_url = _base_url(profile)
if profile.adapter in {"anthropic", "anthropic-compatible"}:
return (
f"{base_url}/models"
if base_url.endswith(("/v1", "/v1beta"))
else f"{base_url}/v1/models"
)
if profile.adapter == "ollama":
return f"{base_url}/api/tags"
return f"{base_url}/models"
def _headers(profile: ProviderProfile) -> dict[str, str]:
if profile.adapter in _API_KEY_REQUIRED_ADAPTERS and not profile.api_key:
raise ProviderOperationError(
f"API key is required for adapter {profile.adapter}."
)
if profile.adapter in {"anthropic", "anthropic-compatible"}:
headers = {"anthropic-version": "2023-06-01"}
if profile.api_key:
headers["x-api-key"] = profile.api_key
return headers
if profile.adapter == "google-genai":
return {"x-goog-api-key": profile.api_key}
if profile.adapter == "ollama" or not profile.api_key:
return {}
return {"Authorization": f"Bearer {profile.api_key}"}
def _provider_error_message(response: httpx.Response) -> str:
message = ""
try:
payload = response.json()
if isinstance(payload, dict):
error = payload.get("error")
if isinstance(error, dict):
raw_message = error.get("message")
if isinstance(raw_message, str):
message = raw_message.strip()
elif isinstance(error, str):
message = error.strip()
if not message:
raw_message = payload.get("message")
if isinstance(raw_message, str):
message = raw_message.strip()
except (ValueError, TypeError):
pass
suffix = f": {message[:500]}" if message else ""
return f"Provider returned HTTP {response.status_code}{suffix}"
def _model_name(raw: dict[str, Any], model_id: str) -> str:
for key in ("display_name", "displayName", "name"):
value = raw.get(key)
if isinstance(value, str) and value.strip():
return value.strip()[:120]
return model_id[:120]
def _parse_models(
profile: ProviderProfile, payload: Any
) -> list[DiscoveredProviderModel]:
if not isinstance(payload, dict):
raise ProviderOperationError("Provider returned an invalid model list.")
records = (
payload.get("models") if profile.adapter == "ollama" else payload.get("data")
)
if profile.adapter == "google-genai":
records = payload.get("models")
elif not isinstance(records, list):
records = payload.get("models")
if not isinstance(records, list):
raise ProviderOperationError("Provider response does not contain a model list.")
result: list[DiscoveredProviderModel] = []
for raw in records:
if not isinstance(raw, dict):
continue
model_id_raw = (
(raw.get("model") or raw.get("name"))
if profile.adapter == "ollama"
else raw.get("id")
)
if profile.adapter == "google-genai":
model_id_raw = raw.get("name")
methods = raw.get("supportedGenerationMethods")
if isinstance(methods, list) and "generateContent" not in methods:
continue
if not isinstance(model_id_raw, str) or not model_id_raw.strip():
continue
model_id = model_id_raw.strip()
if profile.adapter == "google-genai":
model_id = model_id.removeprefix("models/")
result.append(
DiscoveredProviderModel(
model_id=model_id[:300],
name=_model_name(raw, model_id),
)
)
return result
async def discover_provider_models(
profile: ProviderProfile,
*,
transport: httpx.AsyncBaseTransport | None = None,
) -> list[DiscoveredProviderModel]:
"""Fetch every model exposed by a provider's model-list endpoint."""
url = _models_url(profile)
headers = _headers(profile)
params: dict[str, str | int] = {}
discovered: list[DiscoveredProviderModel] = []
seen: set[str] = set()
timeout = httpx.Timeout(30.0, connect=10.0)
async with httpx.AsyncClient(timeout=timeout, transport=transport) as client:
for _ in range(_MAX_PAGES):
response: httpx.Response | None = None
last_request_error: httpx.HTTPError | None = None
for attempt in range(_MAX_REQUEST_ATTEMPTS):
try:
response = await client.get(url, headers=headers, params=params)
break
except (httpx.TimeoutException, httpx.NetworkError) as exc:
last_request_error = exc
if attempt + 1 < _MAX_REQUEST_ATTEMPTS:
await asyncio.sleep(0.25 * (attempt + 1))
if response is None:
if isinstance(last_request_error, httpx.TimeoutException):
raise ProviderOperationError(
f"Provider model discovery timed out after "
f"{_MAX_REQUEST_ATTEMPTS} attempts."
) from last_request_error
error_name = (
last_request_error.__class__.__name__
if last_request_error is not None
else "NetworkError"
)
raise ProviderOperationError(
f"Could not connect to provider after "
f"{_MAX_REQUEST_ATTEMPTS} attempts: {error_name}."
) from last_request_error
if response.status_code >= 400:
raise ProviderOperationError(_provider_error_message(response))
try:
payload = response.json()
except ValueError as exc:
content_type = (
response.headers.get("content-type", "unknown")
.split(";", 1)[0]
.strip()
or "unknown"
)
hint = (
" OpenAI-compatible Base URLs normally end in /v1."
if profile.adapter
in {
"openai",
"openai-compatible",
"grok",
"openrouter",
"nvidia",
"antigravity",
}
else ""
)
raise ProviderOperationError(
f"Provider returned a non-JSON model list from {url} "
f"(Content-Type: {content_type}).{hint}"
) from exc
for model in _parse_models(profile, payload):
if model.model_id in seen:
continue
seen.add(model.model_id)
discovered.append(model)
if len(discovered) >= _MAX_DISCOVERED_MODELS:
return sorted(discovered, key=lambda item: item.model_id.lower())
if profile.adapter in {"anthropic", "anthropic-compatible"}:
if not isinstance(payload, dict) or payload.get("has_more") is not True:
break
last_id = payload.get("last_id")
if not isinstance(last_id, str) or not last_id:
break
params = {"after_id": last_id, "limit": 100}
continue
if profile.adapter == "google-genai":
next_page = (
payload.get("nextPageToken") if isinstance(payload, dict) else None
)
if not isinstance(next_page, str) or not next_page:
break
params = {"pageToken": next_page, "pageSize": 1000}
continue
break
return sorted(discovered, key=lambda item: item.model_id.lower())
def _response_text(response: Any) -> str:
content = getattr(response, "content", response)
if isinstance(content, str):
return content.strip()
if isinstance(content, list):
parts: list[str] = []
for block in content:
if isinstance(block, str):
parts.append(block)
elif isinstance(block, dict):
text = block.get("text") or block.get("content")
if isinstance(text, str):
parts.append(text)
return "\n".join(parts).strip()
return str(content).strip()
def _safe_test_error(exc: Exception, profile: ProviderProfile) -> str:
message = str(exc).strip() or exc.__class__.__name__
if profile.api_key:
message = message.replace(profile.api_key, "[redacted]")
return message[:800]
async def test_provider_model(
profile: ProviderProfile,
model: ProviderModel,
) -> ProviderModelTestResult:
"""Build the configured model and run a minimal real inference request."""
from .models import get_profile_chat_model
started = time.perf_counter()
try:
chat_model = await asyncio.to_thread(
get_profile_chat_model,
profile,
model,
max_tokens=64,
skip_auto_config=True,
)
response = await asyncio.wait_for(
chat_model.ainvoke("Reply with exactly: OK"), timeout=45.0
)
except TimeoutError as exc:
raise ProviderOperationError("Model test timed out after 45 seconds.") from exc
except Exception as exc:
raise ProviderOperationError(_safe_test_error(exc, profile)) from exc
latency_ms = max(1, round((time.perf_counter() - started) * 1000))
return ProviderModelTestResult(
latency_ms=latency_ms,
response=_response_text(response)[:500],
)
-401
View File
@@ -1,401 +0,0 @@
"""Server-side, per-run snapshots for custom provider model configuration.
The browser and LangGraph run configuration carry only a random snapshot ID.
The SQLite record stores non-secret connection metadata and runtime limits.
Credentials stay in a process-local cache for the lifetime of a development
deployment, so a restarted deployment fails an old run explicitly instead of
silently using a newly edited credential.
"""
from __future__ import annotations
import json
import sqlite3
import threading
import time
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Any
from ..config.provider_profiles import (
ModelRuntime,
ProviderModel,
ProviderProfile,
ProviderProfileError,
ProviderRuntime,
get_provider_profile_revision,
resolve_model_input_limit,
resolve_provider_model,
)
from ..config.settings import get_config_dir
from .models import (
ResolvedRuntimeOptions,
is_static_provider_id,
normalize_provider_id,
resolve_runtime_options,
)
_SNAPSHOT_TTL_SECONDS = 7 * 24 * 60 * 60
_MAX_SNAPSHOT_ID_LENGTH = 128
_DB_LOCK = threading.RLock()
_MODEL_CACHE_LOCK = threading.RLock()
_MODEL_CACHE: dict[str, Any] = {}
_SNAPSHOT_SECRETS: dict[str, str] = {}
def _validate_static_provider_credentials(provider: str) -> None:
"""Reject known static selections that lack their provider-specific key.
Runtime snapshots are created by the WebUI before a run is queued. This is
the latest point at which we can return a useful client error without
making the graph fail after the user has already submitted a message.
"""
if provider not in {"zhipu", "zhipu-code"}:
return
from ..config.settings import get_effective_config
from .provider_operations import build_builtin_provider_profile
config = get_effective_config()
try:
profile = build_builtin_provider_profile(config, provider)
except ProviderProfileError as exc:
# A stale custom registry must not hide configuration supplied through
# config.yaml or the environment for a built-in provider.
if not str(exc).startswith("PROVIDER_PROFILE_RESET_REQUIRED"):
raise
profile = build_builtin_provider_profile(
config, provider, load_saved=False
)
if not profile.api_key.strip():
raise ProviderProfileError(
"ZHIPU_API_KEY_NOT_CONFIGURED: GLM requires a valid Zhipu API key. "
"Configure ZHIPU_API_KEY before starting a conversation."
)
@dataclass(frozen=True)
class RunRuntimeSnapshot:
"""An immutable server-side configuration captured before one run starts."""
snapshot_id: str
created_at: int
expires_at: int
profile_revision: str | None
profile: ProviderProfile
model: ProviderModel
options: ResolvedRuntimeOptions
def public_payload(self) -> dict[str, Any]:
"""Return diagnostic fields that are safe to attach to a run."""
return {
"snapshot_id": self.snapshot_id,
"provider": self.profile.id,
"model": self.model.id,
"profile_revision": self.profile_revision,
"created_at": self.created_at,
"expires_at": self.expires_at,
"runtime": {
"max_input_tokens": self.options.resolved_input_limit,
"max_output_tokens": self.options.max_output_tokens,
"min_effective_input_tokens": self.options.min_effective_input_tokens,
"timeout_seconds": self.options.timeout_seconds,
"max_retries": self.options.max_retries,
"temperature": self.options.temperature,
"top_p": self.options.top_p,
"reasoning_effort": self.options.reasoning_effort,
"limits_source": self.options.limits_source,
"capabilities": dict(self.options.capabilities),
},
}
def _database_path() -> Path:
return get_config_dir() / "run-runtime-snapshots.sqlite3"
def _connect() -> sqlite3.Connection:
path = _database_path()
path.parent.mkdir(parents=True, exist_ok=True)
try:
path.parent.chmod(0o700)
except OSError:
pass
connection = sqlite3.connect(path)
connection.execute(
"""
CREATE TABLE IF NOT EXISTS run_runtime_snapshots (
snapshot_id TEXT PRIMARY KEY,
expires_at INTEGER NOT NULL,
payload_json TEXT NOT NULL
)
"""
)
try:
path.chmod(0o600)
except OSError:
pass
return connection
def _validate_snapshot_id(snapshot_id: str) -> str:
normalized = snapshot_id.strip()
if not normalized or len(normalized) > _MAX_SNAPSHOT_ID_LENGTH:
raise ProviderProfileError("runtime snapshot ID is invalid.")
if any(ord(character) < 33 or ord(character) > 126 for character in normalized):
raise ProviderProfileError("runtime snapshot ID is invalid.")
return normalized
def _provider_runtime_payload(runtime: ProviderRuntime) -> dict[str, Any]:
return {
"timeout_seconds": runtime.timeout_seconds,
"max_retries": runtime.max_retries,
"default_temperature": runtime.default_temperature,
"default_top_p": runtime.default_top_p,
"default_reasoning_effort": runtime.default_reasoning_effort,
}
def _model_runtime_payload(runtime: ModelRuntime) -> dict[str, Any]:
return {
"limit_mode": runtime.limit_mode,
"context_window_tokens": runtime.context_window_tokens,
"max_input_tokens": runtime.max_input_tokens,
"max_output_tokens": runtime.max_output_tokens,
"min_effective_input_tokens": runtime.min_effective_input_tokens,
"limits_status": runtime.limits_status,
"limits_source": runtime.limits_source,
"temperature": runtime.temperature,
"top_p": runtime.top_p,
"reasoning_effort": runtime.reasoning_effort,
"capabilities": dict(runtime.capabilities),
}
def _profile_payload(profile: ProviderProfile, model: ProviderModel) -> dict[str, Any]:
return {
"id": profile.id,
"name": profile.name,
"adapter": profile.adapter,
"base_url": profile.base_url,
# Credentials stay in the process-local secret cache. The persisted
# snapshot remains safe to inspect through diagnostics and never
# duplicates a Provider Profile API key.
"api_key": "",
"auth_mode": profile.auth_mode,
"enabled": profile.enabled,
"runtime": _provider_runtime_payload(profile.runtime),
"models": [
{
"id": model.id,
"name": model.name,
"model_id": model.model_id,
"enabled": model.enabled,
"runtime": _model_runtime_payload(model.runtime),
}
],
}
def _parse_snapshot_payload(
payload: dict[str, Any], *, require_secret: bool = True
) -> RunRuntimeSnapshot:
try:
profile_raw = payload["profile"]
if not isinstance(profile_raw, dict):
raise ProviderProfileError("runtime snapshot profile is invalid.")
runtime_raw = payload["runtime"]
if not isinstance(runtime_raw, dict):
raise ProviderProfileError("runtime snapshot options are invalid.")
# The source is a private snapshot, not a browser draft. Reuse the
# validated profile parser without looking up the current registry.
from ..config.provider_profiles import _parse_profile
profile = _parse_profile(profile_raw, index=0)
model = profile.models[0]
resolved_input_limit = int(runtime_raw["resolved_input_limit"])
if resolved_input_limit != resolve_model_input_limit(model.runtime):
raise ProviderProfileError("runtime snapshot limits are invalid.")
capabilities_raw = dict(runtime_raw["capabilities"])
options = ResolvedRuntimeOptions(
profile_revision=(
payload["profile_revision"]
if isinstance(payload.get("profile_revision"), str)
else None
),
model_id=str(runtime_raw["model_id"]),
adapter_id=str(runtime_raw["adapter_id"]),
limit_mode=str(runtime_raw["limit_mode"]),
resolved_input_limit=resolved_input_limit,
max_output_tokens=int(runtime_raw["max_output_tokens"]),
min_effective_input_tokens=int(
runtime_raw["min_effective_input_tokens"]
),
timeout_seconds=int(runtime_raw["timeout_seconds"]),
max_retries=int(runtime_raw["max_retries"]),
temperature=runtime_raw.get("temperature"),
top_p=runtime_raw.get("top_p"),
reasoning_effort=str(runtime_raw["reasoning_effort"]),
limits_source=str(runtime_raw["limits_source"]),
capabilities=tuple(
(key, capabilities_raw[key])
for key in ("tools", "vision", "structured_output")
if capabilities_raw.get(key) in {True, False, "auto"}
),
)
snapshot_id = _validate_snapshot_id(str(payload["snapshot_id"]))
secret = _SNAPSHOT_SECRETS.get(snapshot_id)
if require_secret and snapshot_id not in _SNAPSHOT_SECRETS:
raise ProviderProfileError(
"RUN_RUNTIME_SNAPSHOT_CREDENTIAL_UNAVAILABLE: the deployment was "
"restarted after this run began. Start the message again."
)
return RunRuntimeSnapshot(
snapshot_id=snapshot_id,
created_at=int(payload["created_at"]),
expires_at=int(payload["expires_at"]),
profile_revision=options.profile_revision,
profile=replace(profile, api_key=secret or ""),
model=model,
options=options,
)
except (KeyError, TypeError, ValueError) as exc:
raise ProviderProfileError("runtime snapshot is invalid.") from exc
def _snapshot_payload(snapshot: RunRuntimeSnapshot) -> dict[str, Any]:
return {
"snapshot_id": snapshot.snapshot_id,
"created_at": snapshot.created_at,
"expires_at": snapshot.expires_at,
"profile_revision": snapshot.profile_revision,
"profile": _profile_payload(snapshot.profile, snapshot.model),
"runtime": {
"model_id": snapshot.options.model_id,
"adapter_id": snapshot.options.adapter_id,
"limit_mode": snapshot.options.limit_mode,
"resolved_input_limit": snapshot.options.resolved_input_limit,
"max_output_tokens": snapshot.options.max_output_tokens,
"min_effective_input_tokens": snapshot.options.min_effective_input_tokens,
"timeout_seconds": snapshot.options.timeout_seconds,
"max_retries": snapshot.options.max_retries,
"temperature": snapshot.options.temperature,
"top_p": snapshot.options.top_p,
"reasoning_effort": snapshot.options.reasoning_effort,
"limits_source": snapshot.options.limits_source,
"capabilities": dict(snapshot.options.capabilities),
},
}
def create_run_runtime_snapshot(
snapshot_id: str,
*,
model: str,
provider: str,
) -> RunRuntimeSnapshot | None:
"""Freeze one custom provider model configuration for a forthcoming run.
Static built-in providers remain managed by their existing deployment
configuration and return ``None``. A missing custom model is an error so
a model selection never silently falls back to a different model.
"""
snapshot_id = _validate_snapshot_id(snapshot_id)
model = model.strip()
provider = normalize_provider_id(provider)
if not model or not provider:
raise ProviderProfileError("model and provider are required for a runtime snapshot.")
# Built-in providers are resolved from the deployment configuration and
# have no browser-managed secret or runtime profile to freeze. Avoid
# loading custom profiles at all, so a stale development registry cannot
# block a normal built-in run.
if is_static_provider_id(provider):
_validate_static_provider_credentials(provider)
return None
existing = get_run_runtime_snapshot(snapshot_id)
if existing is not None:
if existing.model.id != model or existing.profile.id != provider:
raise ProviderProfileError(
"runtime snapshot ID is already bound to a different model selection."
)
return existing
resolved = resolve_provider_model(provider, model)
if resolved is None:
return None
profile, provider_model = resolved
options = resolve_runtime_options(profile, provider_model)
now = int(time.time())
snapshot = RunRuntimeSnapshot(
snapshot_id=snapshot_id,
created_at=now,
expires_at=now + _SNAPSHOT_TTL_SECONDS,
profile_revision=get_provider_profile_revision(provider),
profile=profile,
model=provider_model,
options=options,
)
encoded = json.dumps(
_snapshot_payload(snapshot), sort_keys=True, separators=(",", ":")
)
with _DB_LOCK, _connect() as connection:
connection.execute(
"DELETE FROM run_runtime_snapshots WHERE expires_at <= ?", (now,)
)
connection.execute(
"INSERT OR REPLACE INTO run_runtime_snapshots(snapshot_id, expires_at, payload_json) "
"VALUES (?, ?, ?)",
(snapshot.snapshot_id, snapshot.expires_at, encoded),
)
connection.commit()
_SNAPSHOT_SECRETS[snapshot.snapshot_id] = profile.api_key
return snapshot
def get_run_runtime_snapshot(snapshot_id: str) -> RunRuntimeSnapshot | None:
"""Load a still-valid runtime snapshot by its opaque ID."""
snapshot_id = _validate_snapshot_id(snapshot_id)
now = int(time.time())
with _DB_LOCK, _connect() as connection:
connection.execute(
"DELETE FROM run_runtime_snapshots WHERE expires_at <= ?", (now,)
)
row = connection.execute(
"SELECT expires_at, payload_json FROM run_runtime_snapshots WHERE snapshot_id = ?",
(snapshot_id,),
).fetchone()
connection.commit()
if row is None or int(row[0]) <= now:
_SNAPSHOT_SECRETS.pop(snapshot_id, None)
return None
try:
payload = json.loads(str(row[1]))
except json.JSONDecodeError as exc:
raise ProviderProfileError("runtime snapshot is unreadable.") from exc
if not isinstance(payload, dict):
raise ProviderProfileError("runtime snapshot is unreadable.")
return _parse_snapshot_payload(payload)
def get_snapshot_chat_model(snapshot_id: str) -> Any:
"""Build (or reuse) the chat model bound to a frozen runtime snapshot."""
snapshot = get_run_runtime_snapshot(snapshot_id)
if snapshot is None:
raise ProviderProfileError(
"RUN_RUNTIME_SNAPSHOT_UNAVAILABLE: the run configuration snapshot "
"expired or is unavailable. Start the message again."
)
with _MODEL_CACHE_LOCK:
cached = _MODEL_CACHE.get(snapshot.snapshot_id)
if cached is not None:
return cached
from .models import get_profile_chat_model
model = get_profile_chat_model(snapshot.profile, snapshot.model)
with _MODEL_CACHE_LOCK:
_MODEL_CACHE[snapshot.snapshot_id] = model
return model
-2
View File
@@ -28,7 +28,6 @@ from .memory_lifecycle import (
default_memory_scheduler,
)
from .message_budget import (
MessageReservePolicy,
count_message_text_tokens,
create_message_budget_middleware,
)
@@ -50,7 +49,6 @@ __all__ = [
"ContextOverflowMapperMiddleware",
"EvoMemoryLifecycleMiddleware",
"EvoMemoryMiddleware",
"MessageReservePolicy",
"Question",
"RuntimeContextMiddleware",
"SchedulerMiddleware",
+7 -3
View File
@@ -11,6 +11,7 @@ sub-agents are *tasks*, future cron is *schedules*).
from __future__ import annotations
import os
from datetime import UTC, datetime
from langchain.agents.middleware import AgentMiddleware
@@ -80,9 +81,12 @@ def run_in_background(
# apply_config_to_env round-trips at startup (and the subprocess inherits) —
# cheaper than reloading the full config from disk on every launch, and uses
# the same truthy parsing as every other bool env flag.
from ..llm.models import _env_flag_enabled
dangerous = _env_flag_enabled("EVOSCIENTIST_DANGEROUS_MODE")
dangerous = os.getenv("EVOSCIENTIST_DANGEROUS_MODE", "").strip().lower() in {
"1",
"true",
"yes",
"on",
}
# Same path-rewriting + validation as execute (shared helper) so virtual paths
# resolve to the workspace and the command can't bypass the sandbox checks.
command, error = prepare_sandbox_command(
+71 -31
View File
@@ -13,10 +13,15 @@ out-of-snapshot model parameter is rejected with
``MODEL_CONFIG_OUTSIDE_SNAPSHOT`` (422 semantics, section 8.2) — run
creation must never mix snapshot and non-snapshot model configuration.
When no ``runtime_snapshot_id`` is present the middleware is a pass-through:
local entry points (CLI/channels/scheduler) are wired to create local
snapshots separately, and their compile-time model binding applies until
then.
Local entry points (CLI/channels/scheduler/sub-agents, section 8.1) create
snapshots through ``SnapshotRuntime.create_local_snapshot`` and put the
``runtime_snapshot_id`` into ``configurable`` before the run starts. Runs
whose entry point could not inject a snapshot up front (langgraph-dev cron
fires, deployed async sub-agent graphs) are healed lazily: the middleware
creates the local snapshot itself, bound to the run's own thread. A run
carrying neither a snapshot nor a bindable thread ID is rejected, and a
bootstrap registry fails closed with ``MODEL_REGISTRY_NOT_READY`` — there
is no pass-through fallback to a compile-time model anymore.
**Reading the config**: ``Runtime`` (per its own docstring) does NOT include
``config``. The official path to reach ``RunnableConfig`` from inside any
@@ -107,6 +112,47 @@ def read_snapshot_binding(
)
def ensure_snapshot_binding(
configurable: Mapping[str, Any],
runtime: SnapshotRuntime,
) -> tuple[str, str, str]:
"""Return ``(snapshot_id, deployment_id, thread_id)`` for the active run.
The run's explicit ``runtime_snapshot_id`` wins (section 8.2 binding).
Without one, the run is a local entry that could not inject a snapshot
up front — a langgraph-dev cron fire or a deployed sub-agent graph — so
the snapshot is created lazily through the same ``SnapshotService``
(section 8.1): ``deployment_id`` is the platform local deployment ID,
``model_selection_revision`` is ``0``, and ``primary`` inherits the
registry defaults. The lazy ``run_request_id`` is derived from the
thread, so every middleware in the run converges on one snapshot and a
fresh thread (each cron fire, each sub-agent task) freezes fresh
defaults.
Raises:
ModelRegistryError: ``MODEL_CONFIG_OUTSIDE_SNAPSHOT`` when the run
carries neither a snapshot nor a thread ID to bind one to;
``MODEL_REGISTRY_NOT_READY`` when the registry is in bootstrap.
"""
binding = read_snapshot_binding(configurable)
if binding is not None:
snapshot_id, deployment_id, thread_id = binding
return (snapshot_id, deployment_id or runtime.local_deployment_id, thread_id or "")
thread_id = configurable.get("thread_id")
if not isinstance(thread_id, str) or not thread_id:
raise ModelRegistryError(
MODEL_CONFIG_OUTSIDE_SNAPSHOT,
"Model configuration must come from a run snapshot; this run "
"carries neither 'runtime_snapshot_id' nor a 'thread_id' a "
"local snapshot could be bound to.",
details=[{"path": "runtime_snapshot_id", "code": MODEL_CONFIG_OUTSIDE_SNAPSHOT}],
)
snapshot = runtime.create_local_snapshot(
thread_id, run_request_id=f"auto:{thread_id}"
)
return (snapshot.snapshot_id, snapshot.deployment_id, snapshot.thread_id)
class ConfigurableModelMiddleware(AgentMiddleware):
"""Re-resolve the chat model from the run snapshot on every call.
@@ -173,51 +219,48 @@ class ConfigurableModelMiddleware(AgentMiddleware):
*message_args,
)
def _load_snapshot(self, snapshot_id: str) -> RuntimeSnapshot:
"""Load the snapshot, verifying its deployment/thread binding."""
def _load_snapshot(self) -> RuntimeSnapshot:
"""Load the run's snapshot, verifying its deployment/thread binding.
The snapshot is the only model configuration entry point: outside
model keys are rejected first, and a run without an explicit
``runtime_snapshot_id`` gets a lazily created local snapshot bound
to its own thread (never a silent compile-time fallback).
"""
configurable = _current_configurable()
binding = read_snapshot_binding(configurable)
assert binding is not None # guarded by the caller
_, deployment_id, thread_id = binding
check_no_outside_snapshot_model_config(configurable)
runtime = self._snapshot_runtime()
snapshot_id, deployment_id, thread_id = ensure_snapshot_binding(
configurable, runtime
)
return runtime.get_snapshot(
snapshot_id,
deployment_id=deployment_id or runtime.local_deployment_id,
thread_id=thread_id or "",
deployment_id=deployment_id,
thread_id=thread_id,
)
def _resolve(self, snapshot_id: str) -> Any:
"""Return a cached or freshly-built chat model for the snapshot."""
def _resolve(self) -> Any:
"""Return a cached or freshly-built chat model for the run's snapshot."""
snapshot = self._load_snapshot()
with self._lock:
cached = self._cache.get(snapshot_id)
cached = self._cache.get(snapshot.snapshot_id)
if cached is not None:
return cached
snapshot = self._load_snapshot(snapshot_id)
# Build outside the lock (SDK init can be slow); two concurrent
# first-time misses for the same snapshot may build twice but the
# second result simply overwrites the first — both are equivalent.
new_model = self._snapshot_runtime().build_role_model(snapshot, self._role)
with self._lock:
self._cache[snapshot_id] = new_model
self._cache[snapshot.snapshot_id] = new_model
self._log_override(snapshot)
return new_model
def _snapshot_id_for_request(self) -> str | None:
"""Return the run's snapshot ID after the outside-config check."""
configurable = _current_configurable()
check_no_outside_snapshot_model_config(configurable)
binding = read_snapshot_binding(configurable)
return binding[0] if binding is not None else None
def wrap_model_call(
self,
request: ModelRequest,
handler: Callable[[ModelRequest], ModelResponse],
) -> ModelResponse:
snapshot_id = self._snapshot_id_for_request()
if snapshot_id is None:
return handler(request)
new_model = self._resolve(snapshot_id)
new_model = self._resolve()
return handler(request.override(model=new_model))
async def awrap_model_call(
@@ -225,13 +268,10 @@ class ConfigurableModelMiddleware(AgentMiddleware):
request: ModelRequest,
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
) -> ModelResponse:
snapshot_id = self._snapshot_id_for_request()
if snapshot_id is None:
return await handler(request)
# Offload first-call SDK init off the event loop: ``_resolve`` reads
# SQLite and can spend hundreds of ms building HTTP clients on a
# cache miss, which would block every other coroutine on the same
# langgraph dev event loop. Cache hits are still fast; the
# thread-pool overhead is irrelevant once warm.
new_model = await asyncio.to_thread(self._resolve, snapshot_id)
new_model = await asyncio.to_thread(self._resolve)
return await handler(request.override(model=new_model))
+63 -139
View File
@@ -6,13 +6,17 @@ consume the fixed reserves frozen into the run snapshot. The measured value
is only textual conversation messages and tool-result text, which is the
part compaction can actually reduce.
Snapshot mode (design doc 6.5, 8.3): when the run carries a
``runtime_snapshot_id``, the input limit and the three fixed reserves come
from the frozen ``ResolvedModelConfig.budget`` — never from the legacy
``llm/runtime_snapshots.py`` payloads, and never from an implicit 32K
default. The per-call message budget is recomputed on every invocation
from those frozen reserves with the current ``has_tools``/``has_attachments``
mode; it is never carried over from a previous call:
Snapshot mode (design doc 6.5, 8.3): every run carries — or lazily creates,
via the section 8.1 local entry convention — a run snapshot; the input
limit and the three fixed reserves come from the frozen
``ResolvedModelConfig.budget`` of the middleware's own ``snapshot_role``
(``primary`` for the main agent and working sub-agents, ``auxiliary`` for
unattended helpers such as the scheduler). There is no 32K default and no
compile-time profile fallback: a bootstrap registry fails closed with
``MODEL_REGISTRY_NOT_READY``. The per-call message budget is recomputed on
every invocation from those frozen reserves with the current
``has_tools``/``has_attachments`` mode; it is never carried over from a
previous call:
message_budget = resolved_input_limit
- fixed_system_reserve_tokens
@@ -33,10 +37,6 @@ scales proportionally (``x 0.70``), replacing the removed ``safety_reserve``
(section 6.5). Budget satisfiability against ``min_effective_input_tokens``
is guaranteed at snapshot creation (run-creation stage, all four
tool/attachment modes), so the middleware does not re-check it per call.
Interim local mode: runs without a snapshot ID (CLI local path before the
local snapshot entry point is wired) keep the legacy behavior — limits read
from the compile-time model's ``profile`` with the default reserve policy.
"""
from __future__ import annotations
@@ -45,11 +45,12 @@ import threading
from collections.abc import Iterable, Mapping
from contextvars import ContextVar
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, get_args
from langchain_core.messages import AnyMessage, SystemMessage
from .configurable_model import _current_configurable, read_snapshot_binding
from ..model_registry.schemas import ModelRole
from .configurable_model import _current_configurable, ensure_snapshot_binding
if TYPE_CHECKING:
from ..model_registry.runtime import SnapshotRuntime
@@ -61,57 +62,7 @@ _ESTIMATE_HARD_FRACTION = 0.90
_SOFT_FRACTION = 0.70
_KEEP_FRACTION = 0.35
@dataclass(frozen=True)
class MessageReservePolicy:
"""Static non-message reserves for the interim no-snapshot local path."""
system_tokens: int = 4_096
memory_tokens: int = 8_192
tool_tokens: int = 8_192
attachment_tokens: int = 8_192
safety_fraction: float = 0.10
minimum_safety_tokens: int = 2_048
soft_fraction: float = 0.70
keep_fraction: float = 0.35
def fixed_tokens(
self, *, has_tools: bool = False, has_attachments: bool = False
) -> int:
"""Return only the fixed reserves applicable to this model request."""
return (
self.system_tokens
+ self.memory_tokens
+ (self.tool_tokens if has_tools else 0)
+ (self.attachment_tokens if has_attachments else 0)
)
def hard_budget(
self,
input_limit: int,
*,
has_tools: bool = False,
has_attachments: bool = False,
) -> int:
safety = max(
self.minimum_safety_tokens, int(input_limit * self.safety_fraction)
)
return max(
1_024,
input_limit
- self.fixed_tokens(has_tools=has_tools, has_attachments=has_attachments)
- safety,
)
def soft_budget(self, input_limit: int, **mode: bool) -> int:
return max(
1_024, int(self.hard_budget(input_limit, **mode) * self.soft_fraction)
)
def keep_budget(self, input_limit: int, **mode: bool) -> int:
return max(
1_024, int(self.hard_budget(input_limit, **mode) * self.keep_fraction)
)
_MODEL_ROLES = get_args(ModelRole)
@dataclass(frozen=True)
@@ -126,10 +77,6 @@ class MessageBudget:
has_attachments: bool
class ContextBudgetUnsatisfiableError(ValueError):
"""Raised before a call when static reserves leave no safe message budget."""
_ACTIVE_BUDGET: ContextVar[MessageBudget | None] = ContextVar(
"evoscientist_message_budget", default=None
)
@@ -177,12 +124,16 @@ def _has_attachments(messages: Iterable[AnyMessage]) -> bool:
def _snapshot_message_budget(
snapshot: RuntimeSnapshot, *, has_tools: bool, has_attachments: bool
snapshot: RuntimeSnapshot,
role: ModelRole,
*,
has_tools: bool,
has_attachments: bool,
) -> MessageBudget:
"""Recompute the section 6.5 per-call budget from the frozen reserves."""
from ..model_registry.snapshots import config_for_role
budget = config_for_role(snapshot, "primary").budget
budget = config_for_role(snapshot, role).budget
reserves = budget.fixed_reserves
message_budget = (
budget.resolved_input_limit
@@ -209,34 +160,44 @@ class MessageBudgetMiddleware:
model: Any,
backend: Any,
*,
policy: MessageReservePolicy | None = None,
has_tools: bool = True,
snapshot_role: ModelRole = "primary",
runtime: SnapshotRuntime | None = None,
):
"""Create the runtime-aware DeepAgents summarization middleware.
Args:
model: Compile-time fallback model, used for the summarizer and
for budget limits when the run carries no snapshot.
model: Compile-time placeholder model, required by the base
summarization middleware; every per-call budget and the
summarizer model are resolved from the run snapshot.
backend: Graph backend forwarded to the summarization middleware.
policy: Reserve policy for the interim no-snapshot local path.
has_tools: Conservative section 6.5 tool-mode flag: true when the
agent has tool capability and a configured toolset. Decided
once here, never from a single request's bound tool count.
snapshot_role: The snapshot role whose frozen limits size this
agent's budget (``primary`` for the main agent and working
sub-agents, ``auxiliary`` for unattended helper agents such
as the scheduler). The summarizer keeps using the snapshot's
``summary`` role mapping regardless.
runtime: Snapshot runtime override; defaults to the shared
process runtime (tests inject an isolated one).
"""
from deepagents.middleware.summarization import SummarizationMiddleware
if snapshot_role not in _MODEL_ROLES:
raise ValueError(
f"Unknown model role {snapshot_role!r}; "
f"expected one of {list(_MODEL_ROLES)}."
)
class _RuntimeMessageBudgetMiddleware(SummarizationMiddleware):
name = "message_budget"
def __init__(self) -> None:
self._fallback_model = model
self._snapshot_model_cache: dict[str, Any] = {}
self._model_cache_lock = threading.RLock()
self._policy = policy or MessageReservePolicy()
self._has_tools = has_tools
self._snapshot_role = snapshot_role
self._runtime = runtime
# Triggering and cutoff are overridden below. The base class is
# still used for safe AI/tool-pair handling, offloading, and
@@ -263,25 +224,28 @@ class MessageBudgetMiddleware:
return get_snapshot_runtime()
def _snapshot(self) -> RuntimeSnapshot | None:
"""Load the run's snapshot, verifying its binding."""
binding = read_snapshot_binding(_current_configurable())
if binding is None:
return None
snapshot_id, deployment_id, thread_id = binding
def _snapshot(self) -> RuntimeSnapshot:
"""Load the run's snapshot, verifying its binding.
A run without an explicit ``runtime_snapshot_id`` gets a
lazily created local snapshot bound to its own thread
(section 8.1); a bootstrap registry fails closed with
``MODEL_REGISTRY_NOT_READY`` instead of a 32K fallback.
"""
runtime = self._snapshot_runtime()
snapshot_id, deployment_id, thread_id = ensure_snapshot_binding(
_current_configurable(), runtime
)
return runtime.get_snapshot(
snapshot_id,
deployment_id=deployment_id or runtime.local_deployment_id,
thread_id=thread_id or "",
deployment_id=deployment_id,
thread_id=thread_id,
)
@property
def model(self) -> Any: # type: ignore[override]
"""Summarizer model: the snapshot's ``summary`` role mapping."""
snapshot = self._snapshot()
if snapshot is None:
return self._fallback_model
with self._model_cache_lock:
cached = self._snapshot_model_cache.get(snapshot.snapshot_id)
if cached is not None:
@@ -293,63 +257,23 @@ class MessageBudgetMiddleware:
self._snapshot_model_cache[snapshot.snapshot_id] = resolved
return resolved
def _interim_budget(self, *, has_attachments: bool) -> MessageBudget:
"""Legacy local path: limits from the compile-time profile."""
profile = getattr(self._fallback_model, "profile", None)
input_limit = 32_768
minimum_effective = 1_024
if isinstance(profile, Mapping):
value = profile.get("max_input_tokens")
if (
isinstance(value, int)
and not isinstance(value, bool)
and value > 0
):
input_limit = value
value = profile.get("min_effective_input_tokens")
if (
isinstance(value, int)
and not isinstance(value, bool)
and value > 0
):
minimum_effective = value
mode = {
"has_tools": self._has_tools,
"has_attachments": has_attachments,
}
hard = self._policy.hard_budget(input_limit, **mode)
if hard < minimum_effective:
raise ContextBudgetUnsatisfiableError(
"CONTEXT_BUDGET_UNSATISFIABLE: configured input limit "
f"{input_limit:,} leaves only {hard:,} "
"tokens after fixed reserves; increase the model window, reduce "
"the output budget, or disable tools/attachments."
)
return MessageBudget(
input_limit=input_limit,
hard_tokens=hard,
soft_tokens=self._policy.soft_budget(input_limit, **mode),
keep_tokens=self._policy.keep_budget(input_limit, **mode),
def _budget_for_request(self, request: Any) -> MessageBudget:
has_attachments = _has_attachments(getattr(request, "messages", []))
return _snapshot_message_budget(
self._snapshot(),
self._snapshot_role,
has_tools=self._has_tools,
has_attachments=has_attachments,
)
def _budget_for_request(self, request: Any) -> MessageBudget:
has_attachments = _has_attachments(getattr(request, "messages", []))
snapshot = self._snapshot()
if snapshot is not None:
return _snapshot_message_budget(
snapshot,
has_tools=self._has_tools,
has_attachments=has_attachments,
)
return self._interim_budget(has_attachments=has_attachments)
def _active_budget(self) -> MessageBudget:
active = _ACTIVE_BUDGET.get()
if active is not None:
return active
return self._interim_budget(has_attachments=False)
if active is None:
raise RuntimeError(
"message budget accessed outside a model call; "
"wrap_model_call establishes the per-call budget."
)
return active
def wrap_model_call(self, request: Any, handler: Any) -> Any:
token = _ACTIVE_BUDGET.set(self._budget_for_request(request))
@@ -408,11 +332,11 @@ def create_message_budget_middleware(
model: Any,
backend: Any,
*,
policy: MessageReservePolicy | None = None,
has_tools: bool = True,
snapshot_role: ModelRole = "primary",
runtime: SnapshotRuntime | None = None,
):
"""Construct automatic compaction middleware for a graph backend."""
return MessageBudgetMiddleware.create(
model, backend, policy=policy, has_tools=has_tools, runtime=runtime
model, backend, has_tools=has_tools, snapshot_role=snapshot_role, runtime=runtime
)
+42
View File
@@ -22,6 +22,7 @@ retry option. Secrets are resolved per call and never logged.
from __future__ import annotations
import threading
import uuid
from pathlib import Path
import yaml
@@ -122,6 +123,47 @@ class SnapshotRuntime:
config = self.resolve_role_config(role)
return self._build(config, self._resolve_credential(config))
# --- local entry points (design doc 8.1) ---------------------------------
def create_local_snapshot(
self, thread_id: str, *, run_request_id: str | None = None
) -> RuntimeSnapshot:
"""Create (or reuse) a run snapshot for a local entry point.
Local entries — CLI, channels, scheduled tasks, and sub-agents — do
not go through the BFF; they share the same ``SnapshotService`` and
the same snapshot table as the HTTP API (section 8.1). The binding
convention is fixed: ``deployment_id`` is the platform
``local_deployment_id``, ``thread_id`` is the local session/task
identifier, ``model_selection_revision`` is ``0``, and ``primary``
is ``None`` (inherit — the registry defaults are resolved and frozen
at creation time).
``run_request_id`` scopes idempotency: reusing the same
``{deployment_id, thread_id, run_request_id}`` triple returns the
existing snapshot instead of creating a new one. Pass a fresh ID
(the default, a generated UUID) to freeze a new snapshot per run, or
a deterministic one to share a snapshot across collaborators in the
same run.
Raises:
ModelRegistryError: ``MODEL_REGISTRY_NOT_READY`` when the
registry is still in bootstrap (no enabled primary model).
"""
from .snapshots import SnapshotCreateRequest
creation = self._snapshots.create(
SnapshotCreateRequest(
run_request_id=run_request_id or uuid.uuid4().hex,
thread_id=thread_id,
deployment_id=self._local_deployment_id,
model_selection_revision=0,
primary=None,
auxiliary=None,
)
)
return creation.snapshot
# --- internals -------------------------------------------------------------
def _resolve_credential(self, config: ResolvedModelConfig) -> str:
@@ -2,7 +2,8 @@
"""Improve a skill description based on eval results.
Takes eval results (from run_eval.py) and generates an improved description
using EvoSci's LLM layer (multi-provider support).
using ``langchain.chat_models.init_chat_model`` (multi-provider support via
environment-provided API keys, optionally seeded from the EvoSci config).
"""
import argparse
@@ -16,8 +17,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from langchain_core.messages import HumanMessage
from EvoScientist.config import apply_config_to_env, get_effective_config
from EvoScientist.llm import get_chat_model
from scripts.run_eval import _config_defaults, _init_chat_model
from scripts.utils import parse_skill_md
@@ -141,13 +141,15 @@ I'd encourage you to be creative and mix up the style in different iterations si
Please respond with only the new description text in <new_description> tags, nothing else."""
# Initialise model via EvoSci's LLM layer
config = get_effective_config()
apply_config_to_env(config)
chat_model = get_chat_model(
model=model or config.model,
provider=provider or config.provider,
)
# Initialise model (API keys come from the environment / EvoSci config)
default_model, default_provider = _config_defaults()
effective_model = model or default_model
if not effective_model:
raise RuntimeError(
"No model specified: pass --model or configure one (or set a "
"provider's default model via environment)."
)
chat_model = _init_chat_model(effective_model, provider or default_provider)
response = chat_model.invoke([HumanMessage(content=prompt)])
@@ -253,9 +255,8 @@ def main():
name, _, content = parse_skill_md(skill_path)
current_description = eval_results["description"]
# Load EvoSci config for defaults
config = get_effective_config()
apply_config_to_env(config)
# Load EvoSci config for defaults (best-effort; env vars alone suffice)
default_model, default_provider = _config_defaults()
if args.verbose:
print(f"Current: {current_description}", file=sys.stderr)
@@ -270,8 +271,8 @@ def main():
current_description=current_description,
eval_results=eval_results,
history=history,
model=args.model or config.model,
provider=args.provider or config.provider,
model=args.model or default_model,
provider=args.provider or default_provider,
)
if args.verbose:
@@ -2,8 +2,10 @@
"""Run trigger evaluation for a skill description.
Tests whether a skill's description causes an LLM to trigger (load the skill)
for a set of queries. Uses EvoSci's multi-provider LLM layer with tool calling
to simulate the agent's skill selection behavior.
for a set of queries. Uses ``langchain.chat_models.init_chat_model`` with
tool calling to simulate the agent's skill selection behavior. API keys come
from the environment (optionally seeded from the EvoSci config when
available).
"""
import argparse
@@ -18,13 +20,28 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from scripts.utils import parse_skill_md
def _init_config():
"""Initialize EvoSci config and apply env vars (once per process)."""
from EvoScientist.config import apply_config_to_env, get_effective_config
def _config_defaults() -> tuple[str | None, str | None]:
"""Best-effort (model, provider) defaults from the EvoSci config.
config = get_effective_config()
apply_config_to_env(config)
return config
Also applies configured API keys to the process environment so
``init_chat_model`` can pick them up. Returns ``(None, None)`` when the
config layer is unavailable — environment variables alone suffice.
"""
try:
from EvoScientist.config import apply_config_to_env, get_effective_config
config = get_effective_config()
apply_config_to_env(config)
return getattr(config, "model", None), getattr(config, "provider", None)
except Exception:
return None, None
def _init_chat_model(model: str, provider: str | None, **kwargs):
"""Build a chat model; API keys are read from the environment."""
from langchain.chat_models import init_chat_model
return init_chat_model(model=model, model_provider=provider, **kwargs)
def run_single_query(
@@ -43,12 +60,14 @@ def run_single_query(
from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.tools import tool
from EvoScientist.llm import get_chat_model
config = _init_config()
effective_model = model or config.model
effective_provider = provider or config.provider
default_model, default_provider = _config_defaults()
effective_model = model or default_model
effective_provider = provider or default_provider
if not effective_model:
raise RuntimeError(
"No model specified: pass --model or configure one (or set a "
"provider's default model via environment)."
)
@tool
def load_skill(name: str) -> str:
@@ -72,9 +91,9 @@ If no skill is relevant, respond directly to the user without calling any tools.
pass
try:
chat_model = get_chat_model(
model=effective_model,
provider=effective_provider,
chat_model = _init_chat_model(
effective_model,
effective_provider,
**eval_kwargs,
)
model_with_tools = chat_model.bind_tools([load_skill])
@@ -98,9 +117,9 @@ If no skill is relevant, respond directly to the user without calling any tools.
# Fallback for providers that don't support tool calling:
# Use a text-based approach
try:
chat_model = get_chat_model(
model=effective_model,
provider=effective_provider,
chat_model = _init_chat_model(
effective_model,
effective_provider,
**eval_kwargs,
)
fallback_prompt = f"""You are a helpful AI assistant with specialized skills available.
@@ -221,12 +240,14 @@ def main():
"--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold"
)
parser.add_argument(
"--model", default=None, help="Model to use (default: user's configured model)"
"--model",
default=None,
help="Model to use (default: EvoSci config model; required otherwise)",
)
parser.add_argument(
"--provider",
default=None,
help="LLM provider (default: user's configured provider)",
help="LLM provider (default: EvoSci config provider, else inferred from model)",
)
parser.add_argument(
"--verbose", action="store_true", help="Print progress to stderr"
@@ -18,10 +18,9 @@ from pathlib import Path
# Ensure skill-creator root is on sys.path for `from scripts.xxx` imports
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from EvoScientist.config import apply_config_to_env, get_effective_config
from scripts.generate_report import generate_html
from scripts.improve_description import improve_description
from scripts.run_eval import run_eval
from scripts.run_eval import _config_defaults, run_eval
from scripts.utils import parse_skill_md
@@ -66,8 +65,10 @@ def run_loop(
log_dir: Path | None = None,
) -> dict:
"""Run the eval + improvement loop."""
config = get_effective_config()
apply_config_to_env(config)
# Seed API keys into the environment from the EvoSci config (best-effort);
# model/provider defaults are resolved per call by run_eval /
# improve_description when not passed explicitly.
_config_defaults()
name, original_description, content = parse_skill_md(skill_path)
current_description = description_override or original_description
+6
View File
@@ -797,6 +797,7 @@ async def stream_agent_events(
thread_id: str,
metadata: dict[str, Any] | None = None,
media: list[str] | None = None,
runtime_snapshot_id: str | None = None,
) -> AsyncGenerator[dict[str, Any], None]:
"""Stream events from a DeepAgents/LangGraph v3 run.
@@ -812,6 +813,9 @@ async def stream_agent_events(
metadata: Optional metadata dict merged into the LangGraph config
(e.g. agent_name, updated_at for checkpoint persistence).
media: Optional list of local file paths for attachments.
runtime_snapshot_id: Optional frozen run snapshot ID (design doc
8.1/8.2) placed into ``configurable`` — the only model
configuration a run may carry.
Yields:
Event dicts: thinking, text, tool_call, tool_result,
@@ -821,6 +825,8 @@ async def stream_agent_events(
config: dict[str, Any] = {"configurable": {"thread_id": thread_id}}
if metadata:
config["metadata"] = metadata
if runtime_snapshot_id is not None:
config["configurable"]["runtime_snapshot_id"] = runtime_snapshot_id
emitter = StreamEventEmitter()
existing_summarization_event: Mapping[str, object] | None = None
try:
+44 -5
View File
@@ -103,13 +103,52 @@ def build_async_subagent_graph(name: str) -> Any:
# context and `/memories/profile/...` file guidance as the main agent.
#
# The compile-time model binding is resolved from the active registry's
# defaults via build_chat_model — never from config.yaml free strings.
# Per-run calls are re-resolved from the run snapshot by
# ConfigurableModelMiddleware; the scheduler (an unattended timer task)
# binds the cheaper auxiliary role, working sub-agents bind primary.
# defaults — never from config.yaml free strings. Per-run calls are
# re-resolved from the run snapshot by ConfigurableModelMiddleware; the
# scheduler (an unattended timer task) binds the cheaper auxiliary role,
# working sub-agents bind primary.
#
# Bootstrap registry: no defaults exist yet, so no compile-time model
# can be built. The graph must still materialize — one failing factory
# must not take down the whole langgraph dev service — so we bind a
# placeholder that raises MODEL_REGISTRY_NOT_READY on the first model
# call. Every run fails with that clear structured error until a
# primary model is configured and enabled; no 32K/implicit fallback.
from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.messages import BaseMessage
from langchain_core.outputs import ChatResult
from EvoScientist.model_registry.errors import (
MODEL_REGISTRY_NOT_READY,
ModelRegistryError,
)
class _RegistryNotReadyChatModel(BaseChatModel):
"""Placeholder that fails every call with MODEL_REGISTRY_NOT_READY."""
detail: str
@property
def _llm_type(self) -> str:
return "evoscientist-registry-not-ready"
def _generate(
self,
messages: list[BaseMessage],
stop: list[str] | None = None,
run_manager: Any = None,
**kwargs: Any,
) -> ChatResult:
raise ModelRegistryError(MODEL_REGISTRY_NOT_READY, self.detail)
runtime = get_snapshot_runtime()
snapshot_role = "auxiliary" if name == "scheduler" else "primary"
model = runtime.build_default_role_model(snapshot_role)
try:
model = runtime.build_default_role_model(snapshot_role)
except ModelRegistryError as exc:
if exc.code != MODEL_REGISTRY_NOT_READY:
raise
model = _RegistryNotReadyChatModel(detail=str(exc))
subagents = []
_ensure_general_purpose_subagent(subagents)
+24
View File
@@ -3,6 +3,16 @@
import pytest
@pytest.fixture(autouse=True)
def _isolated_config_home(tmp_path, monkeypatch):
"""Keep every test away from the real ``~/.config/evoscientist``.
The startup legacy-artifact guard reads the real config dir otherwise,
and tests must never depend on (or trip over) developer-machine state.
"""
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "xdg-config"))
@pytest.fixture(autouse=True)
def _reset_tool_selection_state():
"""Isolate the process-global tool-selection state around every test.
@@ -46,6 +56,20 @@ def isolated_snapshot_runtime(tmp_path):
set_snapshot_runtime_for_tests(None)
@pytest.fixture
def active_snapshot_runtime(isolated_snapshot_runtime):
"""Shared runtime backed by an active registry with verified models.
For tests that exercise code paths building compile-time models or
creating local run snapshots (both fail closed with
``MODEL_REGISTRY_NOT_READY`` against the default bootstrap registry).
"""
from tests.registry_fixtures import activate_store
activate_store(isolated_snapshot_runtime.store)
return isolated_snapshot_runtime
@pytest.fixture
def sample_tool_call():
"""A minimal tool call dict."""
-8
View File
@@ -77,14 +77,6 @@ class FakeCommandUI(CommandUI):
) -> list | None:
return None
async def wait_for_model_pick(
self,
entries: list[tuple[str, str, str]],
current_model: str | None,
current_provider: str | None,
) -> tuple[str, str] | None:
return None
def clear_chat(self) -> None:
self.chat_cleared = True
+6 -3
View File
@@ -539,10 +539,11 @@ class TestConfig:
@patch("EvoScientist.middleware.create_tool_selector_middleware", return_value=[])
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_auto_approve_still_includes_ask_user_middleware(
mock_config, mock_model, mock_tool_selector
mock_config, mock_model, mock_aux_model, mock_tool_selector
):
cfg = MagicMock()
cfg.enable_ask_user = True
@@ -560,10 +561,11 @@ def test_auto_approve_still_includes_ask_user_middleware(
@patch("EvoScientist.middleware.create_tool_selector_middleware", return_value=[])
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_auto_mode_disables_ask_user_middleware(
mock_config, mock_model, mock_tool_selector
mock_config, mock_model, mock_aux_model, mock_tool_selector
):
cfg = MagicMock()
cfg.enable_ask_user = True
@@ -581,10 +583,11 @@ def test_auto_mode_disables_ask_user_middleware(
@patch("EvoScientist.middleware.create_tool_selector_middleware", return_value=[])
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_for_async_subagent_omits_ask_user_middleware(
mock_config, mock_model, mock_tool_selector
mock_config, mock_model, mock_aux_model, mock_tool_selector
):
"""``AskUserMiddleware`` uses ``interrupt()`` to wait on user input.
+65 -2
View File
@@ -11,6 +11,8 @@ from __future__ import annotations
from unittest.mock import MagicMock, patch
import pytest
from EvoScientist.config import MemoryObservationWriter
from EvoScientist.memory import MemorySourceType
@@ -167,6 +169,65 @@ def test_scheduler_binds_auxiliary_role(
assert mock_create.call_args.kwargs["model"] is model
@patch("deepagents.create_deep_agent")
@patch("EvoScientist.EvoScientist._load_mcp_tools_cached", return_value={})
@patch("EvoScientist.EvoScientist._get_default_middleware", return_value=[])
@patch("EvoScientist.EvoScientist._get_default_backend")
@patch("EvoScientist.model_registry.runtime.get_snapshot_runtime")
@patch("EvoScientist.utils.load_subagents")
@patch("EvoScientist.config.apply_config_to_env")
@patch("EvoScientist.config.get_effective_config")
def test_bootstrap_registry_binds_not_ready_placeholder(
mock_get_cfg,
mock_apply_env,
mock_load_subs,
mock_get_runtime,
mock_backend,
mock_get_mw,
mock_mcp,
mock_create,
):
"""T6-b: bootstrap registry must not take down graph materialization.
The factory binds a placeholder chat model whose every call raises
``MODEL_REGISTRY_NOT_READY`` instead of crashing the whole service.
"""
from EvoScientist.model_registry.errors import (
MODEL_REGISTRY_NOT_READY,
ModelRegistryError,
)
cfg = MagicMock()
cfg.recursion_limit = 1_000_000
cfg.memory_profile_enabled = False
cfg.memory_observations_enabled = False
cfg.memory_observation_writer = MemoryObservationWriter.ALL
cfg.memory_workers_enabled = False
mock_get_cfg.return_value = cfg
mock_get_runtime.return_value.build_default_role_model.side_effect = (
ModelRegistryError(MODEL_REGISTRY_NOT_READY, "bootstrap")
)
mock_load_subs.return_value = [
{
"name": "writing-agent",
"system_prompt": "",
"tools": [],
"skills": None,
}
]
mock_create.return_value.with_config.return_value = MagicMock()
from EvoScientist.subagents._factory import build_async_subagent_graph
build_async_subagent_graph("writing-agent") # must not raise
model = mock_create.call_args.kwargs["model"]
assert type(model).__name__ == "_RegistryNotReadyChatModel"
with pytest.raises(ModelRegistryError) as excinfo:
model.invoke([])
assert excinfo.value.code == MODEL_REGISTRY_NOT_READY
@patch("EvoScientist.EvoScientist._ensure_chat_model")
def test_inject_subagent_adds_memory_middleware(mock_model, tmp_path):
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
@@ -251,10 +312,11 @@ def test_inject_subagent_worker_only_observation_writer_keeps_live_tool_off(
"EvoScientist.middleware.create_tool_selector_middleware",
return_value=[MagicMock()],
)
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_all_observation_writer_schedules_turn_worker_without_profile_memory(
mock_config, mock_chat, mock_tool_selector
mock_config, mock_chat, mock_aux, mock_tool_selector
):
cfg = MagicMock()
cfg.enable_ask_user = False
@@ -303,10 +365,11 @@ def test_all_observation_writer_schedules_turn_worker_without_profile_memory(
"EvoScientist.middleware.create_tool_selector_middleware",
return_value=[MagicMock()],
)
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_async_subagent_mode_filters_ask_user(
mock_config, mock_chat, mock_tool_selector
mock_config, mock_chat, mock_aux, mock_tool_selector
):
"""``_get_default_middleware(for_async_subagent=True)`` must drop
``AskUserMiddleware`` even when ``enable_ask_user`` is on.
+9 -20
View File
@@ -14,6 +14,10 @@ from unittest.mock import MagicMock, patch
import pytest
import EvoScientist.EvoScientist as E
from EvoScientist.model_registry.errors import (
MODEL_REGISTRY_NOT_READY,
ModelRegistryError,
)
from tests.registry_fixtures import activate_store
@@ -27,13 +31,11 @@ def _reset_model_caches(monkeypatch):
class TestAuxiliaryResolver:
def test_bootstrap_registry_returns_main_instance(
self, monkeypatch, isolated_snapshot_runtime
):
"""Pre-cutover CLI (bootstrap registry): auxiliary mirrors main."""
main = object()
monkeypatch.setattr(E, "_ensure_chat_model", lambda: main)
assert E._ensure_auxiliary_chat_model() is main
def test_bootstrap_registry_fails_closed(self, isolated_snapshot_runtime):
"""Bootstrap registry: auxiliary resolution propagates NOT_READY."""
with pytest.raises(ModelRegistryError) as excinfo:
E._ensure_auxiliary_chat_model()
assert excinfo.value.code == MODEL_REGISTRY_NOT_READY
def test_no_auxiliary_default_returns_main_instance(
self, monkeypatch, isolated_snapshot_runtime
@@ -81,19 +83,6 @@ class TestAuxiliaryResolver:
assert E._ensure_auxiliary_chat_model() is first
assert build.call_count == 1
def test_set_chat_model_resets_aux_cache(self, monkeypatch):
monkeypatch.setattr(E, "_auxiliary_chat_model", object(), raising=False)
monkeypatch.setattr(
E, "_auxiliary_chat_model_key", ("x", "y", 1), raising=False
)
monkeypatch.setattr(
"EvoScientist.llm.get_chat_model", MagicMock(return_value=object())
)
E.set_chat_model("new-m", "new-p")
assert E._auxiliary_chat_model is None
assert E._auxiliary_chat_model_key is None
def _mock_cfg():
cfg = MagicMock()
cfg.enable_ask_user = False
+10 -10
View File
@@ -30,19 +30,19 @@ class TestSlashCommandCompleter:
assert completions == []
def test_exact_command_hides_even_when_prefix_of_another(self):
"""Regression for #293: ``/model`` must hide on exact match so Enter
submits it, even when another command shares the prefix. Before
"""Regression for #293: an exact command match must hide the popup so
Enter submits it, even when another command shares the prefix. Before
the fix the popup stayed visible (two prefix matches) and the TUI's
Enter handler completed the text instead of executing the command.
The original prefix sibling (``/model-fallback``) was removed with
the fallback chain; a temporary stub command recreates the scenario.
A temporary stub command recreates the prefix-sibling scenario
against the existing ``/help`` command.
"""
from EvoScientist.commands.base import Command
from EvoScientist.commands.manager import manager
class _StubCommand(Command):
name = "/model-extra"
name = "/help-extra"
description = "stub"
async def execute(self, ctx, args):
@@ -52,13 +52,13 @@ class TestSlashCommandCompleter:
try:
completer = SlashCommandCompleter()
# Sanity: both commands share the prefix, so a partial prefix lists both.
partial = {c.text for c in completer.get_completions(_doc("/mode"), None)}
assert {"/model", "/model-extra"} <= partial
# Exact ``/model`` with no trailing space → hide.
completions = list(completer.get_completions(_doc("/model"), None))
partial = {c.text for c in completer.get_completions(_doc("/hel"), None)}
assert {"/help", "/help-extra"} <= partial
# Exact ``/help`` with no trailing space → hide.
completions = list(completer.get_completions(_doc("/help"), None))
assert completions == []
finally:
manager._commands.pop("/model-extra", None)
manager._commands.pop("/help-extra", None)
def test_non_slash_returns_empty(self):
completer = SlashCommandCompleter()
+9
View File
@@ -5,10 +5,19 @@ from __future__ import annotations
import os
from types import SimpleNamespace
import pytest
from EvoScientist.cli import commands
from EvoScientist.config import MemoryObservationWriter
@pytest.fixture(autouse=True)
def _active_registry(active_snapshot_runtime):
"""Run entry points freeze a local snapshot per run; that fails closed
against the default bootstrap registry, so activate it for these tests."""
return active_snapshot_runtime
def _make_config(
*,
default_workdir: str = "",
+72
View File
@@ -0,0 +1,72 @@
"""Tests for the CLI startup gates (design doc §10 step 4, §8.1).
``_startup_gates`` is the shared refuse-startup path for the chat entry and
``serve``: legacy model-configuration artifacts abort with the reset guide,
and a bootstrap registry aborts with ``MODEL_REGISTRY_NOT_READY`` — both
with exit code 1, never a silent fall-through to an implicit model.
"""
from __future__ import annotations
from unittest.mock import MagicMock
import pytest
import typer
import EvoScientist.config.legacy_artifacts as legacy_artifacts
import EvoScientist.model_registry.runtime as runtime_module
from EvoScientist.cli.commands import _startup_gates
from EvoScientist.config.legacy_artifacts import LegacyArtifactsError
from EvoScientist.model_registry.errors import (
MODEL_REGISTRY_NOT_READY,
ModelRegistryError,
)
def test_legacy_artifacts_abort_startup(monkeypatch):
monkeypatch.setattr(
legacy_artifacts,
"assert_no_legacy_artifacts",
MagicMock(side_effect=LegacyArtifactsError("legacy providers.yaml found")),
)
with pytest.raises(typer.Exit) as excinfo:
_startup_gates()
assert excinfo.value.exit_code == 1
def test_bootstrap_registry_aborts_startup(monkeypatch):
monkeypatch.setattr(
legacy_artifacts, "assert_no_legacy_artifacts", MagicMock()
)
runtime = MagicMock()
runtime.registry_defaults.side_effect = ModelRegistryError(
MODEL_REGISTRY_NOT_READY, "bootstrap"
)
monkeypatch.setattr(runtime_module, "get_snapshot_runtime", lambda: runtime)
with pytest.raises(typer.Exit) as excinfo:
_startup_gates()
assert excinfo.value.exit_code == 1
def test_clean_environment_passes(monkeypatch):
monkeypatch.setattr(
legacy_artifacts, "assert_no_legacy_artifacts", MagicMock()
)
runtime = MagicMock()
runtime.registry_defaults.return_value = (MagicMock(), None, 2)
monkeypatch.setattr(runtime_module, "get_snapshot_runtime", lambda: runtime)
_startup_gates() # must not raise
def test_non_bootstrap_registry_error_propagates(monkeypatch):
"""Only NOT_READY is converted to a guided exit; other errors bubble up."""
monkeypatch.setattr(
legacy_artifacts, "assert_no_legacy_artifacts", MagicMock()
)
runtime = MagicMock()
runtime.registry_defaults.side_effect = ModelRegistryError(
"SNAPSHOT_EXPIRED", "expired"
)
monkeypatch.setattr(runtime_module, "get_snapshot_runtime", lambda: runtime)
with pytest.raises(ModelRegistryError):
_startup_gates()
+4 -2
View File
@@ -64,14 +64,16 @@ def _invoke_main(monkeypatch, argv):
return calls, result
def test_main_callback_launches_webui_for_fresh_session(monkeypatch):
def test_main_callback_launches_webui_for_fresh_session(
monkeypatch, active_snapshot_runtime
):
"""Bare `EvoSci` with ui_backend=webui opens the browser app."""
calls, result = _invoke_main(monkeypatch, [])
assert result.exit_code == 0
assert calls.get("dispatch") == "webui"
def test_main_callback_resume_falls_back_to_cli(monkeypatch):
def test_main_callback_resume_falls_back_to_cli(monkeypatch, active_snapshot_runtime):
"""`EvoSci --resume <id>` with ui_backend=webui does NOT open the browser;
it resumes the conversation in the Rich CLI (ui_backend forced to 'cli')."""
calls, result = _invoke_main(monkeypatch, ["--resume", "abc123"])
+115 -193
View File
@@ -59,10 +59,8 @@ def temp_config_dir(tmp_path, monkeypatch):
"EvoScientist.config.settings.find_dotenv",
lambda *a, **k: str(tmp_path / ".env"),
)
# Also clear any API keys from environment
# Also clear any relevant keys from environment
for key in [
"ANTHROPIC_API_KEY",
"OPENAI_API_KEY",
"TAVILY_API_KEY",
"EVOSCIENTIST_DEFAULT_MODE",
"EVOSCIENTIST_WORKSPACE_DIR",
@@ -75,9 +73,6 @@ def temp_config_dir(tmp_path, monkeypatch):
"EVOSCIENTIST_MEMORY_SKILL_SYNTHESIS_MODE",
"EVOSCIENTIST_MEMORY_SKILL_SYNTHESIS_CADENCE",
"EVOSCIENTIST_MEMORY_SKILL_SYNTHESIS_TIME",
"EVOSCIENTIST_AUXILIARY_MODEL",
"EVOSCIENTIST_AUXILIARY_PROVIDER",
"EVOSCIENTIST_OPENROUTER_ANTHROPIC_PROMPT_CACHE",
"EVOSCIENTIST_DANGEROUS_MODE",
]:
monkeypatch.delenv(key, raising=False)
@@ -88,8 +83,6 @@ def temp_config_dir(tmp_path, monkeypatch):
def clean_env(monkeypatch):
"""Remove environment variables that affect config (but keep temp config dir)."""
for key in [
"ANTHROPIC_API_KEY",
"OPENAI_API_KEY",
"TAVILY_API_KEY",
"EVOSCIENTIST_DEFAULT_MODE",
"EVOSCIENTIST_WORKSPACE_DIR",
@@ -102,9 +95,6 @@ def clean_env(monkeypatch):
"EVOSCIENTIST_MEMORY_SKILL_SYNTHESIS_MODE",
"EVOSCIENTIST_MEMORY_SKILL_SYNTHESIS_CADENCE",
"EVOSCIENTIST_MEMORY_SKILL_SYNTHESIS_TIME",
"EVOSCIENTIST_AUXILIARY_MODEL",
"EVOSCIENTIST_AUXILIARY_PROVIDER",
"EVOSCIENTIST_OPENROUTER_ANTHROPIC_PROMPT_CACHE",
"EVOSCIENTIST_DANGEROUS_MODE",
]:
monkeypatch.delenv(key, raising=False)
@@ -120,18 +110,13 @@ class TestEvoScientistConfig:
"""Test that default values are set correctly."""
config = EvoScientistConfig()
assert config.anthropic_api_key == ""
assert config.openai_api_key == ""
assert config.tavily_api_key == ""
assert config.provider == "anthropic"
assert config.model == "claude-sonnet-4-6"
assert config.default_mode == "daemon"
assert config.default_workdir == ""
assert config.show_thinking is True
assert config.ui_backend == "tui"
assert config.log_level == "warning"
assert config.reasoning_effort == "high"
assert config.openrouter_anthropic_prompt_cache is True
assert config.memory_profile_enabled is True
assert config.memory_observations_enabled is True
assert config.memory_observation_writer == MemoryObservationWriter.ALL
@@ -142,44 +127,58 @@ class TestEvoScientistConfig:
config.memory_skill_synthesis_cadence == MemorySkillSynthesisCadence.WEEKLY
)
assert config.memory_skill_synthesis_time == "03:00"
assert config.ollama_base_url == ""
assert config.channel_debug_tracing is False
assert config.imessage_enabled is False
assert config.imessage_allowed_senders == ""
def test_auth_mode_default(self):
"""Test that anthropic_auth_mode defaults to api_key."""
def test_llm_fields_removed(self):
"""LLM provider/model/API-key fields live in the model registry, not config."""
config = EvoScientistConfig()
assert config.anthropic_auth_mode == "api_key"
def test_auth_mode_set(self):
"""Test that anthropic_auth_mode can be set."""
config = EvoScientistConfig(anthropic_auth_mode="oauth")
assert config.anthropic_auth_mode == "oauth"
def test_openai_auth_mode_default(self):
"""Test that openai_auth_mode defaults to api_key."""
config = EvoScientistConfig()
assert config.openai_auth_mode == "api_key"
def test_openai_auth_mode_set(self):
"""Test that openai_auth_mode can be set."""
config = EvoScientistConfig(openai_auth_mode="oauth")
assert config.openai_auth_mode == "oauth"
for field_name in [
"provider",
"model",
"model_catalog",
"model_fallbacks",
"auxiliary_provider",
"auxiliary_model",
"anthropic_api_key",
"anthropic_base_url",
"anthropic_auth_mode",
"openai_api_key",
"openai_auth_mode",
"nvidia_api_key",
"google_api_key",
"minimax_api_key",
"minimax_base_url",
"siliconflow_api_key",
"openrouter_api_key",
"deepseek_api_key",
"zhipu_api_key",
"volcengine_api_key",
"dashscope_api_key",
"moonshot_api_key",
"kimi_api_key",
"custom_openai_api_key",
"custom_openai_base_url",
"custom_anthropic_api_key",
"custom_anthropic_base_url",
"ollama_base_url",
"openrouter_anthropic_prompt_cache",
"use_responses_api",
]:
assert not hasattr(config, field_name), field_name
def test_custom_values(self):
"""Test that custom values can be set."""
config = EvoScientistConfig(
anthropic_api_key="sk-ant-test",
provider="openai",
model="gpt-4o",
tavily_api_key="tvly-test",
default_mode="run",
show_thinking=False,
)
assert config.anthropic_api_key == "sk-ant-test"
assert config.provider == "openai"
assert config.model == "gpt-4o"
assert config.tavily_api_key == "tvly-test"
assert config.default_mode == "run"
assert config.show_thinking is False
def test_dangerous_mode_default(self):
"""dangerous_mode defaults off and does not force auto_approve."""
@@ -252,12 +251,12 @@ class TestLoadSaveReset:
def test_load_returns_defaults_when_no_file(self, temp_config_dir, clean_env):
"""Test that load returns defaults when config file doesn't exist."""
config = load_config()
assert config.provider == "anthropic"
assert config.model == "claude-sonnet-4-6"
assert config.default_mode == "daemon"
assert config.ui_backend == "tui"
def test_save_creates_file(self, temp_config_dir, clean_env):
"""Test that save creates the config file."""
config = EvoScientistConfig(provider="openai", model="gpt-4o")
config = EvoScientistConfig(default_mode="run", log_level="INFO")
save_config(config)
config_path = get_config_path()
@@ -265,14 +264,14 @@ class TestLoadSaveReset:
with open(config_path) as f:
data = yaml.safe_load(f)
assert data["provider"] == "openai"
assert data["model"] == "gpt-4o"
assert data["default_mode"] == "run"
assert data["log_level"] == "INFO"
def test_save_restricts_config_permissions(self, temp_config_dir, clean_env):
"""Config file permissions should not depend on the process umask."""
original_umask = os.umask(0)
try:
save_config(EvoScientistConfig(anthropic_api_key="test-key"))
save_config(EvoScientistConfig(tavily_api_key="test-key"))
finally:
os.umask(original_umask)
@@ -288,28 +287,28 @@ class TestLoadSaveReset:
def test_load_reads_saved_config(self, temp_config_dir, clean_env):
"""Test that load reads previously saved config."""
original = EvoScientistConfig(
anthropic_api_key="test-key",
provider="openai",
tavily_api_key="test-key",
default_mode="run",
)
save_config(original)
loaded = load_config()
assert loaded.anthropic_api_key == "test-key"
assert loaded.provider == "openai"
assert loaded.tavily_api_key == "test-key"
assert loaded.default_mode == "run"
def test_load_reads_manually_edited_utf8_config(self, temp_config_dir, clean_env):
"""Config loading handles localized content from manually edited YAML."""
config_path = get_config_path()
config_path.parent.mkdir(parents=True, exist_ok=True)
config_path.write_text(
"provider: openai\nmodel: gpt-4o\ndefault_workdir: /tmp/café\n",
"default_mode: run\nlog_level: INFO\ndefault_workdir: /tmp/café\n",
encoding="utf-8",
)
loaded = load_config()
assert loaded.provider == "openai"
assert loaded.model == "gpt-4o"
assert loaded.default_mode == "run"
assert loaded.log_level == "INFO"
assert loaded.default_workdir == "/tmp/café"
def test_save_writes_utf8_config(self, temp_config_dir, clean_env):
@@ -323,7 +322,7 @@ class TestLoadSaveReset:
def test_reset_deletes_config_file(self, temp_config_dir, clean_env):
"""Test that reset deletes the config file."""
config = EvoScientistConfig(provider="openai")
config = EvoScientistConfig(default_mode="run")
save_config(config)
config_path = get_config_path()
@@ -344,7 +343,7 @@ class TestLoadSaveReset:
with open(config_path, "w") as f:
yaml.safe_dump(
{
"provider": "openai",
"default_mode": "run",
"unknown_field": "should_be_ignored",
"another_bad": 123,
},
@@ -352,9 +351,25 @@ class TestLoadSaveReset:
)
config = load_config()
assert config.provider == "openai"
assert config.default_mode == "run"
assert not hasattr(config, "unknown_field")
def test_load_ignores_legacy_llm_fields(self, temp_config_dir, clean_env):
"""Legacy LLM keys left in config.yaml are dropped, not loaded."""
config_path = get_config_path()
config_path.parent.mkdir(parents=True, exist_ok=True)
config_path.write_text(
"provider: openai\nmodel: gpt-4o\nanthropic_api_key: sk-old\n"
"default_mode: run\n",
encoding="utf-8",
)
config = load_config()
assert config.default_mode == "run"
assert not hasattr(config, "provider")
assert not hasattr(config, "model")
assert not hasattr(config, "anthropic_api_key")
# =============================================================================
# Test get/set single values
@@ -364,24 +379,31 @@ class TestLoadSaveReset:
class TestGetSetValues:
def test_get_config_value(self, temp_config_dir, clean_env):
"""Test getting a single config value."""
config = EvoScientistConfig(model="gpt-4o-mini")
config = EvoScientistConfig(log_level="INFO")
save_config(config)
assert get_config_value("model") == "gpt-4o-mini"
assert get_config_value("log_level") == "INFO"
def test_get_config_value_invalid_key(self, temp_config_dir, clean_env):
"""Test getting an invalid key returns None."""
assert get_config_value("nonexistent_key") is None
def test_get_config_value_removed_llm_key(self, temp_config_dir, clean_env):
"""Removed LLM keys are invalid config keys now."""
assert get_config_value("anthropic_api_key") is None
assert set_config_value("anthropic_api_key", "sk-x") is False
assert set_config_value("provider", "openai") is False
assert set_config_value("model", "gpt-4o") is False
def test_set_config_value(self, temp_config_dir, clean_env):
"""Test setting a single config value."""
save_config(EvoScientistConfig())
result = set_config_value("model", "gpt-4o")
result = set_config_value("log_level", "INFO")
assert result is True
config = load_config()
assert config.model == "gpt-4o"
assert config.log_level == "INFO"
def test_set_config_value_invalid_key(self, temp_config_dir, clean_env):
"""Test setting an invalid key returns False."""
@@ -509,13 +531,13 @@ class TestGetSetValues:
def test_list_config(self, temp_config_dir, clean_env):
"""Test listing all config values."""
config = EvoScientistConfig(provider="openai", model="gpt-4o")
config = EvoScientistConfig(default_mode="run", log_level="INFO")
save_config(config)
all_config = list_config()
assert isinstance(all_config, dict)
assert all_config["provider"] == "openai"
assert all_config["model"] == "gpt-4o"
assert all_config["default_mode"] == "run"
assert all_config["log_level"] == "INFO"
# =============================================================================
@@ -528,8 +550,6 @@ class TestPriorityChain:
"""Test defaults are used when nothing is configured."""
# Ensure no env vars affect the test
for key in [
"ANTHROPIC_API_KEY",
"OPENAI_API_KEY",
"TAVILY_API_KEY",
"EVOSCIENTIST_DEFAULT_MODE",
"EVOSCIENTIST_WORKSPACE_DIR",
@@ -537,15 +557,15 @@ class TestPriorityChain:
]:
monkeypatch.delenv(key, raising=False)
config = get_effective_config()
assert config.provider == "anthropic"
assert config.ui_backend == "tui"
assert config.default_mode == "daemon"
def test_file_overrides_defaults(self, temp_config_dir, clean_env):
"""Test file config overrides defaults."""
save_config(EvoScientistConfig(provider="openai"))
save_config(EvoScientistConfig(default_mode="run"))
config = get_effective_config()
assert config.provider == "openai"
assert config.default_mode == "run"
def test_env_overrides_file(self, temp_config_dir, monkeypatch):
"""Test environment variables override file config."""
@@ -564,10 +584,10 @@ class TestPriorityChain:
def test_cli_overrides_file(self, temp_config_dir, clean_env):
"""Test CLI arguments override file config."""
save_config(EvoScientistConfig(model="gpt-4o"))
save_config(EvoScientistConfig(log_level="INFO"))
config = get_effective_config(cli_overrides={"model": "claude-opus-4-8"})
assert config.model == "claude-opus-4-8"
config = get_effective_config(cli_overrides={"log_level": "DEBUG"})
assert config.log_level == "DEBUG"
def test_env_ui_backend_override(self, temp_config_dir, monkeypatch):
"""UI backend can be selected via environment variable."""
@@ -590,6 +610,14 @@ class TestPriorityChain:
config = get_effective_config()
assert config.reasoning_effort == "high"
def test_env_tavily_key_override(self, temp_config_dir, monkeypatch):
"""Tavily key from env overrides file (search tool key, not LLM)."""
save_config(EvoScientistConfig(tavily_api_key="file-key"))
monkeypatch.setenv("TAVILY_API_KEY", "env-key")
config = get_effective_config()
assert config.tavily_api_key == "env-key"
def test_env_channel_debug_tracing_override(self, temp_config_dir, monkeypatch):
"""Channel tracing can be enabled via environment variable."""
save_config(EvoScientistConfig(channel_debug_tracing=False))
@@ -664,47 +692,6 @@ class TestPriorityChain:
assert set_config_value("sandbox_execute_timeout", 600) is True
assert get_config_value("sandbox_execute_timeout") == 600
def test_env_api_key_override(self, temp_config_dir, monkeypatch):
"""Test API keys from env override file."""
save_config(EvoScientistConfig(anthropic_api_key="file-key"))
monkeypatch.setenv("ANTHROPIC_API_KEY", "env-key")
config = get_effective_config()
assert config.anthropic_api_key == "env-key"
def test_env_auth_mode_override(self, temp_config_dir, monkeypatch):
"""Test auth mode from env overrides file."""
save_config(EvoScientistConfig(anthropic_auth_mode="api_key"))
monkeypatch.setenv("EVOSCIENTIST_ANTHROPIC_AUTH_MODE", "oauth")
config = get_effective_config()
assert config.anthropic_auth_mode == "oauth"
def test_env_openai_auth_mode_override(self, temp_config_dir, monkeypatch):
"""Test openai_auth_mode from env overrides file."""
save_config(EvoScientistConfig(openai_auth_mode="api_key"))
monkeypatch.setenv("EVOSCIENTIST_OPENAI_AUTH_MODE", "oauth")
config = get_effective_config()
assert config.openai_auth_mode == "oauth"
def test_env_openrouter_anthropic_prompt_cache_opt_out(
self, temp_config_dir, monkeypatch
):
"""Test OpenRouter Anthropic prompt cache opt-out env overrides file."""
save_config(EvoScientistConfig(openrouter_anthropic_prompt_cache=True))
monkeypatch.setenv("EVOSCIENTIST_OPENROUTER_ANTHROPIC_PROMPT_CACHE", "false")
config = get_effective_config()
assert config.openrouter_anthropic_prompt_cache is False
def test_set_openrouter_anthropic_prompt_cache(self, temp_config_dir, clean_env):
"""Test OpenRouter Anthropic prompt cache can be set through config."""
save_config(EvoScientistConfig())
assert set_config_value("openrouter_anthropic_prompt_cache", "false") is True
assert get_config_value("openrouter_anthropic_prompt_cache") is False
# =============================================================================
# Test apply_config_to_env
@@ -712,51 +699,38 @@ class TestPriorityChain:
class TestApplyConfigToEnv:
def test_applies_api_keys_when_not_set(self, clean_env):
"""Test that API keys are applied to env when not already set."""
config = EvoScientistConfig(
anthropic_api_key="config-ant-key",
openai_api_key="config-oai-key",
tavily_api_key="config-tav-key",
)
def test_applies_tavily_key_when_not_set(self, clean_env):
"""Test that the Tavily key is applied to env when not already set."""
config = EvoScientistConfig(tavily_api_key="config-tav-key")
apply_config_to_env(config)
assert os.environ.get("ANTHROPIC_API_KEY") == "config-ant-key"
assert os.environ.get("OPENAI_API_KEY") == "config-oai-key"
assert os.environ.get("TAVILY_API_KEY") == "config-tav-key"
def test_does_not_override_existing_env(self, monkeypatch):
"""Test that existing env vars are not overridden."""
monkeypatch.setenv("ANTHROPIC_API_KEY", "existing-key")
monkeypatch.setenv("TAVILY_API_KEY", "existing-key")
config = EvoScientistConfig(anthropic_api_key="config-key")
config = EvoScientistConfig(tavily_api_key="config-key")
apply_config_to_env(config)
assert os.environ.get("ANTHROPIC_API_KEY") == "existing-key"
assert os.environ.get("TAVILY_API_KEY") == "existing-key"
def test_empty_config_keys_not_applied(self, clean_env):
def test_empty_config_keys_not_applied(self, clean_env, monkeypatch):
"""Test that empty config keys don't create env vars."""
config = EvoScientistConfig() # All empty
monkeypatch.delenv("EVOSCIENTIST_REASONING_EFFORT", raising=False)
config = EvoScientistConfig(tavily_api_key="")
apply_config_to_env(config)
assert os.environ.get("ANTHROPIC_API_KEY") is None
assert os.environ.get("OPENAI_API_KEY") is None
assert os.environ.get("EVOSCIENTIST_OPENROUTER_ANTHROPIC_PROMPT_CACHE") is None
assert os.environ.get("TAVILY_API_KEY") is None
def test_openrouter_anthropic_prompt_cache_opt_out_applied(
self, clean_env, monkeypatch
):
"""Test OpenRouter Anthropic prompt cache opt-out config is applied to env."""
monkeypatch.delenv(
"EVOSCIENTIST_OPENROUTER_ANTHROPIC_PROMPT_CACHE", raising=False
)
config = EvoScientistConfig(openrouter_anthropic_prompt_cache=False)
def test_reasoning_effort_applied(self, clean_env, monkeypatch):
"""reasoning_effort is round-tripped to env for downstream consumers."""
monkeypatch.delenv("EVOSCIENTIST_REASONING_EFFORT", raising=False)
config = EvoScientistConfig(reasoning_effort="medium")
apply_config_to_env(config)
assert os.environ.get("EVOSCIENTIST_OPENROUTER_ANTHROPIC_PROMPT_CACHE") == (
"false"
)
assert os.environ.get("EVOSCIENTIST_REASONING_EFFORT") == "medium"
def test_dangerous_mode_round_trips_to_env(self, clean_env, monkeypatch):
"""dangerous_mode set via CLI override must survive a fresh re-read.
@@ -796,58 +770,6 @@ class TestApplyConfigToEnv:
apply_config_to_env(EvoScientistConfig()) # dangerous off
assert os.environ.get("EVOSCIENTIST_DANGEROUS_MODE") is None
def test_ollama_base_url_applied(self, clean_env, monkeypatch):
"""Test that ollama_base_url is applied to OLLAMA_BASE_URL env var."""
monkeypatch.delenv("OLLAMA_BASE_URL", raising=False)
config = EvoScientistConfig(ollama_base_url="http://localhost:11434")
apply_config_to_env(config)
assert os.environ.get("OLLAMA_BASE_URL") == "http://localhost:11434"
def test_ollama_base_url_not_overridden(self, monkeypatch):
"""Test that existing OLLAMA_BASE_URL env var is not overridden."""
monkeypatch.setenv("OLLAMA_BASE_URL", "http://existing:11434")
config = EvoScientistConfig(ollama_base_url="http://new:11434")
apply_config_to_env(config)
assert os.environ.get("OLLAMA_BASE_URL") == "http://existing:11434"
class TestAuxiliaryModelConfig:
"""auxiliary_model / auxiliary_provider config fields (plain str, optional)."""
def test_defaults_empty(self):
cfg = EvoScientistConfig()
assert cfg.auxiliary_model == ""
assert cfg.auxiliary_provider == ""
def test_save_and_load_round_trip(self, temp_config_dir, clean_env):
save_config(
EvoScientistConfig(
auxiliary_model="claude-haiku-4-5",
auxiliary_provider="anthropic",
)
)
loaded = load_config()
assert loaded.auxiliary_model == "claude-haiku-4-5"
assert loaded.auxiliary_provider == "anthropic"
def test_get_set_value(self, temp_config_dir, clean_env):
save_config(EvoScientistConfig())
assert set_config_value("auxiliary_model", "qwen3.6-flash") is True
assert set_config_value("auxiliary_provider", "dashscope") is True
assert get_config_value("auxiliary_model") == "qwen3.6-flash"
assert get_config_value("auxiliary_provider") == "dashscope"
def test_env_overrides_file(self, temp_config_dir, monkeypatch):
save_config(EvoScientistConfig(auxiliary_model="claude-haiku-4-5"))
monkeypatch.setenv("EVOSCIENTIST_AUXILIARY_MODEL", "gpt-5.5")
monkeypatch.setenv("EVOSCIENTIST_AUXILIARY_PROVIDER", "openai")
config = get_effective_config()
assert config.auxiliary_model == "gpt-5.5"
assert config.auxiliary_provider == "openai"
def test_scheduler_config_defaults_and_env(monkeypatch):
from EvoScientist.config.settings import EvoScientistConfig, get_effective_config
+89 -23
View File
@@ -4,7 +4,8 @@ Verifies the snapshot-only contract: the middleware resolves the frozen
``ResolvedModelConfig`` for its role from ``configurable["runtime_snapshot_id"]``
and builds the model via ``build_chat_model``; ``model``/``model_provider``
in ``configurable`` are rejected with ``MODEL_CONFIG_OUTSIDE_SNAPSHOT``; a
run without a snapshot ID passes through unchanged.
run without a snapshot ID gets a lazily created local snapshot bound to its
thread (section 8.1), or fails closed when no thread ID exists either.
"""
from __future__ import annotations
@@ -19,15 +20,18 @@ from langchain_openai import ChatOpenAI
from EvoScientist.middleware.configurable_model import (
ConfigurableModelMiddleware,
check_no_outside_snapshot_model_config,
ensure_snapshot_binding,
read_snapshot_binding,
)
from EvoScientist.model_registry.errors import (
MODEL_CONFIG_OUTSIDE_SNAPSHOT,
MODEL_REGISTRY_NOT_READY,
SNAPSHOT_EXPIRED,
SNAPSHOT_NOT_FOUND,
ModelRegistryError,
)
from EvoScientist.model_registry.runtime import SnapshotRuntime
from EvoScientist.model_registry.store import ModelRuntimeStore
from tests.registry_fixtures import (
ZHIPU_SECRET,
make_active_store,
@@ -178,42 +182,102 @@ class TestReadSnapshotBinding:
# =============================================================================
# 3. Pass-through behavior (no snapshot in configurable)
# 3. Fail-closed behavior (no snapshot in configurable, section 8.1)
# =============================================================================
class TestPassThrough:
def test_sync_no_snapshot_passes_request_unchanged(self, runtime):
class TestFailClosed:
def test_sync_no_snapshot_no_thread_rejected(self, runtime):
mw = ConfigurableModelMiddleware(runtime=runtime)
req = _make_request()
sentinel = object()
handler = MagicMock(return_value=sentinel)
with _patched_config({}):
result = mw.wrap_model_call(req, handler)
assert result is sentinel
handler.assert_called_once_with(req)
handler = MagicMock()
with (
_patched_config({}),
pytest.raises(ModelRegistryError) as excinfo,
):
mw.wrap_model_call(req, handler)
assert excinfo.value.code == MODEL_CONFIG_OUTSIDE_SNAPSHOT
handler.assert_not_called()
req.override.assert_not_called()
async def test_async_no_snapshot_passes_request_unchanged(self, runtime):
async def test_async_no_snapshot_no_thread_rejected(self, runtime):
mw = ConfigurableModelMiddleware(runtime=runtime)
req = _make_request()
async def handler(r):
assert r is req
return "ok"
raise AssertionError("handler must not run") # pragma: no cover
with _patched_config({}):
result = await mw.awrap_model_call(req, handler)
assert result == "ok"
with (
_patched_config({}),
pytest.raises(ModelRegistryError) as excinfo,
):
await mw.awrap_model_call(req, handler)
assert excinfo.value.code == MODEL_CONFIG_OUTSIDE_SNAPSHOT
req.override.assert_not_called()
def test_outside_runnable_context_passes_through(self, runtime):
def test_outside_runnable_context_fails_closed(self, runtime):
"""No runnable context means no thread binding — never pass through."""
mw = ConfigurableModelMiddleware(runtime=runtime)
req = _make_request()
handler = MagicMock(return_value="ok")
with _patched_config(None):
handler = MagicMock()
with (
_patched_config(None),
pytest.raises(ModelRegistryError) as excinfo,
):
mw.wrap_model_call(req, handler)
handler.assert_called_once_with(req)
assert excinfo.value.code == MODEL_CONFIG_OUTSIDE_SNAPSHOT
handler.assert_not_called()
def test_bootstrap_registry_fails_closed_not_ready(self, tmp_path):
bootstrap_runtime = SnapshotRuntime(
ModelRuntimeStore(config_dir=tmp_path / "model-runtime")
)
mw = ConfigurableModelMiddleware(runtime=bootstrap_runtime)
req = _make_request()
with (
_patched_config({"thread_id": "cli-thread"}),
pytest.raises(ModelRegistryError) as excinfo,
):
mw.wrap_model_call(req, MagicMock())
assert excinfo.value.code == MODEL_REGISTRY_NOT_READY
class TestLazyLocalSnapshot:
def test_thread_without_snapshot_lazily_creates_one(self, store, runtime):
mw = ConfigurableModelMiddleware(runtime=runtime)
req = _make_request()
captured = {}
def handler(r):
captured["model"] = r.model
return "ok"
with _patched_config({"thread_id": "cron-thread-1"}):
assert mw.wrap_model_call(req, handler) == "ok"
# The override carries the registry-default primary model.
assert captured["model"] is not req.model
# A snapshot was created and bound to the run's own thread.
found = store.find_active_run_snapshot(
deployment_id=runtime.local_deployment_id,
thread_id="cron-thread-1",
run_request_id="auto:cron-thread-1",
)
assert found is not None
def test_lazy_creation_converges_within_a_thread(self, store, runtime):
"""Every middleware in the run lands on the same lazily created snapshot."""
first = ensure_snapshot_binding({"thread_id": "t-1"}, runtime)
second = ensure_snapshot_binding({"thread_id": "t-1"}, runtime)
assert first == second
assert first[1] == runtime.local_deployment_id
def test_explicit_snapshot_wins_over_lazy_creation(self, store, runtime):
snapshot = make_snapshot(store)
binding = ensure_snapshot_binding(
{"runtime_snapshot_id": snapshot.snapshot_id, "thread_id": "other"},
runtime,
)
assert binding[0] == snapshot.snapshot_id
# =============================================================================
@@ -455,7 +519,7 @@ class TestRunnableContextVarIntegration:
assert isinstance(model, ChatOpenAI)
assert model.model_name == "glm-5.2"
def test_real_contextvar_unset_passes_through(self, runtime):
def test_real_contextvar_unset_fails_closed(self, runtime):
from langchain_core.runnables.config import var_child_runnable_config
# Defensive: ensure no leftover contextvar from another test.
@@ -464,9 +528,11 @@ class TestRunnableContextVarIntegration:
mw = ConfigurableModelMiddleware(runtime=runtime)
req = _make_request()
handler = MagicMock(return_value="ok")
mw.wrap_model_call(req, handler)
with pytest.raises(ModelRegistryError) as excinfo:
mw.wrap_model_call(req, handler)
finally:
var_child_runnable_config.reset(token)
handler.assert_called_once_with(req)
assert excinfo.value.code == MODEL_CONFIG_OUTSIDE_SNAPSHOT
handler.assert_not_called()
req.override.assert_not_called()
+4 -2
View File
@@ -107,9 +107,10 @@ def test_create_middleware_model_none_fallback(mock_model):
"EvoScientist.middleware.create_tool_selector_middleware",
return_value=[MagicMock(), MagicMock()],
)
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_default_middleware_includes_context_editing(mock_config, mock_model, mock_ts):
def test_default_middleware_includes_context_editing(mock_config, mock_model, mock_aux, mock_ts):
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
cfg = MagicMock()
cfg.enable_ask_user = False
@@ -143,9 +144,10 @@ def test_inject_subagent_includes_context_editing(mock_model):
"EvoScientist.middleware.create_tool_selector_middleware",
return_value=[MagicMock(), MagicMock()],
)
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_context_editing_before_overflow_mapper(mock_config, mock_model, mock_ts):
def test_context_editing_before_overflow_mapper(mock_config, mock_model, mock_aux, mock_ts):
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
cfg = MagicMock()
cfg.enable_ask_user = False
+16 -7
View File
@@ -29,6 +29,13 @@ from tests.fakes import (
)
@pytest.fixture(autouse=True)
def _active_registry(active_snapshot_runtime):
"""Run entry points freeze a local snapshot per run; that fails closed
against the default bootstrap registry, so activate it for these tests."""
return active_snapshot_runtime
async def test_local_gateway_streams_from_injected_streamer():
seen: dict[str, Any] = {}
@@ -743,13 +750,15 @@ async def test_langgraph_server_gateway_streams_root_protocol_events():
assert update_metadata["workspace_dir"] == "/tmp/ws"
assert isinstance(update_metadata["updated_at"], str)
assert threads.stream_calls == [("abc12345", "writing-agent")]
assert stream.run.starts == [
{
"input": {"messages": [{"role": "user", "content": "hi"}]},
"config": {"configurable": {"thread_id": "abc12345"}},
"metadata": {"workspace_dir": "/tmp/ws"},
}
]
assert len(stream.run.starts) == 1
start = stream.run.starts[0]
assert start["input"] == {"messages": [{"role": "user", "content": "hi"}]}
assert start["metadata"] == {"workspace_dir": "/tmp/ws"}
configurable = start["config"]["configurable"]
assert configurable["thread_id"] == "abc12345"
# The gateway freezes a local run snapshot per run (design doc 8.1).
assert isinstance(configurable["runtime_snapshot_id"], str)
assert set(configurable) == {"thread_id", "runtime_snapshot_id"}
assert events == [
{"type": "text", "content": "hello"},
{"type": "done", "content": "hello", "response": "hello"},
+22 -665
View File
@@ -1,11 +1,14 @@
"""Smoke tests for the legacy admin routes mounted via langgraph.json's
``http`` field. We test the Starlette app directly — no need to spin up
langgraph dev.
"""Smoke tests for the custom routes mounted via langgraph.json's ``http``
field. We test the Starlette app directly — no need to spin up langgraph dev.
``GET /api/models`` and ``POST /api/runtime-snapshots`` now belong to the
unified model registry API (``EvoScientist.model_registry.http_api``); their
contract tests live in ``tests/test_model_registry_http.py`` and
``tests/test_delegation_auth.py``.
``GET /api/models`` and ``POST /api/runtime-snapshots`` belong to the unified
model registry API (``EvoScientist.model_registry.http_api``); their contract
tests live in ``tests/test_model_registry_http.py`` and
``tests/test_delegation_auth.py``. The legacy provider routes
(``/api/provider-profiles``, ``/api/provider-actions``, ``/api/config``,
``/api/default-model``) and the ``x-evoscientist-admin-token`` check were
removed with the unified model configuration refactor; the first test pins
down that they are gone.
"""
from __future__ import annotations
@@ -16,674 +19,28 @@ import pytest
from langchain_core.messages import AIMessage, HumanMessage
from starlette.testclient import TestClient
from EvoScientist.config import EvoScientistConfig, load_config, save_config
from EvoScientist.config.provider_profiles import (
load_provider_profiles,
)
from EvoScientist.langgraph_dev.http import app
from EvoScientist.llm.provider_operations import (
DiscoveredProviderModel,
ProviderModelTestResult,
ProviderOperationError,
)
client = TestClient(app)
@pytest.fixture(autouse=True)
def isolate_xdg_config(tmp_path, monkeypatch):
"""Keep HTTP tests independent from the developer's saved providers."""
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
def test_provider_profiles_api_requires_admin_token_header(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
response = client.get("/api/provider-profiles")
assert response.status_code == 403
def test_provider_profiles_api_round_trip_redacts_secret(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
payload = {
"providers": [
{
"id": "lab-openai",
"name": "Lab OpenAI",
"adapter": "openai-compatible",
"base_url": "https://llm.example.test/v1",
"api_key": "provider-secret",
"enabled": True,
"runtime": {"timeout_seconds": 90, "max_retries": 1},
"models": [
{
"id": "lab-model",
"name": "Lab Model",
"model_id": "vendor/model",
"enabled": True,
"runtime": {
"limit_mode": "combined",
"context_window_tokens": 32_768,
"max_output_tokens": 4_096,
"min_effective_input_tokens": 4_096,
"limits_status": "confirmed",
"limits_source": "provider",
},
}
],
}
]
}
put_response = client.put("/api/provider-profiles", headers=headers, json=payload)
assert put_response.status_code == 200
assert "provider-secret" not in put_response.text
assert put_response.json()["providers"][0]["api_key_configured"] is True
assert "openai" in put_response.json()["reserved_provider_ids"]
get_response = client.get("/api/provider-profiles", headers=headers)
assert get_response.status_code == 200
assert get_response.json()["providers"][0]["models"][0]["id"] == "lab-model"
assert "provider-secret" not in get_response.text
def test_llm_config_api_requires_admin_token_header(monkeypatch):
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
response = client.get("/api/config")
assert response.status_code == 403
def test_llm_config_api_redacts_secrets_and_reports_env_overrides(monkeypatch):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
monkeypatch.setenv("OPENAI_API_KEY", "environment-secret")
save_config(
EvoScientistConfig(
provider="openai",
model="gpt-5.4",
openai_api_key="file-secret-1234",
default_workdir="/tmp/research",
)
)
response = client.get(
"/api/config",
headers={"X-EvoScientist-Admin-Token": "admin-secret"},
)
assert response.status_code == 200
body = response.json()
assert body["values"]["provider"] == "openai"
assert body["values"]["model"] == "gpt-5.4"
assert "openai_api_key" not in body["values"]
assert body["secrets"]["openai_api_key"] == {
"configured": True,
"hint": "...1234",
}
assert body["env_overrides"]["openai_api_key"] == "OPENAI_API_KEY"
assert "file-secret-1234" not in response.text
assert "environment-secret" not in response.text
assert "default_workdir" not in body["values"]
assert body["model_catalog"] is None
assert body["builtin_model_candidates"]
openai = next(
provider for provider in body["builtin_providers"] if provider["id"] == "openai"
)
assert openai["managed"] is False
assert openai["api_key_configured"] is True
assert "environment-secret" not in response.text
def test_llm_config_api_saves_builtin_registry_without_rewriting_legacy_secret(
monkeypatch,
):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
save_config(
EvoScientistConfig(
provider="openai",
model="chat-main",
openai_api_key="legacy-secret",
default_workdir="/tmp/research",
)
)
loaded = client.get("/api/config", headers=headers).json()
openai = next(
provider
for provider in loaded["builtin_providers"]
if provider["id"] == "openai"
)
openai.update(
{
"managed": True,
"base_url": "https://proxy.example.test/v1",
"models": [
{
"id": "chat-main",
"name": "Chat Main",
"model_id": "gpt-upstream",
"enabled": True,
}
],
}
)
response = client.patch(
"/api/config",
headers=headers,
json={
"revision": loaded["revision"],
"values": loaded["values"],
"secrets": {},
"clear_secrets": [],
"builtin_providers": [openai],
},
)
assert response.status_code == 200
body = response.json()
assert body["restart_required"] is False
assert "builtin_providers" in body["changed_fields"]
saved_profile = load_provider_profiles().builtins[0]
assert saved_profile.id == "openai"
assert saved_profile.api_key == "legacy-secret"
assert saved_profile.base_url == "https://proxy.example.test/v1"
saved_config = load_config()
assert saved_config.openai_api_key == "legacy-secret"
assert saved_config.default_workdir == "/tmp/research"
assert saved_config.model_catalog is None
def test_first_builtin_registry_save_migrates_legacy_default_model(monkeypatch):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://environment.example.test")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
save_config(EvoScientistConfig(provider="anthropic", model="claude-sonnet-4-6"))
loaded = client.get("/api/config", headers=headers).json()
ollama = next(
provider
for provider in loaded["builtin_providers"]
if provider["id"] == "ollama"
)
ollama.update(
{
"managed": True,
"base_url": "http://127.0.0.1:11434",
"models": [],
}
)
response = client.patch(
"/api/config",
headers=headers,
json={
"revision": loaded["revision"],
"values": loaded["values"],
"secrets": {},
"clear_secrets": [],
"builtin_providers": [ollama],
},
)
assert response.status_code == 200
builtins = {profile.id: profile for profile in load_provider_profiles().builtins}
assert set(builtins) == {"anthropic", "ollama"}
assert builtins["anthropic"].models[0].id == "claude-sonnet-4-6"
assert builtins["anthropic"].base_url == ""
def test_llm_config_api_persists_model_catalog(monkeypatch):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
save_config(EvoScientistConfig(provider="openai", model="chat-main"))
revision = client.get("/api/config", headers=headers).json()["revision"]
catalog = [
{
"provider": "openai",
"id": "chat-main",
"name": "Chat Main",
"model_id": "gpt-upstream",
"enabled": True,
}
]
response = client.patch(
"/api/config",
headers=headers,
json={
"revision": revision,
"values": {},
"secrets": {},
"clear_secrets": [],
"model_catalog": catalog,
},
)
assert response.status_code == 200
assert response.json()["model_catalog"] == catalog
assert response.json()["restart_required"] is False
assert load_config().model_catalog == catalog
def test_llm_config_api_rejects_default_outside_model_catalog(monkeypatch):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
save_config(EvoScientistConfig(provider="openai", model="gpt-default"))
revision = client.get("/api/config", headers=headers).json()["revision"]
response = client.patch(
"/api/config",
headers=headers,
json={
"revision": revision,
"values": {},
"secrets": {},
"clear_secrets": [],
"model_catalog": [
{
"provider": "openai",
"id": "different-model",
"name": "Different",
"model_id": "gpt-different",
"enabled": True,
}
],
},
)
assert response.status_code == 400
assert "Default model" in response.json()["error"]
assert load_config().model_catalog is None
def test_llm_config_api_patch_preserves_unrelated_fields_and_secret_by_default(
monkeypatch,
):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
save_config(
EvoScientistConfig(
openai_api_key="saved-openai-key",
anthropic_api_key="saved-anthropic-key",
default_workdir="/tmp/research",
)
)
revision = client.get("/api/config", headers=headers).json()["revision"]
response = client.patch(
"/api/config",
headers=headers,
json={
"revision": revision,
"values": {
"provider": "openai",
"model": "gpt-5.4",
"ollama_base_url": "http://127.0.0.1:11434",
},
"secrets": {"openai_api_key": "replacement-openai-key"},
"clear_secrets": ["anthropic_api_key"],
},
)
assert response.status_code == 200
body = response.json()
assert body["restart_required"] is True
assert set(body["changed_fields"]) == {
"anthropic_api_key",
"model",
"ollama_base_url",
"openai_api_key",
"provider",
}
assert "replacement-openai-key" not in response.text
saved = load_config()
assert saved.provider == "openai"
assert saved.model == "gpt-5.4"
assert saved.ollama_base_url == "http://127.0.0.1:11434"
assert saved.openai_api_key == "replacement-openai-key"
assert saved.anthropic_api_key == ""
assert saved.default_workdir == "/tmp/research"
def test_llm_config_api_rejects_stale_revision(monkeypatch):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
save_config(EvoScientistConfig(model="claude-sonnet-4-6"))
revision = client.get("/api/config", headers=headers).json()["revision"]
save_config(EvoScientistConfig(model="gpt-5.4", provider="openai"))
response = client.patch(
"/api/config",
headers=headers,
json={
"revision": revision,
"values": {"model": "claude-opus-4-8"},
},
)
assert response.status_code == 409
assert "Reload and try again" in response.json()["error"]
assert load_config().model == "gpt-5.4"
@pytest.mark.parametrize(
("payload", "message"),
("method", "path"),
[
(
{"values": {"openai_auth_mode": "password"}},
"openai_auth_mode must be 'api_key' or 'oauth'",
),
(
{"values": {"default_workdir": "/tmp/other"}},
"Unsupported config fields: default_workdir",
),
(
{"values": {"ollama_base_url": "localhost:11434"}},
"ollama_base_url must use http:// or https://",
),
("GET", "/api/provider-profiles"),
("PUT", "/api/provider-profiles"),
("GET", "/api/config"),
("PATCH", "/api/config"),
("POST", "/api/config"),
("POST", "/api/provider-actions"),
("PUT", "/api/default-model"),
],
)
def test_llm_config_api_validates_updates(monkeypatch, payload, message):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
def test_legacy_provider_routes_are_gone(method, path):
"""The removed routes must not exist — even with the old admin token."""
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
revision = client.get("/api/config", headers=headers).json()["revision"]
response = client.patch(
"/api/config",
headers=headers,
json={"revision": revision, **payload},
)
assert response.status_code == 400
assert message in response.json()["error"]
def test_provider_actions_api_requires_admin_token_header(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
response = client.post("/api/provider-actions", json={})
assert response.status_code == 403
def test_provider_actions_lists_models_with_saved_api_key(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
provider = {
"id": "lab-openai",
"name": "Lab OpenAI",
"adapter": "openai-compatible",
"base_url": "https://llm.example.test/v1",
"api_key": "provider-secret",
"enabled": True,
"models": [],
}
assert (
client.put(
"/api/provider-profiles",
headers=headers,
json={"providers": [provider]},
).status_code
== 200
)
provider["api_key"] = ""
async def fake_discover(profile):
assert profile.api_key == "provider-secret"
return [DiscoveredProviderModel("vendor/model", "Vendor Model")]
with patch(
"EvoScientist.langgraph_dev.http.discover_provider_models",
new=fake_discover,
):
response = client.post(
"/api/provider-actions",
headers=headers,
json={"action": "list_models", "provider": provider},
)
assert response.status_code == 200
assert response.json() == {
"models": [{"model_id": "vendor/model", "name": "Vendor Model"}]
}
def test_provider_actions_tests_model(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
async def fake_test(profile, model):
assert profile.adapter == "grok"
assert model.model_id == "grok-4"
return ProviderModelTestResult(latency_ms=123, response="OK")
with patch("EvoScientist.langgraph_dev.http.test_provider_model", new=fake_test):
response = client.post(
"/api/provider-actions",
headers=headers,
json={
"action": "test_model",
"provider": {
"id": "lab-grok",
"name": "Lab Grok",
"adapter": "grok",
"base_url": "",
"api_key": "xai-secret",
"enabled": True,
"models": [],
},
"model": {
"id": "grok",
"name": "Grok",
"model_id": "grok-4",
},
},
)
assert response.status_code == 200
assert response.json() == {"ok": True, "latency_ms": 123, "response": "OK"}
def test_provider_actions_maps_provider_failures_to_bad_gateway(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
async def fake_discover(_profile):
raise ProviderOperationError("Provider returned HTTP 401")
with patch(
"EvoScientist.langgraph_dev.http.discover_provider_models",
new=fake_discover,
):
response = client.post(
"/api/provider-actions",
headers={"X-EvoScientist-Admin-Token": "admin-secret"},
json={
"action": "list_models",
"provider": {
"id": "lab-openai",
"name": "Lab OpenAI",
"adapter": "openai-compatible",
"base_url": "https://llm.example.test/v1",
"api_key": "bad-key",
"enabled": True,
"models": [],
},
},
)
assert response.status_code == 502
assert response.json() == {"error": "Provider returned HTTP 401"}
def test_llm_config_action_discovers_models_with_effective_builtin_secret(
monkeypatch,
):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
monkeypatch.setenv("OPENAI_API_KEY", "environment-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
async def fake_discover(profile):
assert profile.id == "openai"
assert profile.adapter == "openai"
assert profile.api_key == "environment-secret"
return [DiscoveredProviderModel("gpt-discovered", "GPT Discovered")]
with patch(
"EvoScientist.langgraph_dev.http.discover_provider_models",
new=fake_discover,
):
response = client.post(
"/api/config",
headers=headers,
json={"action": "list_models", "provider": {"id": "openai"}},
)
assert response.status_code == 200
assert response.json() == {
"models": [{"model_id": "gpt-discovered", "name": "GPT Discovered"}]
}
def test_llm_config_action_tests_builtin_model_with_draft_connection(monkeypatch):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
async def fake_test(profile, model):
assert profile.id == "custom-openai"
assert profile.adapter == "openai-compatible"
assert profile.base_url == "https://proxy.example.test/v1"
assert profile.api_key == "draft-secret"
assert model.model_id == "vendor/model"
return ProviderModelTestResult(latency_ms=42, response="OK")
with patch("EvoScientist.langgraph_dev.http.test_provider_model", new=fake_test):
response = client.post(
"/api/config",
headers=headers,
json={
"action": "test_model",
"provider": {
"id": "custom-openai",
"base_url": "https://proxy.example.test/v1",
"api_key": "draft-secret",
},
"model": {
"id": "chat-main",
"name": "Chat Main",
"model_id": "vendor/model",
},
},
)
assert response.status_code == 200
assert response.json() == {"ok": True, "latency_ms": 42, "response": "OK"}
def test_default_model_api_requires_admin_token_header(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
response = client.put(
"/api/default-model",
json={"model": "gpt-5.4", "provider": "openai"},
)
assert response.status_code == 403
def test_default_model_api_persists_pair_and_preserves_config(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
save_config(EvoScientistConfig(default_workdir="/tmp/research"))
response = client.put(
"/api/default-model",
headers={"X-EvoScientist-Admin-Token": "admin-secret"},
json={"model": "gpt-5.4", "provider": "openai"},
)
assert response.status_code == 200
assert response.json() == {"default": {"name": "gpt-5.4", "provider": "openai"}}
saved = load_config()
assert saved.model == "gpt-5.4"
assert saved.provider == "openai"
assert saved.default_workdir == "/tmp/research"
def test_default_model_api_accepts_configured_dynamic_model(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
provider_payload = {
"providers": [
{
"id": "lab-openai",
"name": "Lab OpenAI",
"adapter": "openai-compatible",
"base_url": "https://llm.example.test/v1",
"api_key": "provider-secret",
"enabled": True,
"models": [
{
"id": "lab-model",
"name": "Lab Model",
"model_id": "vendor/model",
"enabled": True,
}
],
}
]
}
assert (
client.put(
"/api/provider-profiles", headers=headers, json=provider_payload
).status_code
== 200
)
response = client.put(
"/api/default-model",
headers=headers,
json={"model": "lab-model", "provider": "lab-openai"},
)
assert response.status_code == 200
saved = load_config()
assert (saved.model, saved.provider) == ("lab-model", "lab-openai")
def test_default_model_api_rejects_unconfigured_pair(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
response = client.put(
"/api/default-model",
headers={"X-EvoScientist-Admin-Token": "admin-secret"},
json={"model": "missing-model", "provider": "missing-provider"},
)
assert response.status_code == 400
assert "is not configured" in response.json()["error"]
def test_default_model_api_rejects_invalid_json(monkeypatch):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
response = client.put(
"/api/default-model",
headers={
"X-EvoScientist-Admin-Token": "admin-secret",
"Content-Type": "application/json",
},
content="{invalid",
)
assert response.status_code == 400
assert response.json() == {"error": "Request body must be valid JSON."}
response = client.request(method, path, headers=headers, json={})
assert response.status_code == 404
def test_final_answer_extracts_latest_ai_text_blocks():
+97
View File
@@ -0,0 +1,97 @@
"""Tests for ``EvoScientist.config.legacy_artifacts`` (design doc §10 step 4).
Startup must refuse — with an explicit reset guide — when any pre-Registry
model configuration artifact remains: an old ``providers.yaml``, an old
``run-runtime-snapshots.sqlite3``, or leftover LLM fields in
``config.yaml``. Platform fields in ``config.yaml`` are fine.
"""
from __future__ import annotations
import pytest
import yaml
from EvoScientist.config.legacy_artifacts import (
LEGACY_CONFIG_YAML_KEYS,
LegacyArtifactsError,
assert_no_legacy_artifacts,
find_legacy_artifacts,
)
def test_clean_config_dir_passes(tmp_path):
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"workspace_dir": "/tmp/ws", "langgraph_port": 2024}),
encoding="utf-8",
)
assert find_legacy_artifacts(tmp_path) == []
assert_no_legacy_artifacts(tmp_path) # must not raise
def test_missing_config_yaml_passes(tmp_path):
assert find_legacy_artifacts(tmp_path) == []
assert_no_legacy_artifacts(tmp_path)
def test_legacy_providers_yaml_refused(tmp_path):
(tmp_path / "providers.yaml").write_text("providers: []\n", encoding="utf-8")
found = find_legacy_artifacts(tmp_path)
assert any("providers.yaml" in item for item in found)
with pytest.raises(LegacyArtifactsError) as excinfo:
assert_no_legacy_artifacts(tmp_path)
message = str(excinfo.value)
assert "providers.yaml" in message
# The reset guide is part of the refusal (section 10 step 4).
assert "Delete" in message
assert "Model Registry" in message
def test_legacy_snapshot_database_refused(tmp_path):
(tmp_path / "run-runtime-snapshots.sqlite3").touch()
found = find_legacy_artifacts(tmp_path)
assert any("run-runtime-snapshots.sqlite3" in item for item in found)
with pytest.raises(LegacyArtifactsError, match="run-runtime-snapshots"):
assert_no_legacy_artifacts(tmp_path)
def test_leftover_llm_fields_in_config_yaml_refused(tmp_path):
(tmp_path / "config.yaml").write_text(
yaml.safe_dump(
{
"workspace_dir": "/tmp/ws",
"model": "glm-5.2",
"zhipu_api_key": "sk-old",
"model_fallbacks": ["gpt-5"],
}
),
encoding="utf-8",
)
found = find_legacy_artifacts(tmp_path)
assert len(found) == 1
assert "model" in found[0]
assert "zhipu_api_key" in found[0]
with pytest.raises(LegacyArtifactsError) as excinfo:
assert_no_legacy_artifacts(tmp_path)
assert "zhipu_api_key" in str(excinfo.value)
def test_all_artifacts_reported_together(tmp_path):
(tmp_path / "providers.yaml").touch()
(tmp_path / "run-runtime-snapshots.sqlite3").touch()
(tmp_path / "config.yaml").write_text(
yaml.safe_dump({"provider": "zhipu"}), encoding="utf-8"
)
found = find_legacy_artifacts(tmp_path)
assert len(found) == 3
def test_invalid_config_yaml_not_misread_as_legacy(tmp_path):
(tmp_path / "config.yaml").write_text("{{{{ not yaml", encoding="utf-8")
# Unparseable YAML yields no key evidence — other layers report it.
assert find_legacy_artifacts(tmp_path) == []
def test_legacy_key_set_covers_cutover_fields():
# Spot-check the section-10 cutover list stays comprehensive.
for key in ("provider", "model", "auxiliary_model", "anthropic_api_key"):
assert key in LEGACY_CONFIG_YAML_KEYS
+5 -1075
View File
File diff suppressed because it is too large Load Diff
+177 -63
View File
@@ -1,13 +1,11 @@
"""Tests for message-only context budgeting (design doc 6.5, 8.3).
"""Tests for message-only context budgeting (design doc 6.5, 8.1, 8.3).
Covers both budget modes:
- Snapshot mode: input limit and fixed reserves come from the frozen
``ResolvedModelConfig.budget``; the per-call budget is recomputed on every
invocation with the current has_tools/has_attachments mode; the
summarizer model resolves through the snapshot's ``summary`` role.
- Interim local mode (no snapshot ID): limits from the compile-time model
profile with the default reserve policy.
Every budget comes from the frozen ``ResolvedModelConfig.budget`` of the
run snapshot — the middleware's own ``snapshot_role`` selects which frozen
configuration sizes the budget (section 6.1 role mapping). Runs without an
explicit ``runtime_snapshot_id`` lazily create a local snapshot bound to
their thread (section 8.1); a bootstrap registry fails closed with
``MODEL_REGISTRY_NOT_READY``. There is no interim 32K/profile fallback.
"""
from __future__ import annotations
@@ -21,14 +19,27 @@ from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from langchain_ollama import ChatOllama
from EvoScientist.middleware.message_budget import (
ContextBudgetUnsatisfiableError,
MessageReservePolicy,
_snapshot_message_budget,
count_message_text_tokens,
create_message_budget_middleware,
)
from EvoScientist.model_registry.errors import (
MODEL_CONFIG_OUTSIDE_SNAPSHOT,
MODEL_REGISTRY_NOT_READY,
SNAPSHOT_NOT_FOUND,
ModelRegistryError,
)
from EvoScientist.model_registry.runtime import SnapshotRuntime
from tests.registry_fixtures import make_active_store, make_snapshot
from EvoScientist.model_registry.schemas import CredentialWrite, RegistryV4
from EvoScientist.model_registry.store import ModelRuntimeStore
from tests.registry_fixtures import (
ZHIPU_SECRET,
_verify_model,
make_active_store,
make_snapshot,
ollama_provider_payload,
zhipu_provider_payload,
)
@contextmanager
@@ -75,6 +86,39 @@ def _configurable_for(snapshot, **overrides):
return configurable
def _make_split_limit_store(config_dir):
"""Active store whose auxiliary model has a smaller input limit.
primary (zhipu-glm/glm-5.2): context 1048576 → resolved_input_limit 1015808
auxiliary (local-ollama/qwen3): context 65536 → resolved_input_limit 32768
"""
store = ModelRuntimeStore(config_dir=config_dir)
payload = {
"version": 4,
"revision": 1,
"state": "bootstrap",
"defaults": {
"primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"},
"auxiliary": {"provider_id": "local-ollama", "model_key": "qwen3"},
},
"providers": [
zhipu_provider_payload(),
ollama_provider_payload(context_window_tokens=65536),
],
}
registry = store.save_registry(
expected_revision=1,
registry=RegistryV4.model_validate(payload),
credential_writes=[
CredentialWrite(credential_id="zhipu-primary", secret_value=ZHIPU_SECRET)
],
)
assert registry.state == "active"
_verify_model(store, registry, "zhipu-glm", "glm-5.2")
_verify_model(store, registry, "local-ollama", "qwen3")
return store
def test_text_counter_excludes_attachment_payloads_and_counts_tool_results():
messages = [
HumanMessage(
@@ -105,7 +149,7 @@ class TestSnapshotMessageBudget:
def test_base_mode_deducts_only_system_reserve(self, store):
snapshot = make_snapshot(store)
budget = _snapshot_message_budget(
snapshot, has_tools=False, has_attachments=False
snapshot, "primary", has_tools=False, has_attachments=False
)
message_budget = 1015808 - 4096
assert budget.input_limit == 1015808
@@ -116,12 +160,32 @@ class TestSnapshotMessageBudget:
def test_tools_and_attachments_deduct_their_reserves(self, store):
snapshot = make_snapshot(store)
tools_only = _snapshot_message_budget(
snapshot, has_tools=True, has_attachments=False
snapshot, "primary", has_tools=True, has_attachments=False
)
full = _snapshot_message_budget(
snapshot, "primary", has_tools=True, has_attachments=True
)
full = _snapshot_message_budget(snapshot, has_tools=True, has_attachments=True)
assert tools_only.hard_tokens == int((1015808 - 4096 - 8192) * 0.90)
assert full.hard_tokens == int((1015808 - 4096 - 8192 - 4096) * 0.90)
def test_auxiliary_role_uses_auxiliary_frozen_limits(self, tmp_path):
"""T6 review fix: the budget role is threaded through, not hardcoded
to ``primary`` — a scheduler (auxiliary role) must size its budget
from the auxiliary model's frozen configuration."""
store = _make_split_limit_store(tmp_path / "model-runtime")
snapshot = make_snapshot(store)
primary_budget = _snapshot_message_budget(
snapshot, "primary", has_tools=False, has_attachments=False
)
auxiliary_budget = _snapshot_message_budget(
snapshot, "auxiliary", has_tools=False, has_attachments=False
)
assert primary_budget.input_limit == 1015808
assert auxiliary_budget.input_limit == 32768
assert auxiliary_budget.hard_tokens == int((32768 - 4096) * 0.90)
class TestSnapshotModeMiddleware:
def test_budget_uses_frozen_reserves_not_model_profile(self, store, runtime):
@@ -190,12 +254,33 @@ class TestSnapshotModeMiddleware:
assert budget.has_tools is False
assert budget.hard_tokens == int((1015808 - 4096) * 0.90)
def test_snapshot_binding_is_verified(self, store, runtime):
from EvoScientist.model_registry.errors import (
SNAPSHOT_NOT_FOUND,
ModelRegistryError,
def test_auxiliary_snapshot_role_sizes_budget_from_auxiliary(self, tmp_path):
"""T6 review fix: ``snapshot_role="auxiliary"`` (scheduler) sizes the
budget from the snapshot's auxiliary frozen configuration."""
store = _make_split_limit_store(tmp_path / "model-runtime")
runtime = SnapshotRuntime(store)
middleware = create_message_budget_middleware(
MagicMock(),
MagicMock(),
has_tools=False,
snapshot_role="auxiliary",
runtime=runtime,
)
snapshot = make_snapshot(store)
with _patched_config(_configurable_for(snapshot)):
budget = middleware._budget_for_request(_request([HumanMessage("hi")]))
assert budget.input_limit == 32768
assert budget.hard_tokens == int((32768 - 4096) * 0.90)
def test_unknown_snapshot_role_rejected(self, runtime):
with pytest.raises(ValueError, match="Unknown model role"):
create_message_budget_middleware(
MagicMock(), MagicMock(), snapshot_role="bogus", runtime=runtime
)
def test_snapshot_binding_is_verified(self, store, runtime):
middleware = create_message_budget_middleware(
MagicMock(), MagicMock(), runtime=runtime
)
@@ -232,59 +317,70 @@ class TestSnapshotModeMiddleware:
# =============================================================================
# Interim local mode (no snapshot ID in configurable)
# Local snapshot entry (section 8.1): no explicit snapshot ID
# =============================================================================
class TestInterimMode:
def test_budget_from_compile_time_profile(self, runtime):
model = MagicMock()
model.profile = {"max_input_tokens": 32_768}
class TestLocalSnapshotEntry:
def test_missing_snapshot_lazily_creates_one_bound_to_the_thread(
self, store, runtime
):
middleware = create_message_budget_middleware(
model, MagicMock(), has_tools=False, runtime=runtime
MagicMock(), MagicMock(), has_tools=False, runtime=runtime
)
with _patched_config({}):
budget = middleware._budget_for_request(_request([HumanMessage("hi")]))
policy = MessageReservePolicy()
assert budget.hard_tokens == policy.hard_budget(32_768, has_tools=False)
def test_rejects_when_reserves_exceed_model_limit(self, runtime):
model = MagicMock()
model.profile = {
"max_input_tokens": 28_672,
"min_effective_input_tokens": 4_096,
}
middleware = create_message_budget_middleware(
model, MagicMock(), runtime=runtime
with _patched_config({"thread_id": "cron-thread-1"}):
budget = middleware._budget_for_request(_request([HumanMessage("hi")]))
assert budget.input_limit == 1015808
# The lazy snapshot is persisted, bound to the run's own thread.
row = store.find_active_run_snapshot(
deployment_id="local",
thread_id="cron-thread-1",
run_request_id="auto:cron-thread-1",
)
request = _request(
[HumanMessage(content=[{"type": "image", "url": "https://x.test/a"}])]
assert row is not None
def test_lazy_creation_is_shared_across_middleware_calls(self, store, runtime):
"""The same thread converges on one snapshot (idempotent create)."""
middleware = create_message_budget_middleware(
MagicMock(), MagicMock(), runtime=runtime
)
with _patched_config({"thread_id": "cron-thread-2"}):
first = middleware._snapshot()
middleware._budget_for_request(_request([HumanMessage("hi")]))
second = middleware._snapshot()
assert first.snapshot_id == second.snapshot_id
def test_missing_snapshot_and_thread_id_fails_closed(self, runtime):
middleware = create_message_budget_middleware(
MagicMock(), MagicMock(), runtime=runtime
)
with (
_patched_config({}),
pytest.raises(
ContextBudgetUnsatisfiableError, match="CONTEXT_BUDGET_UNSATISFIABLE"
),
pytest.raises(ModelRegistryError) as excinfo,
):
middleware._budget_for_request(request)
middleware._budget_for_request(_request([HumanMessage("hi")]))
assert excinfo.value.code == MODEL_CONFIG_OUTSIDE_SNAPSHOT
def test_summarizer_falls_back_to_compile_time_model(self, runtime):
fallback = MagicMock()
middleware = create_message_budget_middleware(
fallback, MagicMock(), runtime=runtime
def test_bootstrap_registry_fails_with_not_ready(self, tmp_path):
"""No 32K fallback: bootstrap registry → MODEL_REGISTRY_NOT_READY."""
bootstrap_runtime = SnapshotRuntime(
ModelRuntimeStore(config_dir=tmp_path / "model-runtime")
)
middleware = create_message_budget_middleware(
MagicMock(), MagicMock(), runtime=bootstrap_runtime
)
with _patched_config({}):
assert middleware.model is fallback
def test_outside_runnable_context_uses_interim_mode(self, runtime):
fallback = MagicMock()
middleware = create_message_budget_middleware(
fallback, MagicMock(), has_tools=False, runtime=runtime
)
with _patched_config(None):
budget = middleware._budget_for_request(_request([HumanMessage("hi")]))
assert budget.input_limit == 32_768
with (
_patched_config({"thread_id": "cli-thread"}),
pytest.raises(ModelRegistryError) as excinfo,
):
middleware._budget_for_request(_request([HumanMessage("hi")]))
assert excinfo.value.code == MODEL_REGISTRY_NOT_READY
# =============================================================================
@@ -292,12 +388,22 @@ class TestInterimMode:
# =============================================================================
def test_budget_middleware_uses_message_threshold_and_safe_tool_cutoff(runtime):
model = MagicMock()
model.profile = {"max_input_tokens": 32_768}
middleware = create_message_budget_middleware(
model, MagicMock(), has_tools=False, runtime=runtime
def test_budget_middleware_uses_message_threshold_and_safe_tool_cutoff(tmp_path):
# A small context window keeps the trigger thresholds reachable in a test.
store = make_active_store(
tmp_path / "model-runtime-small",
context_window_tokens=32768,
max_output_tokens=8192,
min_effective_input_tokens=1024,
fixed_system_reserve_tokens=1024,
fixed_tools_reserve_tokens=2048,
fixed_attachments_reserve_tokens=1024,
)
runtime = SnapshotRuntime(store)
middleware = create_message_budget_middleware(
MagicMock(), MagicMock(), has_tools=False, runtime=runtime
)
snapshot = make_snapshot(store)
messages = [
HumanMessage(content="a" * 30_000),
AIMessage(
@@ -311,7 +417,7 @@ def test_budget_middleware_uses_message_threshold_and_safe_tool_cutoff(runtime):
from EvoScientist.middleware.message_budget import _ACTIVE_BUDGET
with _patched_config({}):
with _patched_config(_configurable_for(snapshot)):
# Simulate the per-call budget activation that wrap_model_call sets.
token = _ACTIVE_BUDGET.set(middleware._budget_for_request(_request(messages)))
try:
@@ -323,3 +429,11 @@ def test_budget_middleware_uses_message_threshold_and_safe_tool_cutoff(runtime):
# A cutoff never leaves the tool response without its matching AI tool call.
if cutoff == 1:
assert isinstance(messages[cutoff], AIMessage)
def test_active_budget_outside_model_call_fails_loudly(runtime):
middleware = create_message_budget_middleware(
MagicMock(), MagicMock(), runtime=runtime
)
with pytest.raises(RuntimeError, match="outside a model call"):
middleware._active_budget()
-51
View File
@@ -1,51 +0,0 @@
"""Integration tests for MiniMax direct provider.
These tests validate that the MiniMax provider can connect to the real
MiniMax API (api.minimaxi.com/anthropic by default) and produce chat completions.
Requires MINIMAX_API_KEY environment variable to be set.
Optionally set MINIMAX_BASE_URL to https://api.minimax.io/anthropic for Global keys.
"""
import os
import pytest
from EvoScientist.llm import get_chat_model
pytestmark = pytest.mark.skipif(
not os.environ.get("MINIMAX_API_KEY"),
reason="MINIMAX_API_KEY not set",
)
class TestMiniMaxIntegration:
def test_minimax_m25_chat_completion(self):
"""Test that MiniMax M2.5 can produce a chat completion."""
model = get_chat_model("minimax-m2.5", provider="minimax", temperature=0)
response = model.invoke("Reply with exactly: hello")
assert response.content
assert len(response.content) > 0
def test_minimax_m25_highspeed_chat_completion(self):
"""Test that MiniMax M2.5-highspeed can produce a chat completion."""
model = get_chat_model(
"minimax-m2.5-highspeed", provider="minimax", temperature=0
)
response = model.invoke("Reply with exactly: world")
assert response.content
assert len(response.content) > 0
def test_minimax_with_full_model_id(self):
"""Test using the full model ID directly."""
model = get_chat_model("MiniMax-M2.5", provider="minimax", temperature=0)
response = model.invoke("What is 2+2? Answer with just the number.")
assert response.content
assert "4" in response.content
def test_minimax_m27_chat_completion(self):
"""Test that MiniMax M2.7 can produce a chat completion."""
model = get_chat_model("minimax-m2.7", provider="minimax", temperature=0)
response = model.invoke("Reply with exactly: test")
assert response.content
assert len(response.content) > 0
-941
View File
@@ -1,941 +0,0 @@
"""Tests for the /model command and extract_model_and_provider helper."""
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
class TestExtractModelAndProvider:
"""Unit tests for the argument parser helper."""
def test_known_model_no_provider(self):
from EvoScientist.commands.implementation.model import (
extract_model_and_provider,
)
name, prov = extract_model_and_provider(["claude-sonnet-4-6"])
assert name == "claude-sonnet-4-6"
assert prov == "anthropic"
def test_known_model_with_provider_override(self):
from EvoScientist.commands.implementation.model import (
extract_model_and_provider,
)
name, prov = extract_model_and_provider(["claude-sonnet-4-6", "openrouter"])
assert name == "claude-sonnet-4-6"
assert prov == "openrouter"
def test_unknown_model_no_provider_raises(self):
from EvoScientist.commands.implementation.model import (
extract_model_and_provider,
)
with pytest.raises(ValueError, match="Unknown model"):
extract_model_and_provider(["nonexistent-model-xyz"])
def test_unknown_model_with_provider_still_raises(self):
from EvoScientist.commands.implementation.model import (
extract_model_and_provider,
)
# Unknown models are always rejected, even with an explicit provider
with pytest.raises(ValueError, match="Unknown model"):
extract_model_and_provider(["my-custom-model", "custom-openai"])
def test_provider_override_on_known_model(self):
from EvoScientist.commands.implementation.model import (
extract_model_and_provider,
)
# Known model with explicit provider override uses the override
name, prov = extract_model_and_provider(["claude-sonnet-4-6", "openrouter"])
assert name == "claude-sonnet-4-6"
assert prov == "openrouter"
def test_ollama_provider_accepts_arbitrary_name(self):
"""Ollama models are locally-installed — the registry doesn't know
them. The ``ollama`` provider must pass any name through verbatim."""
from EvoScientist.commands.implementation.model import (
extract_model_and_provider,
)
name, prov = extract_model_and_provider(["llama3.3:8b", "ollama"])
assert name == "llama3.3:8b"
assert prov == "ollama"
def test_ollama_provider_accepts_dotted_tag(self):
from EvoScientist.commands.implementation.model import (
extract_model_and_provider,
)
name, prov = extract_model_and_provider(["qwen3-coder-next:latest", "ollama"])
assert name == "qwen3-coder-next:latest"
assert prov == "ollama"
class TestModelCommandUnknownModel:
"""Verify error message for unknown models."""
async def test_unknown_model_shows_error(self):
from EvoScientist.commands.implementation.model import ModelCommand
cmd = ModelCommand()
ui = MagicMock()
ui.supports_interactive = True
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
ctx = MagicMock()
ctx.ui = ui
with patch(
"EvoScientist.EvoScientist._ensure_config",
return_value=cfg,
):
await cmd.execute(ctx, ["nonexistent-model-xyz"])
ui.append_system.assert_called_once()
call_args = ui.append_system.call_args
assert "Unknown model" in call_args[0][0]
assert call_args[1]["style"] == "red"
class TestModelCommandPickerCancelled:
"""Verify no-op when the interactive picker is cancelled."""
async def test_picker_returns_none(self):
from EvoScientist.commands.implementation.model import ModelCommand
cmd = ModelCommand()
ui = MagicMock()
ui.supports_interactive = True
ui.wait_for_model_pick = AsyncMock(return_value=None)
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
ctx = MagicMock()
ctx.ui = ui
with patch(
"EvoScientist.EvoScientist._ensure_config",
return_value=cfg,
):
await cmd.execute(ctx, [])
# No model switch should have happened
ui.append_system.assert_not_called()
class TestModelCommandSwitch:
"""Verify a successful model switch updates config and rebuilds agent."""
async def test_switch_known_model(self):
from EvoScientist.commands.implementation.model import ModelCommand
cmd = ModelCommand()
ui = MagicMock()
ui.supports_interactive = True
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
new_agent = MagicMock()
ctx = MagicMock()
ctx.ui = ui
ctx.workspace_dir = "/tmp/test"
ctx.checkpointer = MagicMock()
with (
patch(
"EvoScientist.EvoScientist._ensure_config",
return_value=cfg,
),
patch("EvoScientist.EvoScientist._build_chat_model"),
patch("EvoScientist.EvoScientist.set_active_config") as set_cfg,
patch("EvoScientist.EvoScientist.set_chat_model_instance"),
patch(
"EvoScientist.cli.agent._load_agent",
return_value=new_agent,
),
):
await cmd.execute(ctx, ["claude-opus-4-8"])
# The switch is committed via set_active_config(temp_cfg), not by
# mutating the original cfg object in place.
set_cfg.assert_called_once()
committed = set_cfg.call_args[0][0]
assert committed.model == "claude-opus-4-8"
assert committed.provider == "anthropic"
# Agent should be replaced on context
assert ctx.agent == new_agent
# Success message shown
ui.append_system.assert_called_once()
msg = ui.append_system.call_args[0][0]
assert "claude-opus-4-8" in msg
assert "anthropic" in msg
async def test_switch_with_save_flag(self):
from EvoScientist.commands.implementation.model import ModelCommand
cmd = ModelCommand()
ui = MagicMock()
ui.supports_interactive = True
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
ctx = MagicMock()
ctx.ui = ui
ctx.workspace_dir = "/tmp/test"
ctx.checkpointer = MagicMock()
with (
patch(
"EvoScientist.EvoScientist._ensure_config",
return_value=cfg,
),
patch("EvoScientist.EvoScientist._build_chat_model"),
patch("EvoScientist.EvoScientist.set_active_config"),
patch("EvoScientist.EvoScientist.set_chat_model_instance"),
patch(
"EvoScientist.cli.agent._load_agent",
return_value=MagicMock(),
),
patch("EvoScientist.config.settings.set_config_value") as mock_save,
):
await cmd.execute(ctx, ["claude-opus-4-8", "--save"])
# Config file should be updated
mock_save.assert_any_call("model", "claude-opus-4-8")
mock_save.assert_any_call("provider", "anthropic")
# Success message should mention save
msg = ui.append_system.call_args[0][0]
assert "saved to config" in msg
async def test_switch_without_save_flag_does_not_persist(self):
from EvoScientist.commands.implementation.model import ModelCommand
cmd = ModelCommand()
ui = MagicMock()
ui.supports_interactive = True
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
ctx = MagicMock()
ctx.ui = ui
ctx.workspace_dir = "/tmp/test"
ctx.checkpointer = MagicMock()
with (
patch(
"EvoScientist.EvoScientist._ensure_config",
return_value=cfg,
),
patch("EvoScientist.EvoScientist._build_chat_model"),
patch("EvoScientist.EvoScientist.set_active_config"),
patch("EvoScientist.EvoScientist.set_chat_model_instance"),
patch(
"EvoScientist.cli.agent._load_agent",
return_value=MagicMock(),
),
patch("EvoScientist.config.settings.set_config_value") as mock_save,
):
await cmd.execute(ctx, ["claude-opus-4-8"])
# Config file should NOT be updated
mock_save.assert_not_called()
# Message should not mention save
msg = ui.append_system.call_args[0][0]
assert "saved to config" not in msg
class TestModelCommandFailure:
"""Verify error handling when chat-model construction raises."""
async def test_build_chat_model_error(self):
from EvoScientist.commands.implementation.model import ModelCommand
cmd = ModelCommand()
ui = MagicMock()
ui.supports_interactive = True
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
ctx = MagicMock()
ctx.ui = ui
ctx.workspace_dir = "/tmp/test"
ctx.checkpointer = MagicMock()
with (
patch(
"EvoScientist.EvoScientist._ensure_config",
return_value=cfg,
),
patch(
"EvoScientist.EvoScientist._build_chat_model",
side_effect=RuntimeError("API key missing"),
) as mock_build,
):
await cmd.execute(ctx, ["claude-opus-4-8"])
mock_build.assert_called_once()
ui.append_system.assert_called_once()
call_args = ui.append_system.call_args
assert "Failed to switch model" in call_args[0][0]
assert call_args[1]["style"] == "red"
@pytest.fixture
def evo_module_state():
"""Snapshot and restore ``EvoScientist.EvoScientist`` module globals.
The chat-model cache tests mutate ``_chat_model`` / ``_chat_model_key``
/ ``_config`` / ``_EvoScientist_agent`` directly. This fixture
guarantees all four are restored — even if a test body grows an early
return — so later tests in the suite see a clean module state.
"""
import EvoScientist.EvoScientist as mod
snapshot = (
mod._chat_model,
mod._chat_model_key,
mod._config,
mod._EvoScientist_agent,
)
try:
yield mod
finally:
(
mod._chat_model,
mod._chat_model_key,
mod._config,
mod._EvoScientist_agent,
) = snapshot
class TestEnsureChatModelCacheInvalidation:
"""Regression tests for issue #179: /model switch lagged by one step.
Root cause: ``_ensure_chat_model()`` returned a cached ``_chat_model``
without checking whether ``cfg.model`` / ``cfg.provider`` had changed
since the cache was populated. ``ModelCommand._apply_model`` builds
a new agent *before* ``set_chat_model`` runs, so the new agent was
bound to the *previous* cached model.
Fix: track a ``(model, provider)`` key alongside ``_chat_model`` and
rebuild on mismatch.
"""
def test_cache_rebuilds_when_config_model_changes(self, evo_module_state):
"""After cfg.model changes, _ensure_chat_model must return a new instance."""
mod = evo_module_state
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
m1 = MagicMock(name="model-1")
m2 = MagicMock(name="model-2")
mod._chat_model = None
mod._chat_model_key = None
mod._config = cfg
mod._EvoScientist_agent = None
with patch("EvoScientist.llm.get_chat_model", side_effect=[m1, m2]) as gm:
first = mod._ensure_chat_model()
assert first is m1
# Same config → cache hit, no rebuild.
again = mod._ensure_chat_model()
assert again is m1
assert gm.call_count == 1
# Simulate /model switch writing the new choice into cfg.
cfg.model = "minimax-m2.7"
cfg.provider = "openrouter"
second = mod._ensure_chat_model()
# Must be the NEW model instance, not the cached one.
assert second is m2
assert second is not first
assert gm.call_count == 2
# Second call used the new model name + provider.
_, kwargs = gm.call_args
assert kwargs == {"model": "minimax-m2.7", "provider": "openrouter"}
def test_cache_rebuilds_when_only_provider_changes(self, evo_module_state):
"""Same model name, different provider (openrouter vs anthropic) must rebuild."""
mod = evo_module_state
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
m1 = MagicMock(name="anthropic-model")
m2 = MagicMock(name="openrouter-model")
mod._chat_model = None
mod._chat_model_key = None
mod._config = cfg
mod._EvoScientist_agent = None
with patch("EvoScientist.llm.get_chat_model", side_effect=[m1, m2]) as gm:
assert mod._ensure_chat_model() is m1
cfg.provider = "openrouter"
assert mod._ensure_chat_model() is m2
assert gm.call_count == 2
def test_set_chat_model_updates_key(self, evo_module_state):
"""set_chat_model must keep _chat_model_key in sync to avoid
an extra rebuild on the very next _ensure_chat_model() call."""
mod = evo_module_state
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
m_set = MagicMock(name="explicit-set-model")
mod._chat_model = None
mod._chat_model_key = None
mod._config = cfg
mod._EvoScientist_agent = None
with patch("EvoScientist.llm.get_chat_model", return_value=m_set) as gm:
mod.set_chat_model("minimax-m2.7", provider="openrouter")
assert mod._chat_model is m_set
assert mod._chat_model_key == ("minimax-m2.7", "openrouter")
# Align cfg to what set_chat_model was called with.
cfg.model = "minimax-m2.7"
cfg.provider = "openrouter"
# Now _ensure_chat_model should NOT rebuild (key matches cfg).
assert mod._ensure_chat_model() is m_set
assert gm.call_count == 1
def test_set_chat_model_is_no_op_when_key_already_matches(self, evo_module_state):
"""set_chat_model must NOT rebuild when the cache already holds the
requested (model, provider).
Under the /model flow, ``_load_agent`` already rebuilt ``_chat_model``
via ``_ensure_chat_model`` before ``set_chat_model`` is reached, so
the subsequent set should be idempotent — reusing the same Python
instance (and thus the same underlying HTTP client) that ``ctx.agent``
is already bound to.
"""
mod = evo_module_state
existing = MagicMock(name="existing-model")
mod._chat_model = existing
mod._chat_model_key = ("minimax-m2.7", "openrouter")
mod._config = SimpleNamespace(model="minimax-m2.7", provider="openrouter")
mod._EvoScientist_agent = None
with patch("EvoScientist.llm.get_chat_model") as gm:
returned = mod.set_chat_model("minimax-m2.7", provider="openrouter")
# Fast path: returned the EXISTING instance, no rebuild.
assert returned is existing
assert mod._chat_model is existing
gm.assert_not_called()
class TestApplyModelIntegration:
"""End-to-end regression for #179 + #183: `_apply_model` must produce an
agent bound to the NEW model, threaded in via the pure ``create_cli_agent``
path.
Exercises the real chain:
``_apply_model → _build_chat_model → _load_agent(chat_model=...) →
set_active_config / set_chat_model_instance``
Only ``_load_agent`` is faked (to avoid spinning up deepagents, MCP tools,
middleware, and subagent YAML); it binds the ``chat_model`` it receives.
``get_chat_model`` returns a distinct sentinel per ``(model, provider)``
pair so we can assert on identity.
"""
async def test_new_agent_is_bound_to_newly_selected_model(self, evo_module_state):
from EvoScientist.commands.implementation.model import ModelCommand
from EvoScientist.config.settings import EvoScientistConfig
mod = evo_module_state
sentinels: dict[tuple[str, str | None], MagicMock] = {}
def _fake_get_chat_model(model, provider=None):
key = (model, provider)
if key not in sentinels:
sentinels[key] = MagicMock(name=f"chat_model[{model}|{provider}]")
sentinels[key]._bound_model = model
sentinels[key]._bound_provider = provider
return sentinels[key]
def _fake_load_agent(
workspace_dir=None,
checkpointer=None,
config=None,
chat_model=None,
*,
on_mcp_progress=None,
):
# The pure path threads the freshly built chat model in; bind it
# directly instead of re-deriving via _ensure_chat_model.
agent = MagicMock(name="fake-agent")
agent._bound_model = chat_model
return agent
cfg = EvoScientistConfig(model="claude-sonnet-4-6", provider="anthropic")
ctx = MagicMock()
ctx.ui = MagicMock()
ctx.ui.supports_interactive = True
ctx.workspace_dir = "/tmp/test_integration"
ctx.checkpointer = None
# Prime: _chat_model already holds the OLD (default) model —
# this is the state that caused the off-by-one in production.
mod._config = cfg
mod._chat_model = _fake_get_chat_model("claude-sonnet-4-6", "anthropic")
mod._chat_model_key = ("claude-sonnet-4-6", "anthropic")
mod._EvoScientist_agent = None
old_model = mod._chat_model
with (
patch(
"EvoScientist.llm.get_chat_model",
side_effect=_fake_get_chat_model,
),
patch(
"EvoScientist.cli.agent._load_agent",
side_effect=_fake_load_agent,
),
):
cmd = ModelCommand()
await cmd._apply_model(ctx, "minimax-m2.7", "openrouter")
# The agent produced by _apply_model must be bound to the
# NEWLY requested model, threaded in via chat_model=.
assert ctx.agent._bound_model is not old_model
assert ctx.agent._bound_model._bound_model == "minimax-m2.7"
assert ctx.agent._bound_model._bound_provider == "openrouter"
# Global state reflects the switch end-to-end, committed via the setters.
assert mod._chat_model_key == ("minimax-m2.7", "openrouter")
assert mod._chat_model is sentinels[("minimax-m2.7", "openrouter")]
# The switch is applied to the LIVE cfg in place and ``_config`` stays
# bound to that same object, so callers holding the active config by
# reference (e.g. serve's ``agent_holder["config"]``) observe the swap.
assert mod._config is cfg
assert cfg.model == "minimax-m2.7"
assert cfg.provider == "openrouter"
# User-visible success message.
msg = ctx.ui.append_system.call_args[0][0]
assert "minimax-m2.7" in msg
assert "openrouter" in msg
class TestApplyModelPreservesConfigByReference:
"""Regression for the din0s review on #267: serve mode (and any long-lived
caller) holds the active config object by reference via
``agent_holder["config"]``. The pure-path commit must apply the switch to
that LIVE object in place — not rebind ``_config`` to a fresh ``temp_cfg``
copy — otherwise a later workspace-changing ``/resume`` reloads the agent
from the stale startup config and silently reverts the ``/model`` switch.
Critically this must hold across *repeated* switches: the earlier
"also mutate cfg but still rebind to temp_cfg" remedy only survives one
switch (the held object stops being the active ``_config`` after the first).
"""
async def test_held_config_reference_tracks_repeated_switches(
self, evo_module_state
):
from EvoScientist.commands.implementation.model import ModelCommand
from EvoScientist.config.settings import EvoScientistConfig
mod = evo_module_state
def _fake_get_chat_model(model, provider=None):
m = MagicMock(name=f"chat_model[{model}|{provider}]")
m._bound_model = model
return m
def _fake_load_agent(
workspace_dir=None,
checkpointer=None,
config=None,
chat_model=None,
*,
on_mcp_progress=None,
):
return MagicMock(name="fake-agent")
cfg = EvoScientistConfig(model="claude-sonnet-4-6", provider="anthropic")
mod._config = cfg
mod._chat_model = None
mod._chat_model_key = None
mod._EvoScientist_agent = None
# Simulate serve capturing the startup config object once (commands.py:
# ``agent_holder = {... "config": config}``) and never re-reading it.
agent_holder = {"config": cfg}
ctx = MagicMock()
ctx.ui = MagicMock()
ctx.ui.supports_interactive = True
ctx.workspace_dir = "/tmp/test_byref"
ctx.checkpointer = None
cmd = ModelCommand()
with (
patch(
"EvoScientist.llm.get_chat_model",
side_effect=_fake_get_chat_model,
),
patch(
"EvoScientist.cli.agent._load_agent",
side_effect=_fake_load_agent,
),
):
for model, provider in [
("claude-opus-4-8", "anthropic"),
("minimax-m2.7", "openrouter"),
("claude-sonnet-4-6", "anthropic"),
]:
await cmd._apply_model(ctx, model, provider)
# The held reference must reflect the LATEST switch on every
# iteration — not just the first — and stay the active config.
assert agent_holder["config"].model == model
assert agent_holder["config"].provider == provider
assert mod._config is agent_holder["config"]
class TestModelCommandLoadAgentFailure:
"""Verify the transactional ordering: when ``_load_agent`` raises,
nothing downstream (``set_chat_model``, ``cfg`` mutation,
``set_config_value``) should happen.
This is the core guarantee of the refactor that established
"build agent first, commit state only on success". Without this test
the ordering could silently regress (e.g. if ``_apply_model`` were
reordered to call ``set_chat_model`` first)."""
async def test_load_agent_error_is_transactional(self):
from EvoScientist.commands.implementation.model import ModelCommand
cmd = ModelCommand()
ui = MagicMock()
ui.supports_interactive = True
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
ctx = MagicMock()
ctx.ui = ui
ctx.workspace_dir = "/tmp/test"
ctx.checkpointer = MagicMock()
with (
patch(
"EvoScientist.EvoScientist._ensure_config",
return_value=cfg,
),
patch("EvoScientist.EvoScientist._build_chat_model"),
patch(
"EvoScientist.cli.agent._load_agent",
side_effect=RuntimeError("agent build failed"),
) as mock_load,
patch(
"EvoScientist.EvoScientist.set_active_config",
) as mock_set_cfg,
patch(
"EvoScientist.EvoScientist.set_chat_model_instance",
) as mock_set_model,
patch(
"EvoScientist.config.settings.set_config_value",
) as mock_save,
):
# Pass ``--save`` to strengthen the assertion: if the ordering
# ever regresses, ``set_config_value`` would be called with
# stale data.
await cmd.execute(ctx, ["claude-opus-4-8", "--save"])
# _load_agent was attempted (transactional first step).
mock_load.assert_called_once()
# Commit setters and downstream side-effects must NOT have happened.
mock_set_cfg.assert_not_called()
mock_set_model.assert_not_called()
mock_save.assert_not_called()
# Config must be untouched.
assert cfg.model == "claude-sonnet-4-6"
assert cfg.provider == "anthropic"
# User sees a red error message.
ui.append_system.assert_called_once()
call_args = ui.append_system.call_args
assert "Failed to switch model" in call_args[0][0]
assert call_args[1]["style"] == "red"
class TestApplyModelLoadAgentFailureTransactional:
"""Regression for #183: when agent construction fails, the session stays on
the original model with no snapshot/restore.
Because ``create_cli_agent(config=..., chat_model=...)`` is now pure — it
writes none of the four config/model globals — and ``_apply_model`` commits
only after a successful build, a failing ``_load_agent`` leaves all four
globals untouched. This replaces the old snapshot/restore rollback test.
Complements :class:`TestModelCommandLoadAgentFailure`, which asserts the
downstream setters never run on failure.
"""
async def test_globals_unchanged_when_load_agent_raises(self, evo_module_state):
from EvoScientist.commands.implementation.model import ModelCommand
from EvoScientist.config.settings import EvoScientistConfig
mod = evo_module_state
def _fake_load_agent(
workspace_dir=None,
checkpointer=None,
config=None,
chat_model=None,
*,
on_mcp_progress=None,
):
# The pure path writes no globals; mimic a failure partway through
# agent wiring (middleware build, deepagents, MCP reconnect, ...).
raise RuntimeError("middleware build failed")
cfg = EvoScientistConfig(model="claude-sonnet-4-6", provider="anthropic")
old_model = MagicMock(name="old-model")
old_agent = MagicMock(name="old-default-agent")
mod._config = cfg
mod._chat_model = old_model
mod._chat_model_key = ("claude-sonnet-4-6", "anthropic")
mod._EvoScientist_agent = old_agent
ctx = MagicMock()
ctx.ui = MagicMock()
ctx.ui.supports_interactive = True
ctx.workspace_dir = "/tmp/test_rollback"
ctx.checkpointer = None
with (
patch(
"EvoScientist.EvoScientist._build_chat_model",
return_value=MagicMock(name="new-model"),
),
patch(
"EvoScientist.cli.agent._load_agent",
side_effect=_fake_load_agent,
),
):
cmd = ModelCommand()
await cmd._apply_model(ctx, "minimax-m2.7", "openrouter")
# All four globals are unchanged — nothing was committed.
assert mod._config is cfg
assert mod._chat_model is old_model
assert mod._chat_model_key == ("claude-sonnet-4-6", "anthropic")
assert mod._EvoScientist_agent is old_agent
# The ``cfg`` object itself was not mutated.
assert cfg.model == "claude-sonnet-4-6"
assert cfg.provider == "anthropic"
# User sees an error message.
msg = ctx.ui.append_system.call_args[0][0]
assert "Failed to switch model" in msg
class TestModelCommandOllamaPicker:
"""Verify Ollama discovery augments the picker entries and the sentinel
is always present when Ollama is configured."""
@pytest.fixture(autouse=True)
def _isolate_provider_registry(self, tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
def _make_ctx_and_cfg(self, *, ollama_base_url: str | None):
cfg = SimpleNamespace(
model="claude-sonnet-4-6",
provider="anthropic",
ollama_base_url=ollama_base_url,
)
ui = MagicMock()
ui.supports_interactive = True
ui.wait_for_model_pick = AsyncMock(return_value=None)
ctx = MagicMock()
ctx.ui = ui
return ctx, cfg, ui
async def test_picker_entries_include_detected_ollama_models(self):
"""When Ollama is reachable, detected models appear in entries with
provider='ollama' and the Custom sentinel is appended."""
from EvoScientist.commands.implementation.model import ModelCommand
ctx, cfg, ui = self._make_ctx_and_cfg(ollama_base_url="http://localhost:11434")
async def fake_discover(base_url, *, timeout):
return ["llama3.3:latest", "qwen3:8b"]
with (
patch(
"EvoScientist.EvoScientist._ensure_config",
return_value=cfg,
),
patch(
"EvoScientist.llm.ollama_discovery.discover_ollama_models",
side_effect=fake_discover,
),
):
await ModelCommand().execute(ctx, [])
entries = ui.wait_for_model_pick.call_args[0][0]
ollama_rows = [(n, mid, p) for (n, mid, p) in entries if p == "ollama"]
assert ("llama3.3:latest", "llama3.3:latest", "ollama") in ollama_rows
assert ("qwen3:8b", "qwen3:8b", "ollama") in ollama_rows
assert (
"Custom Ollama model...",
"__custom_ollama__",
"ollama",
) in ollama_rows
async def test_picker_entries_include_sentinel_when_discovery_empty(self):
"""Daemon unreachable / no models pulled — sentinel is the user's
escape hatch and must always be present."""
from EvoScientist.commands.implementation.model import ModelCommand
ctx, cfg, ui = self._make_ctx_and_cfg(ollama_base_url="http://localhost:11434")
async def fake_discover(base_url, *, timeout):
return []
with (
patch(
"EvoScientist.EvoScientist._ensure_config",
return_value=cfg,
),
patch(
"EvoScientist.llm.ollama_discovery.discover_ollama_models",
side_effect=fake_discover,
),
):
await ModelCommand().execute(ctx, [])
entries = ui.wait_for_model_pick.call_args[0][0]
ollama_rows = [(n, mid, p) for (n, mid, p) in entries if p == "ollama"]
assert ollama_rows == [
("Custom Ollama model...", "__custom_ollama__", "ollama")
]
async def test_picker_skips_ollama_section_when_not_configured(self):
"""ollama_base_url unset → no discovery call, no ollama entries,
no sentinel (issue non-goal: no implicit localhost detection)."""
from EvoScientist.commands.implementation.model import ModelCommand
ctx, cfg, ui = self._make_ctx_and_cfg(ollama_base_url="")
discovery = AsyncMock(return_value=["should-never-appear"])
with (
patch(
"EvoScientist.EvoScientist._ensure_config",
return_value=cfg,
),
patch(
"EvoScientist.llm.ollama_discovery.discover_ollama_models",
discovery,
),
):
await ModelCommand().execute(ctx, [])
discovery.assert_not_called()
entries = ui.wait_for_model_pick.call_args[0][0]
assert not any(p == "ollama" for (_, _, p) in entries)
async def test_picker_handles_cfg_without_ollama_base_url_attr(self):
"""getattr(cfg, 'ollama_base_url', None) fallback: old configs
(or SimpleNamespace test fixtures) may not carry the attribute
at all. Must not raise AttributeError, must not probe."""
from EvoScientist.commands.implementation.model import ModelCommand
# Deliberately omit ollama_base_url from the namespace.
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
ui = MagicMock()
ui.supports_interactive = True
ui.wait_for_model_pick = AsyncMock(return_value=None)
ctx = MagicMock()
ctx.ui = ui
discovery = AsyncMock(return_value=["should-never-appear"])
with (
patch(
"EvoScientist.EvoScientist._ensure_config",
return_value=cfg,
),
patch(
"EvoScientist.llm.ollama_discovery.discover_ollama_models",
discovery,
),
):
await ModelCommand().execute(ctx, [])
discovery.assert_not_called()
entries = ui.wait_for_model_pick.call_args[0][0]
assert not any(p == "ollama" for (_, _, p) in entries)
async def test_picker_sentinel_result_is_treated_as_cancel(self):
"""Defense-in-depth: if the widget ever returns the sentinel name
itself (shouldn't happen — it should substitute the typed name),
dispatch treats it as a cancel and does NOT call _apply_model."""
from EvoScientist.commands.implementation.model import ModelCommand
ctx, cfg, ui = self._make_ctx_and_cfg(ollama_base_url="http://localhost:11434")
ui.wait_for_model_pick = AsyncMock(return_value=("__custom_ollama__", "ollama"))
async def fake_discover(base_url, *, timeout):
return []
with (
patch(
"EvoScientist.EvoScientist._ensure_config",
return_value=cfg,
),
patch(
"EvoScientist.llm.ollama_discovery.discover_ollama_models",
side_effect=fake_discover,
),
patch("EvoScientist.cli.agent._load_agent") as load_agent,
):
await ModelCommand().execute(ctx, [])
load_agent.assert_not_called()
assert cfg.model == "claude-sonnet-4-6" # unchanged
async def test_picker_applies_detected_ollama_model(self):
"""User picks a live-detected Ollama model → _apply_model is invoked
with (name, "ollama") and the agent is rebuilt."""
from EvoScientist.commands.implementation.model import ModelCommand
ctx, cfg, ui = self._make_ctx_and_cfg(ollama_base_url="http://localhost:11434")
ctx.workspace_dir = "/tmp/test"
ctx.checkpointer = MagicMock()
ui.wait_for_model_pick = AsyncMock(return_value=("llama3.3", "ollama"))
async def fake_discover(base_url, *, timeout):
return ["llama3.3"]
with (
patch(
"EvoScientist.EvoScientist._ensure_config",
return_value=cfg,
),
patch(
"EvoScientist.llm.ollama_discovery.discover_ollama_models",
side_effect=fake_discover,
),
patch("EvoScientist.EvoScientist._build_chat_model"),
patch("EvoScientist.EvoScientist.set_active_config") as set_cfg,
patch("EvoScientist.EvoScientist.set_chat_model_instance"),
patch(
"EvoScientist.cli.agent._load_agent",
return_value=MagicMock(),
),
):
await ModelCommand().execute(ctx, [])
# Committed via set_active_config(temp_cfg); original cfg untouched.
set_cfg.assert_called_once()
committed = set_cfg.call_args[0][0]
assert committed.model == "llama3.3"
assert committed.provider == "ollama"
+29 -6
View File
@@ -21,6 +21,13 @@ import pytest
from EvoScientist.llm import patches as patches_mod
@pytest.fixture(autouse=True)
def _active_registry(active_snapshot_runtime):
"""The patch freezes a local run snapshot per child thread; that needs an
active registry (bootstrap fails closed with MODEL_REGISTRY_NOT_READY)."""
return active_snapshot_runtime
# =============================================================================
# Helpers
# =============================================================================
@@ -348,7 +355,13 @@ class TestNonInterceptedMethods:
class TestNoContextNoConfig:
def test_no_scope_no_config_kwarg(self, restore_model_passthrough_patch):
def test_no_scope_only_snapshot_config(self, restore_model_passthrough_patch):
"""No inherited scope → only the frozen child snapshot ID is added.
Since the snapshot-driven migration (design doc 8.1/8.2), every
child-thread ``runs.create`` carries ``runtime_snapshot_id``; scope
keys are still only forwarded when present in the parent context.
"""
try:
from deepagents.middleware import async_subagents as ds_mod
except ImportError:
@@ -366,8 +379,17 @@ class TestNoContextNoConfig:
runs_sync.create.assert_called_once()
kwargs = runs_sync.create.call_args.kwargs
# No config kwarg should be added when there's nothing to inherit.
assert "config" not in kwargs
configurable = kwargs["config"]["configurable"]
assert "runtime_snapshot_id" in configurable
for key in (
"workspace_scope_id",
"workspace_scope_owner_id",
"workspace_scope_revision",
"workspace_deployment_id",
):
assert key not in configurable
assert "model" not in configurable
assert "model_provider" not in configurable
# =============================================================================
@@ -412,6 +434,7 @@ class TestPreserveExistingConfig:
{"thread_id": "t1", "config": _Sentinel()}
)
assert merged["thread_id"] == "t1"
# Non-dict input was replaced with a fresh dict; nothing is injected
# when there is no runnable context to inherit from.
assert merged["config"] == {"configurable": {}}
# Non-dict input was replaced with a fresh dict; only the frozen
# child snapshot ID is injected when there is no context to inherit.
configurable = merged["config"]["configurable"]
assert set(configurable) == {"runtime_snapshot_id"}
+15 -614
View File
@@ -52,16 +52,12 @@ def _patch_all_questionary(mock_q):
class TestConstants:
def test_steps_has_thirteen_items(self):
"""Test that STEPS contains exactly 13 steps."""
assert len(STEPS) == 13
def test_steps_has_nine_items(self):
"""Test that STEPS contains exactly 9 steps."""
assert len(STEPS) == 9
assert STEPS == [
"UI",
"LangGraph Port",
"Provider",
"API Key",
"Model",
"Auxiliary Model",
"Tavily Key",
"Workspace",
"Thinking",
@@ -92,44 +88,11 @@ class TestSharedConstantsAlignment:
"""Drift guard: the canonical valid-value sets in
``EvoScientist.config.onboard.constants`` must match the actual
``Choice(value=...)`` ids built by the interactive step functions in
``steps.py``. Without this, adding a new provider to one file but not
``steps.py``. Without this, adding a new value to one file but not
the other would silently break either CLI flag validation or the
interactive picker.
"""
def test_provider_constants_match_step_choices(self):
"""Every value in `_step_provider`'s Choice list must be in
``VALID_PROVIDERS`` — and vice versa."""
from unittest.mock import MagicMock, patch
from EvoScientist.config.onboard.constants import VALID_PROVIDERS
from EvoScientist.config.onboard.steps import _step_provider
# Intercept questionary.select to capture the choices list before
# any user prompt happens.
captured = {}
def _capture(*args, **kwargs):
captured["choices"] = kwargs.get("choices") or (
args[1] if len(args) > 1 else []
)
fake = MagicMock()
fake.ask.return_value = "anthropic"
return fake
with patch(
"EvoScientist.config.onboard.steps.questionary.select",
side_effect=_capture,
):
_step_provider(EvoScientistConfig())
actual_provider_ids = {c.value for c in captured["choices"]}
assert actual_provider_ids == set(VALID_PROVIDERS), (
"VALID_PROVIDERS in constants.py drifted from _step_provider's "
f"choices. Only-in-constants: {set(VALID_PROVIDERS) - actual_provider_ids}; "
f"only-in-choices: {actual_provider_ids - set(VALID_PROVIDERS)}"
)
def test_ui_constants_match_step_choices(self):
from EvoScientist.config.onboard.constants import VALID_UI_BACKENDS
@@ -142,79 +105,6 @@ class TestSharedConstantsAlignment:
assert VALID_WORKSPACE_MODES == frozenset({"daemon", "run"})
def test_valid_providers_aligns_with_provider_key_attr(self):
"""Every provider in ``VALID_PROVIDERS`` (except ``openai`` and
``ollama``, which the wizard handles via fallbacks) must appear in
``_PROVIDER_KEY_ATTR`` — otherwise the auth-mode flow won't know
which config attribute to write the validated key to."""
from EvoScientist.config.onboard.constants import VALID_PROVIDERS
from EvoScientist.config.onboard.wizard import _PROVIDER_KEY_ATTR
expected = set(VALID_PROVIDERS) - {"openai", "ollama"}
missing = expected - set(_PROVIDER_KEY_ATTR.keys())
assert not missing, (
"VALID_PROVIDERS has providers missing from _PROVIDER_KEY_ATTR "
f"in wizard.py: {missing}"
)
extra = set(_PROVIDER_KEY_ATTR.keys()) - expected
assert not extra, (
f"_PROVIDER_KEY_ATTR in wizard.py has keys not in VALID_PROVIDERS: {extra}"
)
def test_valid_providers_aligns_with_provider_key_info(self):
"""``_provider_key_info`` uses ``mapping.get(provider, <openai
fallback>)``, so an unknown provider silently behaves like OpenAI.
Verify every non-``openai`` provider in ``VALID_PROVIDERS`` has an
explicit entry — detected by checking the returned display name is
not the OpenAI fallback string."""
from EvoScientist.config.onboard.constants import VALID_PROVIDERS
from EvoScientist.config.onboard.helpers import _provider_key_info
cfg = EvoScientistConfig()
for provider in VALID_PROVIDERS:
display_name, _, _ = _provider_key_info(cfg, provider)
if provider == "openai":
assert display_name == "OpenAI"
else:
assert display_name != "OpenAI", (
f"Provider {provider!r} is missing from _provider_key_info "
"in helpers.py — it falls through to the OpenAI default, "
"which would silently send the wrong validator and "
"current-key lookup."
)
class TestOAuthModeReconcile:
def test_reconcile_preserves_auxiliary_openai_oauth(self):
from EvoScientist.config.onboard.wizard import _reconcile_oauth_modes
config = EvoScientistConfig(
provider="minimax",
auxiliary_provider="openai",
auxiliary_model="gpt-5.5",
openai_auth_mode="oauth",
anthropic_auth_mode="oauth",
)
_reconcile_oauth_modes(config)
assert config.openai_auth_mode == "oauth"
assert config.anthropic_auth_mode == "api_key"
def test_reconcile_preserves_auxiliary_provider_without_model(self):
from EvoScientist.config.onboard.wizard import _reconcile_oauth_modes
config = EvoScientistConfig(
provider="minimax",
auxiliary_provider="openai",
auxiliary_model="",
openai_auth_mode="oauth",
)
_reconcile_oauth_modes(config)
assert config.openai_auth_mode == "oauth"
# =============================================================================
# Test render_progress
@@ -229,7 +119,7 @@ class TestRenderProgress:
assert panel.title is not None
# The renderable content should contain step names
content_str = str(panel.renderable)
assert "Provider" in content_str
assert "Tavily Key" in content_str
def test_renders_completed_steps(self):
"""Test that completed steps are marked differently."""
@@ -378,206 +268,6 @@ class TestChoiceValidator:
# =============================================================================
class TestStepProvider:
def test_returns_selected_provider(self):
"""Test that _step_provider returns selected provider."""
from EvoScientist.config.onboard.steps import _step_provider
config = EvoScientistConfig()
with patch("EvoScientist.config.onboard.steps.questionary") as mock_q:
mock_q.select.return_value.ask.return_value = "anthropic"
result = _step_provider(config)
assert result == "anthropic"
mock_q.select.assert_called_once()
def test_default_value_and_label_override(self):
"""default_value preselects a provider (re-run co-pilot default) and
label customizes the prompt text."""
from EvoScientist.config.onboard.steps import _step_provider
config = EvoScientistConfig(provider="anthropic")
with patch("EvoScientist.config.onboard.steps.questionary") as mock_q:
mock_q.select.return_value.ask.return_value = "openai"
_step_provider(config, label="co-pilot", default_value="openrouter")
call = mock_q.select.call_args
assert call.kwargs["default"] == "openrouter" # override, not config.provider
assert "co-pilot" in call.args[0]
def test_raises_keyboard_interrupt_on_cancel(self):
"""Test that _step_provider raises KeyboardInterrupt on cancel."""
from EvoScientist.config.onboard.steps import _step_provider
config = EvoScientistConfig()
with patch("EvoScientist.config.onboard.steps.questionary") as mock_q:
mock_q.select.return_value.ask.return_value = None
with pytest.raises(KeyboardInterrupt):
_step_provider(config)
class TestStepOAuthAuthMode:
@pytest.mark.parametrize(
(
"step_name",
"config_attr",
"provider_label",
"oauth_choice_label",
"ccproxy_provider",
"status_label",
"question_label",
"login_prompt",
),
[
(
"_step_anthropic_auth_mode",
"anthropic_auth_mode",
"Anthropic",
"Claude Code OAuth",
"claude_api",
"OAuth",
"Authentication mode",
"Log in to Claude now?",
),
(
"_step_openai_auth_mode",
"openai_auth_mode",
"OpenAI",
"Codex OAuth",
"codex",
"Codex OAuth",
"OpenAI authentication mode",
"Log in to Codex now?",
),
],
)
def test_oauth_wrappers_use_provider_specific_ccproxy_flow(
self,
step_name,
config_attr,
provider_label,
oauth_choice_label,
ccproxy_provider,
status_label,
question_label,
login_prompt,
):
"""Anthropic/OpenAI wrappers share flow but keep provider-specific IDs."""
from EvoScientist.config.onboard import steps as onboard_steps
config = EvoScientistConfig(**{config_attr: "oauth"})
select_question = MagicMock()
select_question.ask.return_value = "oauth"
confirm_question = MagicMock()
confirm_question.ask.return_value = True
with (
patch(
"EvoScientist.ccproxy_manager.is_ccproxy_available", return_value=True
),
patch(
"EvoScientist.ccproxy_manager.check_ccproxy_auth",
return_value=(False, "not authenticated"),
) as mock_check_auth,
patch(
"EvoScientist.config.onboard.prompter.install_navigation_keys"
) as mock_nav,
patch(
"EvoScientist.config.onboard.steps.questionary.select",
return_value=select_question,
) as mock_select,
patch(
"EvoScientist.config.onboard.steps.questionary.confirm",
return_value=confirm_question,
) as mock_confirm,
patch(
"EvoScientist.config.onboard.steps._prompt_ccproxy_port"
) as mock_port,
patch("EvoScientist.config.onboard.steps._run_ccproxy_login") as mock_login,
):
result = getattr(onboard_steps, step_name)(config)
assert result == "oauth"
mock_nav.assert_called_once_with(select_question, with_back=True)
mock_port.assert_called_once_with(config)
mock_check_auth.assert_called_once_with(ccproxy_provider)
mock_login.assert_called_once_with(ccproxy_provider, status_label)
select_call = mock_select.call_args
assert select_call.args[0] == f"{question_label} [Esc/← to go back]:"
assert select_call.kwargs["default"] == "oauth"
choice_titles = [
choice.title
for choice in select_call.kwargs["choices"]
if getattr(choice, "value", None) in {"api_key", "oauth"}
]
assert choice_titles == [
f"API Key (direct {provider_label} access)",
f"{oauth_choice_label} (via ccproxy — no API key needed)",
]
mock_confirm.assert_called_once()
assert mock_confirm.call_args.args[0] == login_prompt
class TestStepModel:
def test_returns_selected_model(self):
"""Test that _step_model returns selected model."""
from EvoScientist.config.onboard.steps import _step_model
config = EvoScientistConfig()
with patch("EvoScientist.config.onboard.steps.questionary") as mock_q:
mock_q.select.return_value.ask.return_value = "claude-sonnet-4-6"
result = _step_model(config, "anthropic")
assert result == "claude-sonnet-4-6"
def test_main_model_not_in_provider_list_defaults_to_first(self):
"""Reset/main flow: a config.model that isn't in the chosen provider's
list (e.g. provider switched to google-genai) defaults to that
provider's first model, NOT the custom 'Type a model name...' entry."""
from EvoScientist.config.onboard.steps import _step_model
from EvoScientist.llm.models import get_models_for_provider
config = EvoScientistConfig(model="claude-sonnet-4-6")
entries = get_models_for_provider("google-genai")
with patch("EvoScientist.config.onboard.steps.questionary") as mock_q:
mock_q.select.return_value.ask.return_value = entries[0][0]
_step_model(config, "google-genai")
default = mock_q.select.call_args.kwargs["default"]
assert default == entries[0][0]
assert default != "__custom__"
def test_custom_default_value_preselects_and_prefills(self):
"""Co-pilot re-run: a saved custom (non-registry) model preselects and
prefills the 'Type a model name...' entry."""
from EvoScientist.config.onboard.steps import _step_model
config = EvoScientistConfig()
with patch("EvoScientist.config.onboard.steps.questionary") as mock_q:
mock_q.select.return_value.ask.return_value = "__custom__"
mock_q.text.return_value.ask.return_value = "my-private/model"
result = _step_model(config, "openrouter", default_value="my-private/model")
assert mock_q.select.call_args.kwargs["default"] == "__custom__"
assert mock_q.text.call_args.kwargs["default"] == "my-private/model"
assert result == "my-private/model"
def test_raises_keyboard_interrupt_on_cancel(self):
"""Test that _step_model raises KeyboardInterrupt on cancel."""
from EvoScientist.config.onboard.steps import _step_model
config = EvoScientistConfig()
with patch("EvoScientist.config.onboard.steps.questionary") as mock_q:
mock_q.select.return_value.ask.return_value = None
with pytest.raises(KeyboardInterrupt):
_step_model(config, "anthropic")
class TestStepWorkspace:
def test_returns_daemon_mode(self):
"""Test workspace step returns selected mode."""
@@ -1338,26 +1028,22 @@ class TestRunOnboard:
mock_load.return_value = EvoScientistConfig()
# Mock all questionary calls — order matches the wizard's select
# sequence: UI → Provider → Anthropic auth_mode → Model →
# Workspace → Thinking → (channels skipped via empty checkbox).
# sequence: UI → Workspace → Thinking → (channels skipped via
# empty checkbox). No provider/model steps exist anymore: models
# are configured via the WebUI / model registry.
mock_q.select.return_value.ask.side_effect = [
"tui", # UI backend
"anthropic", # Provider
"api_key", # Anthropic auth mode (API key, not OAuth)
"claude-sonnet-4-6", # Model
"skip", # Auxiliary: Skip (single driver)
"daemon", # Workspace mode
True, # Show thinking
]
mock_q.password.return_value.ask.side_effect = [
"", # Provider API key (keep current)
"", # Tavily key (keep current)
]
mock_q.confirm.return_value.ask.side_effect = [
True, # Save config
]
mock_q.text.return_value.ask.side_effect = [
"", # Workspace directory (empty = use cwd)
"", # LangGraph port (empty = keep current)
]
mock_q.checkbox.return_value.ask.return_value = [] # Skills: skip
@@ -1370,281 +1056,9 @@ class TestRunOnboard:
# against silent shift bugs where a missing side_effect entry causes
# downstream prompts to consume the wrong values.
final_config = mock_save.call_args_list[-1].args[0]
assert final_config.provider == "anthropic"
assert final_config.model == "claude-sonnet-4-6"
assert final_config.anthropic_auth_mode == "api_key"
assert final_config.ui_backend == "tui"
assert final_config.default_mode == "daemon"
def test_auxiliary_model_enabled_collects_provider_and_key(self):
"""Enabling the auxiliary step stores its provider, model, and the
chosen provider's API key (a different company than the main agent)."""
from EvoScientist.config.onboard.wizard import run_onboard
mock_q = MagicMock()
with (
_patch_all_questionary(mock_q),
patch("EvoScientist.config.onboard.wizard.load_config") as mock_load,
patch("EvoScientist.config.onboard.wizard.save_config") as mock_save,
patch("EvoScientist.config.onboard.wizard.console"),
patch("EvoScientist.config.onboard.steps.console"),
patch("EvoScientist.config.onboard.channels.console"),
patch("EvoScientist.config.onboard.helpers.console"),
patch("EvoScientist.config.onboard.wizard._step_tinytex"),
):
mock_load.return_value = EvoScientistConfig()
mock_q.select.return_value.ask.side_effect = [
"tui", # UI backend
"anthropic", # Provider
"api_key", # Anthropic auth mode
"claude-sonnet-4-6", # Model
"assemble", # Auxiliary: Assemble
"openai", # Auxiliary provider (a different company)
"api_key", # Auxiliary OpenAI auth mode
"gpt-5.5", # Auxiliary model
"daemon", # Workspace mode
True, # Show thinking
]
mock_q.password.return_value.ask.side_effect = [
"", # Main provider API key (keep current)
"sk-aux-openai", # Auxiliary provider API key
"", # Tavily key (keep current)
]
mock_q.confirm.return_value.ask.side_effect = [
True, # Save config
]
mock_q.text.return_value.ask.side_effect = [
"", # Workspace directory
]
mock_q.checkbox.return_value.ask.return_value = [] # Skills: skip
result = run_onboard(skip_validation=True)
assert result is True
final_config = mock_save.call_args_list[-1].args[0]
assert final_config.auxiliary_provider == "openai"
assert final_config.auxiliary_model == "gpt-5.5"
assert final_config.openai_auth_mode == "api_key"
# The auxiliary provider's key is stored in its per-provider field.
assert final_config.openai_api_key == "sk-aux-openai"
# Main agent is untouched.
assert final_config.provider == "anthropic"
assert final_config.model == "claude-sonnet-4-6"
def test_auxiliary_same_provider_reuses_main_credentials(self):
"""Same-provider co-pilot should not imply separate credentials exist."""
from EvoScientist.config.onboard.wizard import run_onboard
mock_q = MagicMock()
with (
_patch_all_questionary(mock_q),
patch("EvoScientist.config.onboard.wizard.load_config") as mock_load,
patch("EvoScientist.config.onboard.wizard.save_config") as mock_save,
patch("EvoScientist.config.onboard.wizard.console"),
patch("EvoScientist.config.onboard.steps.console"),
):
mock_load.return_value = EvoScientistConfig(
provider="openai",
model="gpt-5.5",
openai_api_key="sk-main-openai",
)
mock_q.select.return_value.ask.side_effect = [
"assemble", # Auxiliary: Assemble
"openai", # Same provider as the main model
"gpt-5.5", # Auxiliary model
]
mock_q.confirm.return_value.ask.side_effect = [True] # Save config
result = run_onboard(
skip_validation=True, only_sections={"auxiliary_model"}
)
assert result is True
final_config = mock_save.call_args_list[-1].args[0]
assert final_config.auxiliary_provider == "openai"
assert final_config.auxiliary_model == "gpt-5.5"
assert final_config.openai_api_key == "sk-main-openai"
mock_q.password.assert_not_called()
assert mock_q.select.return_value.ask.call_count == 3
def test_auxiliary_same_provider_prompts_when_shared_key_missing(self):
"""Same-provider reuse should not hide a missing shared API key."""
from EvoScientist.config.onboard.wizard import run_onboard
mock_q = MagicMock()
with (
_patch_all_questionary(mock_q),
patch("EvoScientist.config.onboard.wizard.load_config") as mock_load,
patch("EvoScientist.config.onboard.wizard.save_config") as mock_save,
patch("EvoScientist.config.onboard.wizard.console"),
patch("EvoScientist.config.onboard.steps.console"),
patch("EvoScientist.config.onboard.helpers.console"),
patch(
"EvoScientist.ccproxy_manager.is_ccproxy_available", return_value=True
),
):
mock_load.return_value = EvoScientistConfig(
provider="openai",
model="gpt-5.5",
openai_auth_mode="api_key",
openai_api_key="",
)
mock_q.select.return_value.ask.side_effect = [
"assemble", # Auxiliary: Assemble
"openai", # Same provider as the main model
"api_key", # Shared OpenAI auth mode
"gpt-5.5", # Auxiliary model
]
mock_q.password.return_value.ask.side_effect = [
"sk-shared-openai",
]
mock_q.confirm.return_value.ask.side_effect = [True] # Save config
result = run_onboard(
skip_validation=True, only_sections={"auxiliary_model"}
)
assert result is True
final_config = mock_save.call_args_list[-1].args[0]
assert final_config.auxiliary_provider == "openai"
assert final_config.auxiliary_model == "gpt-5.5"
assert final_config.openai_auth_mode == "api_key"
assert final_config.openai_api_key == "sk-shared-openai"
mock_q.password.assert_called_once()
assert mock_q.select.return_value.ask.call_count == 4
def test_auxiliary_openai_oauth_skips_api_key(self):
"""Auxiliary OpenAI now uses the shared auth flow and skips keys on OAuth."""
from EvoScientist.config.onboard.wizard import run_onboard
mock_q = MagicMock()
with (
_patch_all_questionary(mock_q),
patch("EvoScientist.config.onboard.wizard.load_config") as mock_load,
patch("EvoScientist.config.onboard.wizard.save_config") as mock_save,
patch("EvoScientist.config.onboard.wizard.console"),
patch("EvoScientist.config.onboard.steps.console"),
patch("EvoScientist.config.onboard.helpers.console"),
patch(
"EvoScientist.ccproxy_manager.is_ccproxy_available", return_value=True
),
patch(
"EvoScientist.ccproxy_manager.check_ccproxy_auth",
return_value=(False, "not authenticated"),
) as mock_auth,
):
mock_load.return_value = EvoScientistConfig()
mock_q.select.return_value.ask.side_effect = [
"assemble", # Auxiliary: Assemble
"openai", # Auxiliary provider
"oauth", # OpenAI auth mode
"gpt-5.5", # Auxiliary model
]
mock_q.text.return_value.ask.side_effect = [
"", # ccproxy port (keep default)
]
mock_q.confirm.return_value.ask.side_effect = [
False, # Do not log in to Codex now
True, # Save config
]
result = run_onboard(
skip_validation=True, only_sections={"auxiliary_model"}
)
assert result is True
final_config = mock_save.call_args_list[-1].args[0]
assert final_config.auxiliary_provider == "openai"
assert final_config.auxiliary_model == "gpt-5.5"
assert final_config.openai_auth_mode == "oauth"
assert final_config.openai_api_key == ""
mock_q.password.assert_not_called()
mock_auth.assert_called_once_with("codex")
def test_auxiliary_reconfigure_clears_unused_openai_oauth(self):
"""Switching co-pilot away from OpenAI clears stale OpenAI OAuth mode."""
from EvoScientist.config.onboard.wizard import run_onboard
mock_q = MagicMock()
with (
_patch_all_questionary(mock_q),
patch("EvoScientist.config.onboard.wizard.load_config") as mock_load,
patch("EvoScientist.config.onboard.wizard.save_config") as mock_save,
patch("EvoScientist.config.onboard.wizard.console"),
patch("EvoScientist.config.onboard.steps.console"),
patch("EvoScientist.config.onboard.helpers.console"),
):
mock_load.return_value = EvoScientistConfig(
provider="anthropic",
model="claude-sonnet-4-6",
anthropic_auth_mode="oauth",
auxiliary_provider="openai",
auxiliary_model="gpt-5.5",
openai_auth_mode="oauth",
)
mock_q.select.return_value.ask.side_effect = [
"assemble", # Auxiliary: Assemble
"minimax", # Auxiliary provider no longer uses OpenAI
"global", # MiniMax region
"minimax-m2", # Auxiliary model
]
mock_q.password.return_value.ask.side_effect = [
"sk-minimax", # MiniMax API key
]
mock_q.confirm.return_value.ask.side_effect = [True] # Save config
result = run_onboard(
skip_validation=True, only_sections={"auxiliary_model"}
)
assert result is True
final_config = mock_save.call_args_list[-1].args[0]
assert final_config.auxiliary_provider == "minimax"
assert final_config.openai_auth_mode == "api_key"
assert final_config.anthropic_auth_mode == "oauth"
def test_auxiliary_custom_provider_collects_base_url(self):
"""Regression for the custom-provider fix: a custom auxiliary provider
collects its base URL (provider -> base URL -> key -> model order)."""
from EvoScientist.config.onboard.wizard import run_onboard
mock_q = MagicMock()
with (
_patch_all_questionary(mock_q),
patch("EvoScientist.config.onboard.wizard.load_config") as mock_load,
patch("EvoScientist.config.onboard.wizard.save_config") as mock_save,
patch("EvoScientist.config.onboard.wizard.console"),
patch("EvoScientist.config.onboard.steps.console"),
patch("EvoScientist.config.onboard.channels.console"),
patch("EvoScientist.config.onboard.helpers.console"),
):
mock_load.return_value = EvoScientistConfig()
mock_q.select.return_value.ask.side_effect = [
"assemble", # Auxiliary: Assemble
"custom-openai", # Auxiliary provider
"gpt-5.5", # Auxiliary model (from the custom-openai registry)
]
mock_q.text.return_value.ask.side_effect = [
"https://my-endpoint/v1", # Auxiliary base URL (custom provider)
]
mock_q.password.return_value.ask.side_effect = [
"sk-aux-custom", # Auxiliary provider API key
]
mock_q.confirm.return_value.ask.side_effect = [True] # Save
result = run_onboard(
skip_validation=True, only_sections={"auxiliary_model"}
)
assert result is True
final_config = mock_save.call_args_list[-1].args[0]
assert final_config.auxiliary_provider == "custom-openai"
assert final_config.auxiliary_model == "gpt-5.5"
# Base URL must be collected for the custom auxiliary provider.
assert final_config.custom_openai_base_url == "https://my-endpoint/v1"
assert final_config.custom_openai_api_key == "sk-aux-custom"
def test_returns_false_on_cancel(self):
"""Test that run_onboard returns False when cancelled."""
from EvoScientist.config.onboard.wizard import run_onboard
@@ -1687,19 +1101,15 @@ class TestRunOnboard:
mock_q.select.return_value.ask.side_effect = [
"tui", # UI backend
"anthropic", # Provider
"api_key", # Anthropic auth mode
"claude-sonnet-4-6", # Model
"skip", # Auxiliary: Skip (single driver)
"daemon", # Workspace mode
True, # Show thinking
]
mock_q.password.return_value.ask.side_effect = ["", ""]
mock_q.password.return_value.ask.side_effect = [""]
mock_q.confirm.return_value.ask.side_effect = [
False, # Save config - NO
]
mock_q.text.return_value.ask.side_effect = [
"", # Workspace directory (empty = use cwd)
"", # LangGraph port (empty = keep current)
]
mock_q.checkbox.return_value.ask.return_value = [] # Skills: skip
@@ -1724,10 +1134,9 @@ class TestRunOnboard:
config_file = tmp_path / "config.yaml"
existing = EvoScientistConfig(
provider="openai",
model="gpt-5",
openai_api_key="sk-existing",
tavily_api_key="tvly-existing",
ui_backend="cli",
default_mode="run",
)
mock_q = MagicMock()
@@ -1758,14 +1167,10 @@ class TestRunOnboard:
mock_q.select.return_value.ask.side_effect = [
"reset", # Keep/Modify/Reset → Reset
"tui",
"anthropic",
"api_key",
"claude-sonnet-4-6",
"skip", # Auxiliary: Skip (single driver)
"daemon",
True,
]
mock_q.password.return_value.ask.side_effect = ["", ""]
mock_q.password.return_value.ask.side_effect = [""]
mock_q.confirm.return_value.ask.side_effect = [
False, # Save? = No
]
@@ -1808,14 +1213,10 @@ class TestRunOnboard:
):
mock_q.select.return_value.ask.side_effect = [
"tui",
"anthropic",
"api_key",
"claude-sonnet-4-6",
"skip", # Auxiliary: Skip (single driver)
"daemon",
True,
]
mock_q.password.return_value.ask.side_effect = ["", ""]
mock_q.password.return_value.ask.side_effect = [""]
mock_q.confirm.return_value.ask.side_effect = [
False, # Save? = No
]
-42
View File
@@ -1,42 +0,0 @@
from __future__ import annotations
import os
from EvoScientist.config.provider_admin import (
get_provider_admin_token,
get_provider_admin_token_path,
)
def test_explicit_provider_admin_token_wins(monkeypatch, tmp_path):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "explicit-token")
assert get_provider_admin_token() == "explicit-token"
assert not get_provider_admin_token_path().exists()
def test_provider_admin_token_is_created_and_reused(monkeypatch, tmp_path):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
first = get_provider_admin_token()
second = get_provider_admin_token()
path = get_provider_admin_token_path()
assert first
assert first == second
assert path.read_text(encoding="utf-8").strip() == first
if os.name != "nt":
assert path.parent.stat().st_mode & 0o777 == 0o700
assert path.stat().st_mode & 0o777 == 0o600
def test_provider_admin_token_reads_existing_file(monkeypatch, tmp_path):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
path = get_provider_admin_token_path()
path.parent.mkdir(parents=True)
path.write_text("shared-token\n", encoding="utf-8")
assert get_provider_admin_token() == "shared-token"
-280
View File
@@ -1,280 +0,0 @@
"""Tests for provider-specific model discovery and connectivity checks."""
from __future__ import annotations
from types import SimpleNamespace
from unittest.mock import patch
import httpx
import pytest
from EvoScientist.config.provider_profiles import ProviderModel, ProviderProfile
from EvoScientist.llm.provider_operations import (
ProviderOperationError,
discover_provider_models,
)
from EvoScientist.llm.provider_operations import (
test_provider_model as run_provider_model_test,
)
def _profile(
adapter: str,
*,
base_url: str = "",
api_key: str = "provider-secret",
) -> ProviderProfile:
return ProviderProfile(
id="test-provider",
name="Test provider",
adapter=adapter,
base_url=base_url,
api_key=api_key,
enabled=True,
models=(),
)
@pytest.mark.asyncio
async def test_openai_discovery_uses_bearer_token_and_models_endpoint():
def handler(request: httpx.Request) -> httpx.Response:
assert request.url == "https://api.openai.com/v1/models"
assert request.headers["Authorization"] == "Bearer provider-secret"
return httpx.Response(
200,
json={
"data": [
{"id": "gpt-z"},
{"id": "gpt-a", "name": "GPT A"},
]
},
)
models = await discover_provider_models(
_profile("openai"), transport=httpx.MockTransport(handler)
)
assert [(model.model_id, model.name) for model in models] == [
("gpt-a", "GPT A"),
("gpt-z", "gpt-z"),
]
@pytest.mark.asyncio
async def test_anthropic_discovery_uses_native_headers_and_pagination():
requests: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
assert request.url.path == "/v1/models"
assert request.headers["x-api-key"] == "provider-secret"
assert request.headers["anthropic-version"] == "2023-06-01"
if request.url.params.get("after_id"):
return httpx.Response(
200,
json={
"data": [{"id": "claude-opus", "display_name": "Claude Opus"}],
"has_more": False,
},
)
return httpx.Response(
200,
json={
"data": [{"id": "claude-sonnet"}],
"has_more": True,
"last_id": "claude-sonnet",
},
)
models = await discover_provider_models(
_profile("anthropic"), transport=httpx.MockTransport(handler)
)
assert len(requests) == 2
assert requests[1].url.params["after_id"] == "claude-sonnet"
assert [model.model_id for model in models] == ["claude-opus", "claude-sonnet"]
@pytest.mark.asyncio
async def test_google_discovery_filters_non_generation_models_and_paginates():
def handler(request: httpx.Request) -> httpx.Response:
assert request.url.path == "/v1beta/models"
assert request.headers["x-goog-api-key"] == "provider-secret"
if request.url.params.get("pageToken"):
return httpx.Response(
200,
json={
"models": [
{
"name": "models/gemini-2.5-pro",
"displayName": "Gemini 2.5 Pro",
"supportedGenerationMethods": ["generateContent"],
}
]
},
)
return httpx.Response(
200,
json={
"models": [
{
"name": "models/text-embedding-004",
"supportedGenerationMethods": ["embedContent"],
},
{
"name": "models/gemini-2.5-flash",
"supportedGenerationMethods": ["generateContent"],
},
],
"nextPageToken": "next-page",
},
)
models = await discover_provider_models(
_profile("google-genai"), transport=httpx.MockTransport(handler)
)
assert [(model.model_id, model.name) for model in models] == [
("gemini-2.5-flash", "models/gemini-2.5-flash"),
("gemini-2.5-pro", "Gemini 2.5 Pro"),
]
@pytest.mark.asyncio
async def test_grok_uses_xai_default_and_antigravity_uses_configured_proxy():
requests: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
return httpx.Response(200, json={"data": [{"id": "grok-4"}]})
transport = httpx.MockTransport(handler)
await discover_provider_models(_profile("grok"), transport=transport)
await discover_provider_models(
_profile("antigravity", base_url="http://127.0.0.1:8317/v1"),
transport=transport,
)
assert str(requests[0].url) == "https://api.x.ai/v1/models"
assert str(requests[1].url) == "http://127.0.0.1:8317/v1/models"
@pytest.mark.asyncio
async def test_ollama_discovery_uses_tags_without_api_key():
def handler(request: httpx.Request) -> httpx.Response:
assert request.url == "http://127.0.0.1:11434/api/tags"
assert "Authorization" not in request.headers
return httpx.Response(
200,
json={"models": [{"name": "qwen3:8b"}, {"model": "llama3.3:latest"}]},
)
models = await discover_provider_models(
_profile("ollama", api_key=""), transport=httpx.MockTransport(handler)
)
assert [model.model_id for model in models] == ["llama3.3:latest", "qwen3:8b"]
@pytest.mark.asyncio
async def test_discovery_requires_native_provider_api_key():
with pytest.raises(ProviderOperationError, match="API key is required"):
await discover_provider_models(_profile("openai", api_key=""))
@pytest.mark.asyncio
async def test_provider_http_error_surfaces_message_without_response_body_dump():
def handler(_request: httpx.Request) -> httpx.Response:
return httpx.Response(
401,
json={"error": {"message": "Invalid credential"}, "secret": "hidden"},
)
with pytest.raises(
ProviderOperationError, match="Provider returned HTTP 401: Invalid credential"
):
await discover_provider_models(
_profile("openai"), transport=httpx.MockTransport(handler)
)
@pytest.mark.asyncio
async def test_bare_openai_origin_uses_v1_and_reports_non_json_content_type():
def handler(request: httpx.Request) -> httpx.Response:
assert request.url == "https://api.example.test/v1/models"
return httpx.Response(
200,
text="<!doctype html>",
headers={"content-type": "text/html; charset=utf-8"},
)
with pytest.raises(ProviderOperationError) as raised:
await discover_provider_models(
_profile("openai", base_url="https://api.example.test"),
transport=httpx.MockTransport(handler),
)
message = str(raised.value)
assert "https://api.example.test/v1/models" in message
assert "Content-Type: text/html" in message
assert "normally end in /v1" in message
@pytest.mark.asyncio
async def test_discovery_retries_a_transient_connection_failure():
attempts = 0
def handler(request: httpx.Request) -> httpx.Response:
nonlocal attempts
attempts += 1
if attempts == 1:
raise httpx.ConnectTimeout("temporary timeout", request=request)
return httpx.Response(200, json={"data": [{"id": "gpt-recovered"}]})
models = await discover_provider_models(
_profile("openai"), transport=httpx.MockTransport(handler)
)
assert attempts == 2
assert [model.model_id for model in models] == ["gpt-recovered"]
@pytest.mark.asyncio
async def test_model_test_builds_draft_model_and_returns_text():
class FakeChatModel:
async def ainvoke(self, prompt: str):
assert prompt == "Reply with exactly: OK"
return SimpleNamespace(content=[{"type": "text", "text": "OK"}])
profile = _profile("grok")
model = ProviderModel(
id="grok-test", name="Grok test", model_id="grok-4", enabled=True
)
with patch(
"EvoScientist.llm.models.get_profile_chat_model", return_value=FakeChatModel()
) as build_model:
result = await run_provider_model_test(profile, model)
assert result.response == "OK"
assert result.latency_ms >= 1
build_model.assert_called_once_with(
profile,
model,
max_tokens=64,
skip_auto_config=True,
)
@pytest.mark.asyncio
async def test_model_test_redacts_api_key_from_errors():
profile = _profile("openai", api_key="secret-value")
model = ProviderModel(id="test", name="Test", model_id="gpt-test")
with patch(
"EvoScientist.llm.models.get_profile_chat_model",
side_effect=ValueError("request rejected for secret-value"),
):
with pytest.raises(ProviderOperationError) as raised:
await run_provider_model_test(profile, model)
assert "secret-value" not in str(raised.value)
assert "[redacted]" in str(raised.value)
-485
View File
@@ -1,485 +0,0 @@
"""Tests for runtime-editable provider profiles."""
from __future__ import annotations
import os
from unittest.mock import patch
import pytest
from EvoScientist.config import EvoScientistConfig, apply_config_to_env, save_config
from EvoScientist.config.provider_profiles import (
ProviderProfileError,
get_builtin_provider_profile,
get_provider_profile_revision,
get_provider_profiles_path,
list_configured_model_entries,
load_provider_profiles,
provider_profiles_public,
replace_provider_profiles,
resolve_provider_profile_draft,
)
from EvoScientist.llm.models import get_chat_model, list_model_picker_entries
@pytest.fixture
def provider_config_dir(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
return tmp_path / "evoscientist"
def _document(api_key: str = "secret-key") -> dict:
return {
"version": 3,
"providers": [
{
"id": "lab-openai",
"name": "Lab OpenAI",
"adapter": "openai-compatible",
"base_url": "https://llm.example.test/v1/",
"api_key": api_key,
"enabled": True,
"runtime": {
"timeout_seconds": 90,
"max_retries": 1,
"default_temperature": 0.3,
"default_top_p": None,
"default_reasoning_effort": "auto",
},
"models": [
{
"id": "research-model",
"name": "Research Model",
"model_id": "vendor/research-1",
"enabled": True,
"runtime": {
"limit_mode": "combined",
"context_window_tokens": 32768,
"max_output_tokens": 4096,
"min_effective_input_tokens": 4096,
"limits_status": "confirmed",
"limits_source": "user",
},
},
{
"id": "disabled-model",
"name": "Disabled Model",
"model_id": "vendor/disabled",
"enabled": False,
"runtime": {
"limit_mode": "combined",
"context_window_tokens": 32768,
"max_output_tokens": 4096,
"min_effective_input_tokens": 4096,
"limits_status": "confirmed",
"limits_source": "user",
},
},
],
}
]
}
def _builtin_document(api_key: str = "builtin-secret") -> dict:
return {
"version": 3,
"builtins": [
{
"id": "openai",
"name": "OpenAI",
"adapter": "openai",
"base_url": "https://proxy.example.test/v1",
"api_key": api_key,
"auth_mode": "api_key",
"enabled": True,
"runtime": {},
"models": [
{
"id": "chat-main",
"name": "Chat Main",
"model_id": "gpt-upstream",
"enabled": True,
"runtime": {
"limit_mode": "combined",
"context_window_tokens": 32768,
"max_output_tokens": 4096,
"min_effective_input_tokens": 4096,
"limits_status": "confirmed",
"limits_source": "user",
},
}
],
}
]
}
def test_replace_round_trip_and_redacts_api_key(provider_config_dir):
saved = replace_provider_profiles(_document())
assert saved.providers[0].base_url == "https://llm.example.test/v1"
assert load_provider_profiles() == saved
public = provider_profiles_public(saved)
assert "api_key" not in public["providers"][0]
assert public["providers"][0]["api_key_configured"] is True
assert public["providers"][0]["api_key_hint"] == "sec...-key"
assert "openai" in public["reserved_provider_ids"]
path = get_provider_profiles_path()
assert path == provider_config_dir / "providers.yaml"
if os.name != "nt":
assert path.parent.stat().st_mode & 0o777 == 0o700
assert path.stat().st_mode & 0o777 == 0o600
def test_v1_document_requires_reset(provider_config_dir):
path = get_provider_profiles_path()
path.parent.mkdir(parents=True)
path.write_text(
"version: 1\nproviders:\n"
" - id: lab-openai\n"
" name: Lab OpenAI\n"
" adapter: openai-compatible\n"
" base_url: https://llm.example.test/v1\n"
" api_key: old-secret\n"
" enabled: true\n"
" models: []\n",
encoding="utf-8",
)
with pytest.raises(ProviderProfileError, match="PROVIDER_PROFILE_RESET_REQUIRED"):
load_provider_profiles()
def test_builtin_profiles_round_trip_and_redact_secret(provider_config_dir):
saved = replace_provider_profiles(_builtin_document())
assert saved.builtins[0].id == "openai"
assert get_builtin_provider_profile("openai") == saved.builtins[0]
public = provider_profiles_public(saved)
assert public["builtins"][0]["api_key_configured"] is True
assert "builtin-secret" not in str(public)
def test_builtin_private_revision_changes_with_connection_secret(
provider_config_dir,
):
replace_provider_profiles(_builtin_document("first-secret"))
first = get_provider_profile_revision("openai")
replace_provider_profiles(_builtin_document("second-secret"))
second = get_provider_profile_revision("openai")
assert first
assert second
assert first != second
def test_custom_update_preserves_builtin_profiles(provider_config_dir):
replace_provider_profiles(_builtin_document())
saved = replace_provider_profiles(_document())
assert saved.builtins[0].id == "openai"
assert saved.providers[0].id == "lab-openai"
def test_builtin_update_preserves_custom_profiles(provider_config_dir):
replace_provider_profiles(_document())
saved = replace_provider_profiles(_builtin_document())
assert saved.builtins[0].id == "openai"
assert saved.providers[0].id == "lab-openai"
def test_empty_api_key_keeps_existing_secret(provider_config_dir):
replace_provider_profiles(_document("first-secret"))
update = _document("")
update["providers"][0]["name"] = "Renamed"
saved = replace_provider_profiles(update)
assert saved.providers[0].name == "Renamed"
assert saved.providers[0].api_key == "first-secret"
def test_public_revision_does_not_fingerprint_api_key(provider_config_dir):
first = provider_profiles_public(
replace_provider_profiles(_document("abc-first-1234"))
)["revision"]
second = provider_profiles_public(
replace_provider_profiles(_document("abc-second-1234"))
)["revision"]
assert first == second
def test_clear_api_key_is_explicit(provider_config_dir):
replace_provider_profiles(_document("first-secret"))
update = _document("")
update["providers"][0]["clear_api_key"] = True
saved = replace_provider_profiles(update)
assert saved.providers[0].api_key == ""
def test_draft_reuses_saved_api_key_without_exposing_it(provider_config_dir):
replace_provider_profiles(_document("saved-secret"))
draft = _document("")["providers"][0]
draft["models"] = []
resolved = resolve_provider_profile_draft(draft)
assert resolved.api_key == "saved-secret"
assert resolved.models == ()
def test_draft_can_explicitly_clear_saved_api_key(provider_config_dir):
replace_provider_profiles(_document("saved-secret"))
draft = _document("")["providers"][0]
draft.update({"models": [], "clear_api_key": True})
resolved = resolve_provider_profile_draft(draft)
assert resolved.api_key == ""
def test_reserved_provider_id_is_rejected(provider_config_dir):
raw = _document()
raw["providers"][0]["id"] = "openai"
with pytest.raises(ProviderProfileError, match="openai-custom"):
replace_provider_profiles(raw)
@pytest.mark.parametrize(
("adapter", "expected"),
[
("openai", "https://llm.example.test/v1"),
("openai-compatible", "https://llm.example.test/v1"),
("openrouter", "https://llm.example.test/api/v1"),
("google-genai", "https://llm.example.test/v1beta"),
("anthropic", "https://llm.example.test"),
],
)
def test_bare_provider_origin_gets_adapter_api_path(
provider_config_dir, adapter, expected
):
raw = _document()
raw["providers"][0].update(
{
"id": "lab-provider",
"adapter": adapter,
"base_url": "https://llm.example.test",
}
)
saved = replace_provider_profiles(raw)
assert saved.providers[0].base_url == expected
def test_dynamic_models_are_listed_and_disabled_models_are_hidden(
provider_config_dir,
):
replace_provider_profiles(_document())
assert list_configured_model_entries() == [
("research-model", "vendor/research-1", "lab-openai")
]
@pytest.mark.asyncio
async def test_custom_registry_replaces_static_picker_entries(provider_config_dir):
replace_provider_profiles(_document())
entries = await list_model_picker_entries(None, include_custom_ollama=False)
assert entries == [("research-model", "vendor/research-1", "lab-openai")]
@pytest.mark.asyncio
async def test_empty_registry_hides_static_picker_entries(provider_config_dir):
replace_provider_profiles({"providers": []})
entries = await list_model_picker_entries(None, include_custom_ollama=False)
assert entries == []
@pytest.mark.asyncio
async def test_managed_builtins_replace_static_picker_entries(provider_config_dir):
replace_provider_profiles(_builtin_document())
entries = await list_model_picker_entries(None, include_custom_ollama=False)
assert entries == [("chat-main", "gpt-upstream", "openai")]
def test_dynamic_profile_routes_through_selected_adapter(provider_config_dir):
replace_provider_profiles(_document())
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("research-model", provider="lab-openai")
_, kwargs = init_model.call_args
assert kwargs["model"] == "vendor/research-1"
assert kwargs["model_provider"] == "openai"
assert kwargs["base_url"] == "https://llm.example.test/v1"
assert kwargs["api_key"] == "secret-key"
assert kwargs["timeout"] == 90
assert kwargs["max_retries"] == 1
assert kwargs["max_tokens"] == 4096
assert kwargs["default_headers"]["User-Agent"] == "codex_cli_rs/0.0.0"
def test_dynamic_native_openai_adapter_uses_gateway_compatible_user_agent(
provider_config_dir,
):
raw = _document()
raw["providers"][0]["adapter"] = "openai"
replace_provider_profiles(raw)
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("research-model", provider="lab-openai")
_, kwargs = init_model.call_args
assert kwargs["model_provider"] == "openai"
assert kwargs["default_headers"]["User-Agent"] == "codex_cli_rs/0.0.0"
def test_builtin_catalog_alias_routes_to_upstream_model(provider_config_dir):
save_config(
EvoScientistConfig(
provider="openai",
model="chat-main",
model_catalog=[
{
"provider": "openai",
"id": "chat-main",
"name": "Chat Main",
"model_id": "gpt-upstream",
"enabled": True,
}
],
)
)
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("chat-main", provider="openai")
_, kwargs = init_model.call_args
assert kwargs["model"] == "gpt-upstream"
assert kwargs["model_provider"] == "openai"
def test_managed_builtin_routes_connection_without_restart(
provider_config_dir, monkeypatch
):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
replace_provider_profiles(_builtin_document())
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("chat-main", provider="openai")
_, kwargs = init_model.call_args
assert kwargs["model"] == "gpt-upstream"
assert kwargs["model_provider"] == "openai"
assert kwargs["base_url"] == "https://proxy.example.test/v1"
assert kwargs["api_key"] == "builtin-secret"
def test_environment_overrides_managed_builtin_connection(
provider_config_dir, monkeypatch
):
replace_provider_profiles(_builtin_document())
monkeypatch.setenv("OPENAI_API_KEY", "environment-secret")
monkeypatch.setenv("OPENAI_BASE_URL", "https://env.example.test/v1")
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("chat-main", provider="openai")
_, kwargs = init_model.call_args
assert kwargs["base_url"] == "https://env.example.test/v1"
assert kwargs["api_key"] == "environment-secret"
def test_managed_builtin_overrides_legacy_config_injected_environment(
provider_config_dir, monkeypatch
):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
apply_config_to_env(
EvoScientistConfig(
openai_api_key="legacy-secret",
)
)
replace_provider_profiles(_builtin_document("registry-secret"))
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("chat-main", provider="openai")
_, kwargs = init_model.call_args
assert kwargs["api_key"] == "registry-secret"
def test_dynamic_ollama_profile_does_not_pass_api_key(provider_config_dir):
raw = _document("unused-secret")
raw["providers"][0].update(
{
"id": "local-ollama",
"adapter": "ollama",
"base_url": "http://127.0.0.1:11434",
}
)
replace_provider_profiles(raw)
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("research-model", provider="local-ollama")
_, kwargs = init_model.call_args
assert kwargs["model_provider"] == "ollama"
assert kwargs["base_url"] == "http://127.0.0.1:11434"
assert "api_key" not in kwargs
@pytest.mark.parametrize(
("adapter", "base_url"),
[
("grok", ""),
("antigravity", "http://127.0.0.1:8317/v1"),
],
)
def test_openai_protocol_adapters_route_through_openai(
provider_config_dir, adapter, base_url
):
raw = _document()
raw["providers"][0].update(
{
"id": f"lab-{adapter}",
"adapter": adapter,
"base_url": base_url,
}
)
replace_provider_profiles(raw)
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("research-model", provider=f"lab-{adapter}")
_, kwargs = init_model.call_args
assert kwargs["model_provider"] == "openai"
assert kwargs["base_url"] == (base_url or "https://api.x.ai/v1")
def test_revision_changes_when_credentials_change(provider_config_dir):
replace_provider_profiles(_document("first-secret"))
first = get_provider_profile_revision("lab-openai")
replace_provider_profiles(_document("second-secret"))
second = get_provider_profile_revision("lab-openai")
assert first
assert second
assert first != second
+1 -70
View File
@@ -16,7 +16,7 @@ def _make_ui(**kwargs):
class TestBasicIO:
"""Core CommandUI methods used by /model path."""
"""Core CommandUI console I/O methods."""
def test_supports_interactive_true(self):
ui, _ = _make_ui()
@@ -45,75 +45,6 @@ class TestBasicIO:
console.print.assert_not_called()
class TestWaitForModelPick:
"""CLI model picker fallback: print table + return None."""
async def test_returns_none(self):
ui, _ = _make_ui()
entries = [
("claude-sonnet-4-6", "anthropic/claude-sonnet", "anthropic"),
("gpt-4o", "openai/gpt-4o", "openai"),
]
result = await ui.wait_for_model_pick(
entries,
current_model="claude-sonnet-4-6",
current_provider="anthropic",
)
assert result is None
async def test_prints_table_with_current_model_marker(self):
ui, console = _make_ui()
entries = [
("claude-sonnet-4-6", "anthropic/claude-sonnet", "anthropic"),
("gpt-4o", "openai/gpt-4o", "openai"),
]
await ui.wait_for_model_pick(
entries,
current_model="claude-sonnet-4-6",
current_provider="anthropic",
)
# First call renders the Table (Rich renderable), second prints usage.
assert console.print.call_count == 2
first_arg = console.print.call_args_list[0].args[0]
assert isinstance(first_arg, Table)
usage_arg = console.print.call_args_list[1].args[0]
assert "Usage: /model" in usage_arg
assert "--save" in usage_arg
async def test_no_current_model_no_marker(self):
ui, console = _make_ui()
entries = [("claude-sonnet-4-6", "anthropic/claude-sonnet", "anthropic")]
await ui.wait_for_model_pick(
entries,
current_model=None,
current_provider=None,
)
# Just asserts the coroutine runs without marker-branch issues.
assert console.print.call_count == 2
async def test_empty_entries_still_prints_header_and_usage(self):
ui, console = _make_ui()
result = await ui.wait_for_model_pick(
[],
current_model=None,
current_provider=None,
)
assert result is None
# Header table + usage hint should still be printed even with
# no entries.
assert console.print.call_count == 2
class TestUpdateStatusHook:
"""update_status_after_model_change is a deliberate no-op on CLI."""
def test_no_op(self):
ui, console = _make_ui()
ui.update_status_after_model_change("claude-opus-4-8", "anthropic")
console.print.assert_not_called()
class TestPhaseAMigrated:
"""Phase A migration: quit, clear, thread-pick fallback, and compact status.
+2 -1
View File
@@ -65,10 +65,11 @@ def test_runtime_context_injects_current_date_and_timezone():
"EvoScientist.middleware.create_tool_selector_middleware",
return_value=[MagicMock(), MagicMock()],
)
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_default_middleware_includes_runtime_context(
mock_config, mock_model, mock_tool_selector
mock_config, mock_model, mock_aux_model, mock_tool_selector
):
mock_config.return_value = _mock_config()
mock_model.return_value = MagicMock(profile={"max_input_tokens": 200_000})
-152
View File
@@ -1,152 +0,0 @@
"""Tests for server-side run configuration snapshots."""
from __future__ import annotations
from unittest.mock import MagicMock, patch
import pytest
from EvoScientist.config.provider_profiles import (
ProviderProfileError,
replace_provider_profiles,
)
from EvoScientist.llm.runtime_snapshots import (
create_run_runtime_snapshot,
get_run_runtime_snapshot,
get_snapshot_chat_model,
)
@pytest.fixture
def provider_config_dir(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
return tmp_path / "evoscientist"
def _document(api_key: str = "first-secret", base_url: str = "https://one.test/v1"):
return {
"version": 3,
"providers": [
{
"id": "lab-openai",
"name": "Lab OpenAI",
"adapter": "openai-compatible",
"base_url": base_url,
"api_key": api_key,
"enabled": True,
"runtime": {"timeout_seconds": 90, "max_retries": 1},
"models": [
{
"id": "research-model",
"name": "Research Model",
"model_id": "vendor/research-1",
"enabled": True,
"runtime": {
"limit_mode": "combined",
"context_window_tokens": 32_768,
"max_output_tokens": 4_096,
"min_effective_input_tokens": 4_096,
"limits_status": "confirmed",
"limits_source": "provider",
},
}
],
}
],
}
def test_snapshot_keeps_connection_and_runtime_after_profile_changes(provider_config_dir):
replace_provider_profiles(_document())
created = create_run_runtime_snapshot(
"run-request-a", model="research-model", provider="lab-openai"
)
assert created is not None
assert created.options.resolved_input_limit == 28_672
assert created.profile.api_key == "first-secret"
assert "first-secret" not in str(created.public_payload())
assert "first-secret" not in (
provider_config_dir / "run-runtime-snapshots.sqlite3"
).read_bytes().decode("utf-8", errors="ignore")
replace_provider_profiles(
_document("second-secret", "https://two.test/v1")
)
loaded = get_run_runtime_snapshot("run-request-a")
assert loaded is not None
assert loaded.profile.api_key == "first-secret"
assert loaded.profile.base_url == "https://one.test/v1"
assert loaded.options.timeout_seconds == 90
def test_snapshot_id_is_idempotent_and_cannot_change_model(provider_config_dir):
replace_provider_profiles(_document())
first = create_run_runtime_snapshot(
"run-request-a", model="research-model", provider="lab-openai"
)
repeated = create_run_runtime_snapshot(
"run-request-a", model="research-model", provider="lab-openai"
)
assert repeated == first
with pytest.raises(ValueError, match="already bound"):
create_run_runtime_snapshot(
"run-request-a", model="another-model", provider="lab-openai"
)
def test_static_provider_does_not_need_a_custom_snapshot(provider_config_dir):
assert (
create_run_runtime_snapshot(
"run-request-a", model="gpt-5", provider="openai"
)
is None
)
def test_glm_alias_is_static_even_when_custom_registry_is_stale(
provider_config_dir, monkeypatch
):
monkeypatch.setenv("ZHIPU_API_KEY", "test-zhipu-key")
provider_config_dir.mkdir(parents=True)
(provider_config_dir / "providers.yaml").write_text(
"version: 2\nproviders: []\n", encoding="utf-8"
)
assert (
create_run_runtime_snapshot(
"run-request-glm", model="glm-5.2", provider="glm"
)
is None
)
def test_zhipu_snapshot_requires_its_own_api_key(provider_config_dir, monkeypatch):
monkeypatch.delenv("ZHIPU_API_KEY", raising=False)
monkeypatch.setenv("OPENAI_API_KEY", "unrelated-openai-key")
with pytest.raises(ProviderProfileError, match="ZHIPU_API_KEY_NOT_CONFIGURED"):
create_run_runtime_snapshot(
"run-request-zhipu", model="glm-5.2", provider="zhipu"
)
def test_snapshot_model_builder_uses_frozen_profile(provider_config_dir):
replace_provider_profiles(_document())
create_run_runtime_snapshot(
"run-request-a", model="research-model", provider="lab-openai"
)
model = MagicMock()
with patch(
"EvoScientist.llm.models.get_profile_chat_model", return_value=model
) as factory:
resolved = get_snapshot_chat_model("run-request-a")
assert resolved is model
profile, provider_model = factory.call_args.args
assert profile.api_key == "first-secret"
assert provider_model.model_id == "vendor/research-1"
+31
View File
@@ -209,3 +209,34 @@ class TestEndpointPolicyWiring:
runtime = SnapshotRuntime(store, endpoint_policy=policy)
model = runtime.build_default_role_model("auxiliary")
assert isinstance(model, ChatOllama)
class TestCreateLocalSnapshot:
"""Section 8.1 local entry: fixed binding convention through SnapshotService."""
def test_bootstrap_registry_fails_closed(self, tmp_path):
runtime = SnapshotRuntime(ModelRuntimeStore(config_dir=tmp_path / "db"))
with pytest.raises(ModelRegistryError) as excinfo:
runtime.create_local_snapshot("thread-1")
assert excinfo.value.code == MODEL_REGISTRY_NOT_READY
def test_binding_convention_is_fixed(self, store, runtime):
from EvoScientist.model_registry.snapshots import config_for_role
snapshot = runtime.create_local_snapshot("cli-thread-1")
assert snapshot.thread_id == "cli-thread-1"
assert snapshot.deployment_id == runtime.local_deployment_id
assert snapshot.payload.model_selection_revision == 0
# primary=None inherits — the registry defaults are frozen at creation.
assert config_for_role(snapshot, "primary").model_ref == ZHIPU_REF
assert config_for_role(snapshot, "auxiliary").model_ref == OLLAMA_REF
def test_same_run_request_id_reuses_snapshot(self, store, runtime):
first = runtime.create_local_snapshot("t-1", run_request_id="req-1")
second = runtime.create_local_snapshot("t-1", run_request_id="req-1")
assert second.snapshot_id == first.snapshot_id
def test_default_run_request_id_freezes_per_call(self, store, runtime):
first = runtime.create_local_snapshot("t-1")
second = runtime.create_local_snapshot("t-1")
assert second.snapshot_id != first.snapshot_id
+4 -2
View File
@@ -305,9 +305,10 @@ def test_tracker_captures_tools():
"EvoScientist.middleware.create_tool_selector_middleware",
side_effect=lambda *a, **kw: _patched_create(),
)
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_default_middleware_includes_tool_selector(mock_config, mock_model, mock_ts):
def test_default_middleware_includes_tool_selector(mock_config, mock_model, mock_aux, mock_ts):
mock_model.return_value = _mock_model()
cfg = MagicMock()
cfg.enable_ask_user = False
@@ -341,9 +342,10 @@ def test_subagent_no_tool_selector(mock_model):
"EvoScientist.middleware.create_tool_selector_middleware",
side_effect=lambda *a, **kw: _patched_create(),
)
@patch("EvoScientist.EvoScientist._ensure_auxiliary_chat_model")
@patch("EvoScientist.EvoScientist._ensure_chat_model")
@patch("EvoScientist.EvoScientist._ensure_config")
def test_tool_selector_ordering(mock_config, mock_model, mock_ts):
def test_tool_selector_ordering(mock_config, mock_model, mock_aux, mock_ts):
"""ToolSelector should come after ToolErrorHandler and before Memory."""
mock_model.return_value = _mock_model()
cfg = MagicMock()
+7
View File
@@ -25,6 +25,13 @@ import pytest
pytest.importorskip("textual")
@pytest.fixture(autouse=True)
def _active_registry(active_snapshot_runtime):
"""The TUI status bar resolves the compile-time model from the registry;
bootstrap fails closed, so activate it for these tests."""
return active_snapshot_runtime
# ---------------------------------------------------------------------------
# Module access
#
+6 -8
View File
@@ -42,9 +42,10 @@ class _StubApp:
self.refresh_calls.append(reset_streaming_text)
async def test_sync_tui_command_completion_adopts_agent_swap(monkeypatch):
async def test_sync_tui_command_completion_adopts_agent_swap(
monkeypatch, active_snapshot_runtime
):
import EvoScientist.cli.tui_interactive as tui_mod
from EvoScientist import EvoScientist as evosci_mod
app = _StubApp()
ctx = CommandContext(
@@ -54,18 +55,15 @@ async def test_sync_tui_command_completion_adopts_agent_swap(monkeypatch):
)
cmd = SimpleNamespace(name="/model")
monkeypatch.setattr(
evosci_mod,
"_ensure_config",
lambda: SimpleNamespace(model="gpt-5.5", provider="openai"),
)
monkeypatch.setattr(tui_mod, "_channels_is_running", lambda: True)
app._channel_runtime.bind("old-agent", "old-thread")
await tui_mod._sync_tui_command_completion(app, ctx, "old-agent", cmd)
assert app._agent_loader.adopt_calls == ["new-agent"]
assert app.model_updates == [("gpt-5.5", "openai")]
# The status label comes from the registry defaults (provider_id/model_key),
# not from config free strings.
assert app.model_updates == [("zhipu-glm/glm-5.2", None)]
assert app.refresh_calls == [True]
assert app._channel_runtime.agent == "new-agent"
assert app._channel_runtime.thread_id == "thread-1"
-273
View File
@@ -1090,276 +1090,3 @@ class TestCompletionLogic(unittest.TestCase):
assert handled is False
assert app._comp_index == 0 # unchanged
@unittest.skipUnless(_has_textual, "textual not installed")
class TestModelPickerWidgetOllama(unittest.TestCase):
"""ModelPickerWidget Ollama fallback: sentinel row renders under the
ollama group, selecting it enters free-text input mode, Enter confirms
with ``Picked(typed, "ollama")``, Esc returns to list, and filtering
never hides the sentinel."""
def _make_widget(self, entries=None, *, current_model=None):
"""Build a widget with a mix of providers + the sentinel row.
When ``entries`` is ``None``, a default mix (anthropic + one ollama
model + sentinel) is used. When provided, it fully replaces the
default — callers that need extra rows should pass the complete list.
"""
from unittest.mock import MagicMock
from EvoScientist.cli.widgets.model_picker import (
_CUSTOM_OLLAMA_ID,
ModelPickerWidget,
)
if entries is None:
entries = [
("claude-sonnet-4-6", "claude-sonnet-4-6", "anthropic"),
("llama3.3", "llama3.3", "ollama"),
("Custom Ollama model...", _CUSTOM_OLLAMA_ID, "ollama"),
]
w = ModelPickerWidget(entries, current_model=current_model)
# Stub out Textual-dependent side effects so we can drive actions
# directly (follows the module's "no pilot" test pattern).
w.post_message = MagicMock()
w.focus = MagicMock()
# Fake the Input child — real one requires a mounted app.
custom_input = MagicMock()
custom_input.value = ""
custom_input.display = False
w._custom_input = custom_input
return w
def _sentinel_index(self, widget):
from EvoScientist.cli.widgets.model_picker import _CUSTOM_OLLAMA_ID
for i, item in enumerate(widget._items):
if item["type"] == "model" and item.get("model_id") == _CUSTOM_OLLAMA_ID:
return i
raise AssertionError(f"sentinel not found in items: {widget._items}")
def test_sentinel_rendered_under_ollama_group(self):
w = self._make_widget()
# The sentinel must be grouped under an "ollama" header.
headers = [i["label"] for i in w._items if i["type"] == "header"]
assert "ollama" in headers
def test_selecting_regular_row_posts_picked(self):
"""Baseline: non-sentinel selection still works."""
from EvoScientist.cli.widgets.model_picker import ModelPickerWidget
w = self._make_widget()
# Find the claude row
claude_idx = next(
i
for i, item in enumerate(w._items)
if item["type"] == "model" and item["name"] == "claude-sonnet-4-6"
)
w._selected = claude_idx
w.action_select()
assert w._mode == "list"
msgs = [c.args[0] for c in w.post_message.call_args_list]
assert any(
isinstance(m, ModelPickerWidget.Picked)
and m.name == "claude-sonnet-4-6"
and m.provider == "anthropic"
for m in msgs
)
def test_selecting_sentinel_enters_input_mode(self):
w = self._make_widget()
w._selected = self._sentinel_index(w)
w.action_select()
assert w._mode == "input"
assert w._custom_input.display is True
w._custom_input.focus.assert_called_once()
# Entering input mode must NOT post any message — the user hasn't
# submitted anything yet.
w.post_message.assert_not_called()
def test_enter_with_typed_name_posts_picked(self):
from EvoScientist.cli.widgets.model_picker import ModelPickerWidget
w = self._make_widget()
w._mode = "input"
w._custom_input.value = "qwen3-coder-next"
w._custom_input.display = True
w.action_select()
msgs = [c.args[0] for c in w.post_message.call_args_list]
picked = [m for m in msgs if isinstance(m, ModelPickerWidget.Picked)]
assert len(picked) == 1
assert picked[0].name == "qwen3-coder-next"
assert picked[0].provider == "ollama"
def test_enter_with_empty_input_is_noop(self):
w = self._make_widget()
w._mode = "input"
w._custom_input.value = ""
w.action_select()
w.post_message.assert_not_called()
# Still in input mode — user can keep typing or Esc out.
assert w._mode == "input"
def test_enter_with_whitespace_only_input_is_noop(self):
w = self._make_widget()
w._mode = "input"
w._custom_input.value = " \t "
w.action_select()
w.post_message.assert_not_called()
assert w._mode == "input"
def test_esc_in_input_mode_returns_to_list(self):
from EvoScientist.cli.widgets.model_picker import ModelPickerWidget
w = self._make_widget()
w._mode = "input"
w._custom_input.value = "partial"
w._custom_input.display = True
w.action_cancel()
assert w._mode == "list"
assert w._custom_input.display is False
assert w._custom_input.value == ""
# No Cancelled message — Esc from input returns to list, not closes.
cancelled = [
c.args[0]
for c in w.post_message.call_args_list
if isinstance(c.args[0], ModelPickerWidget.Cancelled)
]
assert cancelled == []
def test_esc_in_list_mode_cancels(self):
from EvoScientist.cli.widgets.model_picker import ModelPickerWidget
w = self._make_widget()
w._mode = "list"
w.action_cancel()
msgs = [c.args[0] for c in w.post_message.call_args_list]
assert any(isinstance(m, ModelPickerWidget.Cancelled) for m in msgs)
def test_up_in_input_mode_exits_to_list(self):
w = self._make_widget()
w._mode = "input"
w._custom_input.display = True
w.action_move_up()
assert w._mode == "list"
assert w._custom_input.display is False
def test_down_in_input_mode_absorbed(self):
w = self._make_widget()
w._mode = "input"
before_selected = w._selected
w._custom_input.display = True
w.action_move_down()
# State unchanged — key was absorbed.
assert w._mode == "input"
assert w._custom_input.display is True
assert w._selected == before_selected
def test_backspace_in_input_mode_no_filter_change(self):
w = self._make_widget()
w._mode = "input"
w._filter_text = "foo"
w.action_backspace()
# Input widget handles its own backspace — filter unchanged.
assert w._filter_text == "foo"
def test_printable_key_in_input_mode_does_not_filter(self):
from unittest.mock import MagicMock
w = self._make_widget()
w._mode = "input"
w._filter_text = ""
event = MagicMock()
event.key = "a"
event.character = "a"
w.on_key(event)
assert w._filter_text == ""
def test_duplicate_sentinels_collapsed(self):
"""Defense-in-depth: even if callers pass two sentinel rows (state
reuse, stale merges), only one "Custom Ollama model..." renders."""
from EvoScientist.cli.widgets.model_picker import (
_CUSTOM_OLLAMA_ID,
_build_items,
)
entries = [
("claude-sonnet-4-6", "claude-sonnet-4-6", "anthropic"),
("Custom Ollama model...", _CUSTOM_OLLAMA_ID, "ollama"),
("Custom Ollama model...", _CUSTOM_OLLAMA_ID, "ollama"),
("Custom Ollama model...", _CUSTOM_OLLAMA_ID, "ollama"),
]
items = _build_items(entries)
sentinel_rows = [
i
for i in items
if i["type"] == "model" and i.get("model_id") == _CUSTOM_OLLAMA_ID
]
assert len(sentinel_rows) == 1, f"duplicate sentinels rendered: {items}"
def test_sentinel_survives_filter(self):
"""The Custom Ollama row is the user's escape hatch — filtering
must never hide it."""
from EvoScientist.cli.widgets.model_picker import (
_CUSTOM_OLLAMA_ID,
_build_items,
)
entries = [
("claude-sonnet-4-6", "claude-sonnet-4-6", "anthropic"),
("llama3.3", "llama3.3", "ollama"),
("Custom Ollama model...", _CUSTOM_OLLAMA_ID, "ollama"),
]
# A filter that matches NOTHING in the normal entries.
items = _build_items(entries, filter_text="zzzzzz")
sentinel_rows = [
i
for i in items
if i["type"] == "model" and i.get("model_id") == _CUSTOM_OLLAMA_ID
]
assert len(sentinel_rows) == 1, f"sentinel hidden by filter: {items}"
def test_on_input_submitted_routes_to_submit(self):
"""Belt-and-suspenders: Enter fired inside the Input widget should
be handled the same way as action_select in input mode."""
from unittest.mock import MagicMock
from EvoScientist.cli.widgets.model_picker import ModelPickerWidget
w = self._make_widget()
w._mode = "input"
w._custom_input.value = "mymodel"
event = MagicMock()
event.input = w._custom_input # the Input child we stubbed
w.on_input_submitted(event)
event.stop.assert_called_once()
msgs = [c.args[0] for c in w.post_message.call_args_list]
picked = [m for m in msgs if isinstance(m, ModelPickerWidget.Picked)]
assert len(picked) == 1
assert picked[0].name == "mymodel"
assert picked[0].provider == "ollama"
if __name__ == "__main__":
unittest.main()
+7
View File
@@ -14,6 +14,13 @@ from EvoScientist.langgraph_dev.http import app
from EvoScientist.workspace_scope import DeferredScopedBackend, create_workspace_backend
@pytest.fixture(autouse=True)
def _active_registry(active_snapshot_runtime):
"""Run entry points freeze a local snapshot per run; that fails closed
against the default bootstrap registry, so activate it for these tests."""
return active_snapshot_runtime
def test_scope_routes_require_service_token(monkeypatch):
monkeypatch.delenv("EVOSCIENTIST_BACKEND_SERVICE_TOKEN", raising=False)
with TestClient(app) as client:
Generated
+2
View File
@@ -966,6 +966,7 @@ dependencies = [
{ name = "prompt-toolkit" },
{ name = "psutil" },
{ name = "pydantic" },
{ name = "pyjwt" },
{ name = "python-dotenv" },
{ name = "pyyaml" },
{ name = "questionary" },
@@ -1078,6 +1079,7 @@ requires-dist = [
{ name = "pycryptodome", marker = "extra == 'all-channels'", specifier = ">=3.20" },
{ name = "pycryptodome", marker = "extra == 'wechat'", specifier = ">=3.20" },
{ name = "pydantic", specifier = ">=2.10" },
{ name = "pyjwt", specifier = ">=2.8" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" },
{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.0" },
{ name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0" },