feat: add /model command for changing models inside TUI/CLI (#162)
* rebase main * fix: apply pr comments * fix: fix critical issue * fix: ordering /model in cli mode * feat: refactor /model command handling and add Rich CLI support --------- Co-authored-by: X-iZhang <zacharyzhang2022@gmail.com>
This commit is contained in:
Wiktor Cupiał
@@ -0,0 +1,321 @@
|
||||
"""Tests for the /model command and extract_model_and_provider helper."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import run_async as _run
|
||||
|
||||
|
||||
class TestExtractModelAndProvider:
|
||||
"""Unit tests for the argument parser helper."""
|
||||
|
||||
def test_known_model_no_provider(self):
|
||||
from EvoScientist.commands.implementation.model import (
|
||||
extract_model_and_provider,
|
||||
)
|
||||
|
||||
name, prov = extract_model_and_provider(["claude-sonnet-4-6"])
|
||||
assert name == "claude-sonnet-4-6"
|
||||
assert prov == "anthropic"
|
||||
|
||||
def test_known_model_with_provider_override(self):
|
||||
from EvoScientist.commands.implementation.model import (
|
||||
extract_model_and_provider,
|
||||
)
|
||||
|
||||
name, prov = extract_model_and_provider(["claude-sonnet-4-6", "openrouter"])
|
||||
assert name == "claude-sonnet-4-6"
|
||||
assert prov == "openrouter"
|
||||
|
||||
def test_unknown_model_no_provider_raises(self):
|
||||
from EvoScientist.commands.implementation.model import (
|
||||
extract_model_and_provider,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="Unknown model"):
|
||||
extract_model_and_provider(["nonexistent-model-xyz"])
|
||||
|
||||
def test_unknown_model_with_provider_still_raises(self):
|
||||
from EvoScientist.commands.implementation.model import (
|
||||
extract_model_and_provider,
|
||||
)
|
||||
|
||||
# Unknown models are always rejected, even with an explicit provider
|
||||
with pytest.raises(ValueError, match="Unknown model"):
|
||||
extract_model_and_provider(["my-custom-model", "custom-openai"])
|
||||
|
||||
def test_provider_override_on_known_model(self):
|
||||
from EvoScientist.commands.implementation.model import (
|
||||
extract_model_and_provider,
|
||||
)
|
||||
|
||||
# Known model with explicit provider override uses the override
|
||||
name, prov = extract_model_and_provider(["claude-sonnet-4-6", "openrouter"])
|
||||
assert name == "claude-sonnet-4-6"
|
||||
assert prov == "openrouter"
|
||||
|
||||
|
||||
class TestModelCommandUnknownModel:
|
||||
"""Verify error message for unknown models."""
|
||||
|
||||
def test_unknown_model_shows_error(self):
|
||||
from EvoScientist.commands.implementation.model import ModelCommand
|
||||
|
||||
cmd = ModelCommand()
|
||||
ui = MagicMock()
|
||||
ui.supports_interactive = True
|
||||
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.ui = ui
|
||||
|
||||
with patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=cfg,
|
||||
):
|
||||
_run(cmd.execute(ctx, ["nonexistent-model-xyz"]))
|
||||
|
||||
ui.append_system.assert_called_once()
|
||||
call_args = ui.append_system.call_args
|
||||
assert "Unknown model" in call_args[0][0]
|
||||
assert call_args[1]["style"] == "red"
|
||||
|
||||
|
||||
class TestModelCommandPickerCancelled:
|
||||
"""Verify no-op when the interactive picker is cancelled."""
|
||||
|
||||
def test_picker_returns_none(self):
|
||||
from EvoScientist.commands.implementation.model import ModelCommand
|
||||
|
||||
cmd = ModelCommand()
|
||||
ui = MagicMock()
|
||||
ui.supports_interactive = True
|
||||
ui.wait_for_model_pick = AsyncMock(return_value=None)
|
||||
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.ui = ui
|
||||
|
||||
with patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=cfg,
|
||||
):
|
||||
_run(cmd.execute(ctx, []))
|
||||
|
||||
# No model switch should have happened
|
||||
ui.append_system.assert_not_called()
|
||||
|
||||
|
||||
class TestModelCommandSwitch:
|
||||
"""Verify a successful model switch updates config and rebuilds agent."""
|
||||
|
||||
def test_switch_known_model(self):
|
||||
from EvoScientist.commands.implementation.model import ModelCommand
|
||||
|
||||
cmd = ModelCommand()
|
||||
ui = MagicMock()
|
||||
ui.supports_interactive = True
|
||||
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
|
||||
new_agent = MagicMock()
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.ui = ui
|
||||
ctx.workspace_dir = "/tmp/test"
|
||||
ctx.checkpointer = MagicMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=cfg,
|
||||
),
|
||||
patch(
|
||||
"EvoScientist.EvoScientist.set_chat_model",
|
||||
),
|
||||
patch(
|
||||
"EvoScientist.cli.agent._load_agent",
|
||||
return_value=new_agent,
|
||||
),
|
||||
):
|
||||
_run(cmd.execute(ctx, ["claude-opus-4-6"]))
|
||||
|
||||
# Config should be updated
|
||||
assert cfg.model == "claude-opus-4-6"
|
||||
assert cfg.provider == "anthropic"
|
||||
|
||||
# Agent should be replaced on context
|
||||
assert ctx.agent == new_agent
|
||||
|
||||
# Success message shown
|
||||
ui.append_system.assert_called_once()
|
||||
msg = ui.append_system.call_args[0][0]
|
||||
assert "claude-opus-4-6" in msg
|
||||
assert "anthropic" in msg
|
||||
|
||||
def test_switch_with_save_flag(self):
|
||||
from EvoScientist.commands.implementation.model import ModelCommand
|
||||
|
||||
cmd = ModelCommand()
|
||||
ui = MagicMock()
|
||||
ui.supports_interactive = True
|
||||
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.ui = ui
|
||||
ctx.workspace_dir = "/tmp/test"
|
||||
ctx.checkpointer = MagicMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=cfg,
|
||||
),
|
||||
patch("EvoScientist.EvoScientist.set_chat_model"),
|
||||
patch(
|
||||
"EvoScientist.cli.agent._load_agent",
|
||||
return_value=MagicMock(),
|
||||
),
|
||||
patch("EvoScientist.config.settings.set_config_value") as mock_save,
|
||||
):
|
||||
_run(cmd.execute(ctx, ["claude-opus-4-6", "--save"]))
|
||||
|
||||
# Config file should be updated
|
||||
mock_save.assert_any_call("model", "claude-opus-4-6")
|
||||
mock_save.assert_any_call("provider", "anthropic")
|
||||
|
||||
# Success message should mention save
|
||||
msg = ui.append_system.call_args[0][0]
|
||||
assert "saved to config" in msg
|
||||
|
||||
def test_switch_without_save_flag_does_not_persist(self):
|
||||
from EvoScientist.commands.implementation.model import ModelCommand
|
||||
|
||||
cmd = ModelCommand()
|
||||
ui = MagicMock()
|
||||
ui.supports_interactive = True
|
||||
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.ui = ui
|
||||
ctx.workspace_dir = "/tmp/test"
|
||||
ctx.checkpointer = MagicMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=cfg,
|
||||
),
|
||||
patch("EvoScientist.EvoScientist.set_chat_model"),
|
||||
patch(
|
||||
"EvoScientist.cli.agent._load_agent",
|
||||
return_value=MagicMock(),
|
||||
),
|
||||
patch("EvoScientist.config.settings.set_config_value") as mock_save,
|
||||
):
|
||||
_run(cmd.execute(ctx, ["claude-opus-4-6"]))
|
||||
|
||||
# Config file should NOT be updated
|
||||
mock_save.assert_not_called()
|
||||
|
||||
# Message should not mention save
|
||||
msg = ui.append_system.call_args[0][0]
|
||||
assert "saved to config" not in msg
|
||||
|
||||
|
||||
class TestModelCommandFailure:
|
||||
"""Verify error handling when set_chat_model raises."""
|
||||
|
||||
def test_set_chat_model_error(self):
|
||||
from EvoScientist.commands.implementation.model import ModelCommand
|
||||
|
||||
cmd = ModelCommand()
|
||||
ui = MagicMock()
|
||||
ui.supports_interactive = True
|
||||
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.ui = ui
|
||||
ctx.workspace_dir = "/tmp/test"
|
||||
ctx.checkpointer = MagicMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=cfg,
|
||||
),
|
||||
patch(
|
||||
"EvoScientist.cli.agent._load_agent",
|
||||
return_value=MagicMock(),
|
||||
),
|
||||
patch(
|
||||
"EvoScientist.EvoScientist.set_chat_model",
|
||||
side_effect=RuntimeError("API key missing"),
|
||||
) as mock_set,
|
||||
):
|
||||
_run(cmd.execute(ctx, ["claude-opus-4-6"]))
|
||||
|
||||
mock_set.assert_called_once()
|
||||
ui.append_system.assert_called_once()
|
||||
call_args = ui.append_system.call_args
|
||||
assert "Failed to switch model" in call_args[0][0]
|
||||
assert call_args[1]["style"] == "red"
|
||||
|
||||
|
||||
class TestModelCommandLoadAgentFailure:
|
||||
"""Verify the transactional ordering: when ``_load_agent`` raises,
|
||||
nothing downstream (``set_chat_model``, ``cfg`` mutation,
|
||||
``set_config_value``) should happen.
|
||||
|
||||
This is the core guarantee of the refactor that established
|
||||
"build agent first, commit state only on success". Without this test
|
||||
the ordering could silently regress (e.g. if ``_apply_model`` were
|
||||
reordered to call ``set_chat_model`` first)."""
|
||||
|
||||
def test_load_agent_error_is_transactional(self):
|
||||
from EvoScientist.commands.implementation.model import ModelCommand
|
||||
|
||||
cmd = ModelCommand()
|
||||
ui = MagicMock()
|
||||
ui.supports_interactive = True
|
||||
cfg = SimpleNamespace(model="claude-sonnet-4-6", provider="anthropic")
|
||||
|
||||
ctx = MagicMock()
|
||||
ctx.ui = ui
|
||||
ctx.workspace_dir = "/tmp/test"
|
||||
ctx.checkpointer = MagicMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"EvoScientist.EvoScientist._ensure_config",
|
||||
return_value=cfg,
|
||||
),
|
||||
patch(
|
||||
"EvoScientist.cli.agent._load_agent",
|
||||
side_effect=RuntimeError("agent build failed"),
|
||||
) as mock_load,
|
||||
patch(
|
||||
"EvoScientist.EvoScientist.set_chat_model",
|
||||
) as mock_set,
|
||||
patch(
|
||||
"EvoScientist.config.settings.set_config_value",
|
||||
) as mock_save,
|
||||
):
|
||||
# Pass ``--save`` to strengthen the assertion: if the ordering
|
||||
# ever regresses, ``set_config_value`` would be called with
|
||||
# stale data.
|
||||
_run(cmd.execute(ctx, ["claude-opus-4-6", "--save"]))
|
||||
|
||||
# _load_agent was attempted (transactional first step).
|
||||
mock_load.assert_called_once()
|
||||
# Downstream side-effects must NOT have happened.
|
||||
mock_set.assert_not_called()
|
||||
mock_save.assert_not_called()
|
||||
# Config must be untouched.
|
||||
assert cfg.model == "claude-sonnet-4-6"
|
||||
assert cfg.provider == "anthropic"
|
||||
# User sees a red error message.
|
||||
ui.append_system.assert_called_once()
|
||||
call_args = ui.append_system.call_args
|
||||
assert "Failed to switch model" in call_args[0][0]
|
||||
assert call_args[1]["style"] == "red"
|
||||
Reference in New Issue
Block a user