ouyangbo/EvoScientist
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 3 Wiki Activity

feat: add workspace isolation and provider administration

Browse Source
This commit is contained in:
m4
2026-07-19 12:17:18 +08:00
parent 7a3fcc7c8e
commit 38668c4ce5
56 changed files with 11090 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+38
View File
@@ -30,3 +30,41 @@ OLLAMA_BASE_URL= # http://localhost:11434 (default)
# Web search (optional)
TAVILY_API_KEY= # app.tavily.com
# WebUI provider editor. Local processes running as the same OS user share
# ~/.config/evoscientist/provider-admin-token automatically. For different
# hosts, users, or containers, set the same random value on both processes.
# Integrated `EvoSci` WebUI mode also passes a shared token automatically.
EVOSCIENTIST_PROVIDER_ADMIN_TOKEN=
# WebUI conversation workspace policy. EVOSCIENTIST_WORKSPACE_DIR is the
# deployment root, not a per-conversation directory. In isolated modes each
# conversation is stored under <root>/.evoscientist/conversations/<scope-id>/.
#
# EVOSCIENTIST_WORKSPACE_ISOLATION accepts exactly:
# - legacy: all WebUI conversations share the deployment root. Compatibility
# rollback only; files are visible to every conversation using this deployment.
# - optional: default. New WebUI conversations receive isolated scope folders;
# missing Registry/token/scope fails the request instead of silently sharing.
# - required: isolated scopes plus strict runtime validation. It requires a
# completed cutover and a verified OCI executor; no legacy fallback exists.
#
# This is a deployment-startup security setting. Change it only during a
# maintenance window, restart backend and WebUI afterwards, and never use it to
# convert an existing conversation between shared and isolated directories.
EVOSCIENTIST_WORKSPACE_DIR=
EVOSCIENTIST_WORKSPACE_ISOLATION=optional
# Required mode supports only a single-host Registry topology in v1.
EVOSCIENTIST_SCOPE_REGISTRY_TOPOLOGY=single-host
# Required mode: use a pinned image digest, preserve single-host topology, and
# keep the Code Interpreter disabled unless its scoped implementation is enabled.
# Do not put EVOSCIENTIST_BACKEND_SERVICE_TOKEN here for a same-host `EvoSci
# deploy`: it is generated and passed privately at startup.
# EVOSCIENTIST_WORKSPACE_ISOLATION=required
# EVOSCIENTIST_STRICT_EXECUTOR=oci
# EVOSCIENTIST_STRICT_EXECUTOR_IMAGE=registry.example/evoscientist-runtime@sha256:replace-with-verified-digest
# EVOSCIENTIST_STRICT_CODE_INTERPRETER=disabled
# Conversation workspace isolation retention defaults (used by workspace_maintenance.py).
EVOSCIENTIST_DRAFT_WORKSPACE_TTL_HOURS=24
EVOSCIENTIST_WORKSPACE_TRASH_RETENTION_DAYS=7
.github/workflows/test.yml
+26 -1
View File
@@ -16,11 +16,16 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest]
os: [ubuntu-latest, macos-latest, windows-latest]
python-version: ["3.11", "3.12"]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v5
- name: Check out shared usage fixtures
uses: actions/checkout@v5
with:
repository: EvoScientist/EvoScientist-WebUI
path: EvoScientist-WebUI
- uses: astral-sh/setup-uv@v6
with:
python-version: ${{ matrix.python-version }}
@@ -29,3 +34,23 @@ jobs:
run: uv sync --dev
- name: Run pytest
run: uv run pytest -v --timeout=30
env:
EVOSCIENTIST_USAGE_FIXTURES: ${{ github.workspace }}/EvoScientist-WebUI/docs/schemas/fixtures
usage-spool-benchmark:
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v5
- uses: astral-sh/setup-uv@v6
with:
python-version: "3.11"
cache-dependency-glob: "**/pyproject.toml"
- name: Install dependencies
run: uv sync --dev
- name: Verify durable spool latency
run: uv run python scripts/benchmark_usage_spool.py
EvoScientist/EvoScientist.py
+66 -13
View File
@@ -341,9 +341,15 @@ def _inject_subagent_middleware(
ToolErrorHandlerMiddleware(),
ContextOverflowMapperMiddleware(),
]
if memory_controls.memory_enabled:
if (
memory_controls.memory_enabled
and cfg.workspace_isolation != "required"
):
middleware.append(memory_middleware)
if memory_controls.worker_needed(MemoryObservationTarget.SUBAGENT_WORKER):
if (
memory_controls.worker_needed(MemoryObservationTarget.SUBAGENT_WORKER)
and cfg.workspace_isolation != "required"
):
middleware.append(
create_memory_lifecycle_middleware(
memory_dir,
@@ -486,7 +492,9 @@ def _build_base_kwargs(
tool_registry = {"think_tool": think_tool}
if os.environ.get("TAVILY_API_KEY"):
tool_registry["tavily_search"] = tavily_search
base_tools = [think_tool, skill_manager]
base_tools = [think_tool]
if cfg.workspace_isolation != "required":
base_tools.append(skill_manager)
subs = load_subagents(
SUBAGENTS_CONFIG,
@@ -548,7 +556,9 @@ def load_mcp_and_build_kwargs(
tool_registry = {"think_tool": think_tool}
if os.environ.get("TAVILY_API_KEY"):
tool_registry["tavily_search"] = tavily_search
base_tools = [think_tool, skill_manager]
base_tools = [think_tool]
if cfg.workspace_isolation != "required":
base_tools.append(skill_manager)
# Fresh tool registry — start from base tools + MCP tools
registry = dict(tool_registry)
@@ -594,8 +604,8 @@ def load_mcp_and_build_kwargs(
# =============================================================================
def _get_default_backend():
"""Build the default composite backend from current paths."""
def _get_legacy_backend():
"""Build the deployment-root backend used by CLI and legacy mode only."""
from deepagents.backends import CompositeBackend
from .backends import (
@@ -637,6 +647,36 @@ def _get_default_backend():
)
def _get_default_backend():
"""Return a runtime-scoped backend factory when isolation is enabled."""
from .workspace_cutover import verify_required_cutover
from .workspace_scope import (
create_workspace_backend_factory,
verify_required_executor,
)
cfg = _ensure_config()
if cfg.workspace_isolation == "legacy":
return _get_legacy_backend()
if cfg.workspace_isolation == "required" and cfg.dangerous_mode:
raise RuntimeError("dangerous_mode is incompatible with required workspace isolation")
if cfg.workspace_isolation == "required":
verify_required_cutover(_paths_mod.WORKSPACE_ROOT)
verify_required_executor()
return create_workspace_backend_factory(
_get_legacy_backend,
dangerous=cfg.dangerous_mode,
# The CLI and its stripped async-subagent service keep their configured
# shared workspace. The WebUI deployment must receive a scope from the
# trusted WebUI/API boundary instead.
allow_unscoped_legacy=os.environ.get(
"EVOSCIENTIST_DEPLOY_MODE", ""
).lower()
!= "full",
)
def _get_default_middleware(
*,
for_async_subagent: bool = False,
@@ -740,17 +780,27 @@ def _get_default_middleware(
),
# Interpreter prompt must land before runtime/memory context, so this
# middleware sits ahead of runtime_context in the stack.
create_code_interpreter_middleware(
timeout=cfg.code_interpreter_timeout,
max_result_chars=cfg.code_interpreter_max_result_chars,
*(
[]
if cfg.workspace_isolation == "required"
and cfg.strict_code_interpreter == "disabled"
else [
create_code_interpreter_middleware(
timeout=cfg.code_interpreter_timeout,
max_result_chars=cfg.code_interpreter_max_result_chars,
)
]
),
]
if cfg.enable_scheduler and not for_async_subagent:
mw.append(create_scheduler_middleware())
mw.append(create_runtime_context_middleware())
if memory_controls.memory_enabled:
if memory_controls.memory_enabled and cfg.workspace_isolation != "required":
mw.append(memory_middleware)
if memory_controls.worker_needed(worker_target):
if (
memory_controls.worker_needed(worker_target)
and cfg.workspace_isolation != "required"
):
mw.append(
create_memory_lifecycle_middleware(
memory_dir,
@@ -770,7 +820,7 @@ def _get_default_middleware(
# Background-process tools (run_in_background / check_process / stop_process /
# list_processes) — main agent only. Async sub-agents run on langgraph-dev and
# must not spawn local OS processes.
if not for_async_subagent:
if not for_async_subagent and cfg.workspace_isolation != "required":
from .middleware.background import BackgroundExecutionMiddleware
mw.append(BackgroundExecutionMiddleware())
@@ -824,7 +874,10 @@ def _get_default_agent():
)
)
if os.environ.get("EVOSCIENTIST_DEPLOY_MODE", "").lower() == "stripped":
if (
os.environ.get("EVOSCIENTIST_DEPLOY_MODE", "").lower() == "stripped"
or cfg.workspace_isolation == "required"
):
kwargs = _build_base_kwargs(
be,
mw,
EvoScientist/config/__init__.py
+23
View File
@@ -8,6 +8,16 @@ The onboard module is loaded lazily because it pulls in heavy dependencies
(langchain, llm) that are not needed for normal config operations.
"""
from .provider_profiles import (
ProviderModel,
ProviderProfile,
ProviderProfileError,
ProviderProfiles,
get_builtin_provider_profile,
get_provider_profiles_path,
load_provider_profiles,
save_provider_profiles,
)
from .settings import (
EvoScientistConfig,
MemoryControls,
@@ -19,7 +29,9 @@ from .settings import (
get_config_dir,
get_config_path,
get_config_value,
get_default_workspace_dir,
get_effective_config,
is_config_applied_env,
list_config,
load_config,
reset_config,
@@ -34,18 +46,29 @@ __all__ = [
"MemoryObservationWriter",
"MemorySkillSynthesisCadence",
"MemorySkillSynthesisMode",
# provider profiles
"ProviderModel",
"ProviderProfile",
"ProviderProfileError",
"ProviderProfiles",
"apply_config_to_env",
"get_builtin_provider_profile",
# settings
"get_config_dir",
"get_config_path",
"get_config_value",
"get_default_workspace_dir",
"get_effective_config",
"get_provider_profiles_path",
"is_config_applied_env",
"list_config",
"load_config",
"load_provider_profiles",
"reset_config",
# onboard (lazy)
"run_onboard",
"save_config",
"save_provider_profiles",
"set_config_value",
]
EvoScientist/config/provider_admin.py
+70
View File
@@ -0,0 +1,70 @@
"""Shared local authentication token for WebUI model administration."""
from __future__ import annotations
import os
import secrets
import threading
from pathlib import Path
from .settings import get_config_dir
_TOKEN_FILE_NAME = "provider-admin-token"
_TOKEN_LOCK = threading.RLock()
_cached_file_token: tuple[Path, str] | None = None
def get_provider_admin_token_path() -> Path:
"""Return the user-scoped provider administration token path."""
return get_config_dir() / _TOKEN_FILE_NAME
def _read_token(path: Path) -> str:
try:
return path.read_text(encoding="utf-8").strip()
except OSError:
return ""
def get_provider_admin_token() -> str:
"""Return an explicit token or create a shared local token when possible.
Separate local backend and WebUI processes share the user-scoped token file.
Remote/container deployments can set ``EVOSCIENTIST_PROVIDER_ADMIN_TOKEN``
explicitly when the processes do not share a filesystem.
"""
explicit = os.environ.get("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "").strip()
if explicit:
return explicit
path = get_provider_admin_token_path()
global _cached_file_token
with _TOKEN_LOCK:
if _cached_file_token is not None and _cached_file_token[0] == path:
return _cached_file_token[1]
token = _read_token(path)
if not token:
token = secrets.token_urlsafe(32)
try:
path.parent.mkdir(parents=True, exist_ok=True)
try:
path.parent.chmod(0o700)
except OSError:
pass
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
with os.fdopen(descriptor, "w", encoding="utf-8") as file:
file.write(f"{token}\n")
except FileExistsError:
token = _read_token(path)
except OSError:
return ""
if not token:
return ""
try:
path.chmod(0o600)
except OSError:
pass
_cached_file_token = (path, token)
return token
EvoScientist/config/provider_profiles.py
+595
View File
@@ -0,0 +1,595 @@
"""Persistent, runtime-editable LLM provider profiles.
The registry stores managed built-in connections and custom provider profiles.
``config.yaml`` remains backward compatible with the onboarding flow and acts
as a legacy fallback plus the default provider/model pointer.
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import tempfile
import threading
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import urlsplit, urlunsplit
import yaml
from .settings import get_config_dir
PROVIDER_PROFILES_VERSION = 2
SUPPORTED_PROVIDER_ADAPTERS = (
"openai",
"anthropic",
"google-genai",
"grok",
"openrouter",
"nvidia",
"antigravity",
"openai-compatible",
"anthropic-compatible",
"ollama",
)
_ID_PATTERN = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$")
_BASE_URL_REQUIRED_ADAPTERS = {
"openai-compatible",
"anthropic-compatible",
"antigravity",
"ollama",
}
_DEFAULT_API_PATHS = {
"openai": "/v1",
"openai-compatible": "/v1",
"grok": "/v1",
"openrouter": "/api/v1",
"nvidia": "/v1",
"antigravity": "/v1",
"google-genai": "/v1beta",
}
_RESERVED_PROVIDER_IDS = {
"anthropic",
"openai",
"google-genai",
"minimax",
"nvidia",
"siliconflow",
"openrouter",
"zhipu",
"zhipu-code",
"volcengine",
"dashscope",
"dashscope-code",
"deepseek",
"moonshot",
"kimi-coding",
"custom-openai",
"custom-anthropic",
"ollama",
}
_STORE_LOCK = threading.RLock()
class ProviderProfileError(ValueError):
"""Raised when a provider profile document or selection is invalid."""
@dataclass(frozen=True)
class ProviderModel:
id: str
name: str
model_id: str
enabled: bool = True
@dataclass(frozen=True)
class ProviderProfile:
id: str
name: str
adapter: str
base_url: str
api_key: str
enabled: bool
models: tuple[ProviderModel, ...]
auth_mode: str = "api_key"
@dataclass(frozen=True)
class ProviderProfiles:
version: int = PROVIDER_PROFILES_VERSION
builtins: tuple[ProviderProfile, ...] = ()
providers: tuple[ProviderProfile, ...] = ()
def normalize_provider_base_url(adapter: str, base_url: str) -> str:
"""Add the adapter's standard API path when only an origin is supplied."""
normalized = base_url.strip().rstrip("/")
default_path = _DEFAULT_API_PATHS.get(adapter)
if not normalized or not default_path:
return normalized
parsed = urlsplit(normalized)
if parsed.path not in {"", "/"} or parsed.query or parsed.fragment:
return normalized
return urlunsplit((parsed.scheme, parsed.netloc, default_path, "", ""))
def get_provider_profiles_path() -> Path:
"""Return the user-scoped provider profile path."""
return get_config_dir() / "providers.yaml"
def _required_string(
raw: dict[str, Any],
key: str,
*,
context: str,
max_length: int,
) -> str:
value = raw.get(key)
if not isinstance(value, str) or not value.strip():
raise ProviderProfileError(f"{context}.{key} is required.")
value = value.strip()
if len(value) > max_length:
raise ProviderProfileError(
f"{context}.{key} must be at most {max_length} characters."
)
return value
def _validate_id(value: str, *, context: str) -> str:
if not _ID_PATTERN.fullmatch(value):
raise ProviderProfileError(
f"{context} must start with a lowercase letter or digit and contain only "
"lowercase letters, digits, dots, underscores, or hyphens."
)
return value
def _parse_model(raw: Any, *, provider_id: str, index: int) -> ProviderModel:
context = f"providers[{provider_id}].models[{index}]"
if not isinstance(raw, dict):
raise ProviderProfileError(f"{context} must be an object.")
model_id = _validate_id(
_required_string(raw, "id", context=context, max_length=64),
context=f"{context}.id",
)
return ProviderModel(
id=model_id,
name=_required_string(raw, "name", context=context, max_length=120),
model_id=_required_string(raw, "model_id", context=context, max_length=300),
enabled=bool(raw.get("enabled", True)),
)
def _parse_profile(raw: Any, *, index: int, builtin: bool = False) -> ProviderProfile:
collection = "builtins" if builtin else "providers"
context = f"{collection}[{index}]"
if not isinstance(raw, dict):
raise ProviderProfileError(f"{context} must be an object.")
provider_id = _validate_id(
_required_string(raw, "id", context=context, max_length=64),
context=f"{context}.id",
)
if builtin and provider_id not in _RESERVED_PROVIDER_IDS:
raise ProviderProfileError(
f"Built-in provider ID {provider_id!r} is not recognized."
)
if not builtin and provider_id in _RESERVED_PROVIDER_IDS:
raise ProviderProfileError(
f"Provider ID {provider_id!r} is reserved for a built-in provider. "
f"Use a unique ID such as {f'{provider_id}-custom'!r}."
)
adapter = _required_string(raw, "adapter", context=context, max_length=64)
if adapter not in SUPPORTED_PROVIDER_ADAPTERS:
raise ProviderProfileError(
f"{context}.adapter must be one of: "
f"{', '.join(SUPPORTED_PROVIDER_ADAPTERS)}."
)
base_url_raw = raw.get("base_url", "")
if not isinstance(base_url_raw, str):
raise ProviderProfileError(f"{context}.base_url must be a string.")
base_url = base_url_raw.strip().rstrip("/")
if len(base_url) > 2048:
raise ProviderProfileError(
f"{context}.base_url must be at most 2048 characters."
)
if adapter in _BASE_URL_REQUIRED_ADAPTERS and not base_url:
raise ProviderProfileError(
f"{context}.base_url is required for adapter {adapter}."
)
if base_url and not base_url.startswith(("http://", "https://")):
raise ProviderProfileError(f"{context}.base_url must use http:// or https://.")
base_url = normalize_provider_base_url(adapter, base_url)
api_key_raw = raw.get("api_key", "")
if not isinstance(api_key_raw, str):
raise ProviderProfileError(f"{context}.api_key must be a string.")
if len(api_key_raw) > 8192:
raise ProviderProfileError(
f"{context}.api_key must be at most 8192 characters."
)
auth_mode = raw.get("auth_mode", "api_key")
if not isinstance(auth_mode, str) or auth_mode not in {"api_key", "oauth"}:
raise ProviderProfileError(f"{context}.auth_mode must be 'api_key' or 'oauth'.")
models_raw = raw.get("models", [])
if not isinstance(models_raw, list):
raise ProviderProfileError(f"{context}.models must be a list.")
models = tuple(
_parse_model(model, provider_id=provider_id, index=model_index)
for model_index, model in enumerate(models_raw)
)
model_ids = [model.id for model in models]
if len(model_ids) != len(set(model_ids)):
raise ProviderProfileError(
f"Provider {provider_id!r} contains duplicate model IDs."
)
return ProviderProfile(
id=provider_id,
name=_required_string(raw, "name", context=context, max_length=120),
adapter=adapter,
base_url=base_url,
api_key=api_key_raw,
enabled=bool(raw.get("enabled", True)),
models=models,
auth_mode=auth_mode,
)
def _parse_document(raw: Any) -> ProviderProfiles:
if raw is None:
return ProviderProfiles()
if not isinstance(raw, dict):
raise ProviderProfileError("Provider profile document must be an object.")
version = raw.get("version", 1)
if version not in {1, PROVIDER_PROFILES_VERSION}:
raise ProviderProfileError(f"Unsupported provider profile version {version!r}.")
builtins_raw = raw.get("builtins", [])
if not isinstance(builtins_raw, list):
raise ProviderProfileError("builtins must be a list.")
providers_raw = raw.get("providers", [])
if not isinstance(providers_raw, list):
raise ProviderProfileError("providers must be a list.")
providers = tuple(
_parse_profile(profile, index=index, builtin=False)
for index, profile in enumerate(providers_raw)
)
builtins = tuple(
_parse_profile(profile, index=index, builtin=True)
for index, profile in enumerate(builtins_raw)
)
provider_ids = [provider.id for provider in providers]
if len(provider_ids) != len(set(provider_ids)):
raise ProviderProfileError("Provider IDs must be unique.")
builtin_ids = [provider.id for provider in builtins]
if len(builtin_ids) != len(set(builtin_ids)):
raise ProviderProfileError("Built-in provider IDs must be unique.")
return ProviderProfiles(
version=PROVIDER_PROFILES_VERSION,
builtins=builtins,
providers=providers,
)
def _profile_to_dict(profile: ProviderProfile) -> dict[str, Any]:
return {
"id": profile.id,
"name": profile.name,
"adapter": profile.adapter,
"base_url": profile.base_url,
"api_key": profile.api_key,
"auth_mode": profile.auth_mode,
"enabled": profile.enabled,
"models": [
{
"id": model.id,
"name": model.name,
"model_id": model.model_id,
"enabled": model.enabled,
}
for model in profile.models
],
}
def _document_to_dict(document: ProviderProfiles) -> dict[str, Any]:
return {
"version": PROVIDER_PROFILES_VERSION,
"builtins": [_profile_to_dict(profile) for profile in document.builtins],
"providers": [_profile_to_dict(profile) for profile in document.providers],
}
def load_provider_profiles() -> ProviderProfiles:
"""Load provider profiles, returning an empty document when absent."""
path = get_provider_profiles_path()
with _STORE_LOCK:
if not path.exists():
return ProviderProfiles()
try:
with path.open(encoding="utf-8") as handle:
return _parse_document(yaml.safe_load(handle))
except ProviderProfileError:
raise
except Exception as exc:
raise ProviderProfileError(
f"Could not load provider profiles: {exc}"
) from exc
def save_provider_profiles(document: ProviderProfiles) -> None:
"""Atomically persist provider profiles with private permissions."""
path = get_provider_profiles_path()
payload = _document_to_dict(document)
with _STORE_LOCK:
path.parent.mkdir(parents=True, exist_ok=True)
try:
path.parent.chmod(0o700)
except OSError:
pass
fd, temporary = tempfile.mkstemp(
prefix="providers-", suffix=".yaml.tmp", dir=path.parent
)
try:
with os.fdopen(fd, "w", encoding="utf-8") as handle:
yaml.safe_dump(
payload,
handle,
default_flow_style=False,
sort_keys=False,
allow_unicode=True,
)
handle.flush()
os.fsync(handle.fileno())
try:
Path(temporary).chmod(0o600)
except OSError:
pass
os.replace(temporary, path)
try:
path.chmod(0o600)
except OSError:
pass
finally:
try:
Path(temporary).unlink(missing_ok=True)
except OSError:
pass
def replace_provider_profiles(raw: Any) -> ProviderProfiles:
"""Replace supplied profile collections while preserving the other side.
Existing API keys are retained when an existing profile sends no ``api_key``
or sends an empty value. ``clear_api_key: true`` explicitly removes it.
"""
if not isinstance(raw, dict):
raise ProviderProfileError("Request body must be an object.")
if "providers" not in raw and "builtins" not in raw:
raise ProviderProfileError("providers or builtins must be supplied.")
current = load_provider_profiles()
def merge_collection(
key: str,
supplied: Any,
existing_profiles: tuple[ProviderProfile, ...],
) -> list[dict[str, Any]]:
if supplied is None:
return [_profile_to_dict(profile) for profile in existing_profiles]
if not isinstance(supplied, list):
raise ProviderProfileError(f"{key} must be a list.")
existing = {profile.id: profile for profile in existing_profiles}
merged: list[dict[str, Any]] = []
for index, item in enumerate(supplied):
if not isinstance(item, dict):
raise ProviderProfileError(f"{key}[{index}] must be an object.")
candidate = dict(item)
provider_id = candidate.get("id")
prior = existing.get(provider_id) if isinstance(provider_id, str) else None
if candidate.pop("clear_api_key", False):
candidate["api_key"] = ""
elif not candidate.get("api_key") and prior is not None:
candidate["api_key"] = prior.api_key
else:
candidate.setdefault("api_key", "")
merged.append(candidate)
return merged
document = _parse_document(
{
"version": PROVIDER_PROFILES_VERSION,
"builtins": merge_collection(
"builtins", raw.get("builtins"), current.builtins
),
"providers": merge_collection(
"providers", raw.get("providers"), current.providers
),
}
)
save_provider_profiles(document)
return document
def resolve_provider_profile_draft(raw: Any) -> ProviderProfile:
"""Validate a WebUI draft and restore its saved API key when omitted.
Provider discovery and model tests operate on unsaved form values. The
browser never receives an existing secret, so an empty ``api_key`` keeps
the saved value for the matching provider ID. ``clear_api_key`` explicitly
opts out of that fallback.
"""
if not isinstance(raw, dict):
raise ProviderProfileError("provider must be an object.")
candidate = dict(raw)
provider_id = candidate.get("id")
is_builtin = isinstance(provider_id, str) and provider_id in _RESERVED_PROVIDER_IDS
prior = (
get_builtin_provider_profile(provider_id)
if is_builtin
else get_provider_profile(provider_id)
if isinstance(provider_id, str)
else None
)
if candidate.pop("clear_api_key", False):
candidate["api_key"] = ""
elif not candidate.get("api_key") and prior is not None:
candidate["api_key"] = prior.api_key
else:
candidate.setdefault("api_key", "")
candidate.setdefault("models", [])
return _parse_profile(candidate, index=0, builtin=is_builtin)
def _api_key_hint(api_key: str) -> str | None:
if not api_key:
return None
if len(api_key) <= 8:
return "configured"
return f"{api_key[:3]}...{api_key[-4:]}"
def _provider_profiles_public_payload(document: ProviderProfiles) -> dict[str, Any]:
def profile_payload(profile: ProviderProfile) -> dict[str, Any]:
return {
"id": profile.id,
"name": profile.name,
"adapter": profile.adapter,
"base_url": profile.base_url,
"auth_mode": profile.auth_mode,
"enabled": profile.enabled,
"api_key_configured": bool(profile.api_key),
"api_key_hint": _api_key_hint(profile.api_key),
"models": [
{
"id": model.id,
"name": model.name,
"model_id": model.model_id,
"enabled": model.enabled,
}
for model in profile.models
],
}
return {
"version": document.version,
"builtins": [profile_payload(profile) for profile in document.builtins],
"providers": [profile_payload(profile) for profile in document.providers],
}
def provider_profiles_public(
document: ProviderProfiles | None = None,
) -> dict[str, Any]:
"""Return a browser-safe representation with all secrets removed."""
document = document or load_provider_profiles()
public = _provider_profiles_public_payload(document)
public["reserved_provider_ids"] = sorted(_RESERVED_PROVIDER_IDS)
public["revision"] = provider_profiles_public_revision(document)
return public
def provider_profiles_public_revision(
document: ProviderProfiles | None = None,
) -> str:
"""Return a browser-safe digest that excludes API key material."""
document = document or load_provider_profiles()
encoded = json.dumps(
_provider_profiles_public_payload(document),
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()[:16]
def provider_profiles_revision(
document: ProviderProfiles | None = None,
) -> str:
"""Return a private revision that changes for connection secrets too."""
document = document or load_provider_profiles()
encoded = json.dumps(
_document_to_dict(document), sort_keys=True, separators=(",", ":")
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()[:16]
def get_provider_profile(provider_id: str) -> ProviderProfile | None:
for profile in load_provider_profiles().providers:
if profile.id == provider_id:
return profile
return None
def get_builtin_provider_profile(provider_id: str) -> ProviderProfile | None:
for profile in load_provider_profiles().builtins:
if profile.id == provider_id:
return profile
return None
def get_provider_profile_revision(provider_id: str) -> str | None:
profile = get_builtin_provider_profile(provider_id) or get_provider_profile(
provider_id
)
if profile is None:
return None
encoded = json.dumps(
_profile_to_dict(profile), sort_keys=True, separators=(",", ":")
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()[:16]
def resolve_provider_model(
provider_id: str, model_name: str
) -> tuple[ProviderProfile, ProviderModel] | None:
"""Resolve a configured ``(provider_id, model alias)`` pair."""
profile = get_provider_profile(provider_id)
if profile is None:
return None
if not profile.enabled:
raise ProviderProfileError(f"Provider {provider_id!r} is disabled.")
for model in profile.models:
if model.id == model_name:
if not model.enabled:
raise ProviderProfileError(
f"Model {model_name!r} in provider {provider_id!r} is disabled."
)
return profile, model
raise ProviderProfileError(
f"Model {model_name!r} is not configured for provider {provider_id!r}."
)
def list_configured_model_entries() -> list[tuple[str, str, str]]:
"""Return enabled dynamic models as ``(alias, upstream id, provider id)``."""
entries: list[tuple[str, str, str]] = []
for profile in load_provider_profiles().providers:
if not profile.enabled:
continue
for model in profile.models:
if model.enabled:
entries.append((model.id, model.model_id, profile.id))
return entries
def list_configured_builtin_model_entries() -> list[tuple[str, str, str]]:
"""Return enabled managed built-in models."""
entries: list[tuple[str, str, str]] = []
for profile in load_provider_profiles().builtins:
if not profile.enabled:
continue
for model in profile.models:
if model.enabled:
entries.append((model.id, model.model_id, profile.id))
return entries
EvoScientist/config/settings.py
+103 -45
View File
@@ -23,6 +23,19 @@ from dotenv import find_dotenv, load_dotenv
# (stream/display.py, channels/consumer.py) — keep aligned with the agent's
# `interrupt_on` set in EvoScientist.py.
HITL_SHELL_TOOLS = ("execute", "run_in_background")
_CONFIG_APPLIED_ENV_VALUES: dict[str, str] = {}
def _apply_config_env(name: str, value: str) -> None:
if value and not os.environ.get(name):
os.environ[name] = value
_CONFIG_APPLIED_ENV_VALUES[name] = value
def is_config_applied_env(name: str) -> bool:
"""Return whether the current env value was injected from config.yaml."""
applied = _CONFIG_APPLIED_ENV_VALUES.get(name)
return applied is not None and os.environ.get(name) == applied
class MemoryObservationTarget(StrEnum):
@@ -119,6 +132,11 @@ def get_config_path() -> Path:
return get_config_dir() / "config.yaml"
def get_default_workspace_dir() -> Path:
"""Return the stable workspace used when no directory is configured."""
return Path.home() / ".evoscientist" / "workspace"
# =============================================================================
# Configuration dataclass
# =============================================================================
@@ -139,7 +157,8 @@ class EvoScientistConfig:
auxiliary_provider: Provider for auxiliary_model (empty = use main provider).
auxiliary_model: Model for memory workers + tool selector + scheduler (empty = use main model).
default_mode: Default workspace mode ('daemon' or 'run').
default_workdir: Default workspace directory (empty = use current working directory).
default_workdir: Default workspace directory (empty = use
~/.evoscientist/workspace).
show_thinking: Whether to show thinking panels in CLI.
"""
@@ -171,6 +190,10 @@ class EvoScientistConfig:
# LLM Settings
provider: str = "anthropic"
model: str = "claude-sonnet-4-6"
# Legacy built-in model allowlist. New WebUI saves store built-in and
# custom model catalogs in providers.yaml; this remains as a read-only
# compatibility fallback until the first Registry save.
model_catalog: list[dict[str, Any]] | None = None
model_fallbacks: str = "" # "model:provider,model:provider" fallback chain
# Optional auxiliary model for background/helper LLM calls (memory workers +
# tool selector). Empty = fall back to the main model/provider.
@@ -398,6 +421,17 @@ class EvoScientistConfig:
# blocklist (sudo/chmod/dd/...) still applies. Implies auto_approve.
dangerous_mode: bool = False
# Conversation workspace isolation. New WebUI conversations use a registry-
# validated scope by default. ``optional`` preserves the legacy execution
# path only when a trusted caller cannot provide a scope.
workspace_isolation: str = "optional" # legacy | optional | required
scope_registry_topology: str = "single-host"
strict_executor: str = "oci" # oci
strict_executor_image: str = "" # immutable OCI digest required in required mode
strict_code_interpreter: str = "disabled" # disabled | scoped
draft_workspace_ttl_hours: int = 24
workspace_trash_retention_days: int = 7
# Agent features
enable_ask_user: bool = True # Enable ask_user tool for agent-initiated questions
@@ -458,6 +492,47 @@ class EvoScientistConfig:
if self.dangerous_mode:
self.auto_approve = True
if self.workspace_isolation not in {"legacy", "optional", "required"}:
raise ValueError(
"workspace_isolation must be one of legacy, optional, required"
)
if self.scope_registry_topology != "single-host":
raise ValueError("v1 workspace isolation only supports single-host topology")
if self.strict_executor != "oci":
raise ValueError("strict_executor must be oci")
if self.strict_code_interpreter not in {"disabled", "scoped"}:
raise ValueError("strict_code_interpreter must be disabled or scoped")
if (
not isinstance(self.draft_workspace_ttl_hours, int)
or isinstance(self.draft_workspace_ttl_hours, bool)
or self.draft_workspace_ttl_hours <= 0
):
raise ValueError("draft_workspace_ttl_hours must be a positive integer")
if (
not isinstance(self.workspace_trash_retention_days, int)
or isinstance(self.workspace_trash_retention_days, bool)
or self.workspace_trash_retention_days <= 0
):
raise ValueError("workspace_trash_retention_days must be a positive integer")
if self.workspace_isolation == "required" and self.dangerous_mode:
raise ValueError(
"dangerous_mode is incompatible with required workspace isolation"
)
if (
self.workspace_isolation == "required"
and "@sha256:" not in self.strict_executor_image
):
raise ValueError(
"required workspace isolation needs strict_executor_image pinned by digest"
)
if (
self.workspace_isolation == "required"
and self.strict_code_interpreter != "disabled"
):
raise ValueError(
"strict_code_interpreter=scoped is not available until every PTC tool is scope-verified"
)
_normalize_str_enum_fields(self)
synthesis_time = _normalize_hhmm(self.memory_skill_synthesis_time)
@@ -762,6 +837,13 @@ _ENV_MAPPINGS = {
"EVOSCIENTIST_OPENROUTER_ANTHROPIC_PROMPT_CACHE"
),
"dangerous_mode": "EVOSCIENTIST_DANGEROUS_MODE",
"workspace_isolation": "EVOSCIENTIST_WORKSPACE_ISOLATION",
"scope_registry_topology": "EVOSCIENTIST_SCOPE_REGISTRY_TOPOLOGY",
"strict_executor": "EVOSCIENTIST_STRICT_EXECUTOR",
"strict_executor_image": "EVOSCIENTIST_STRICT_EXECUTOR_IMAGE",
"strict_code_interpreter": "EVOSCIENTIST_STRICT_CODE_INTERPRETER",
"draft_workspace_ttl_hours": "EVOSCIENTIST_DRAFT_WORKSPACE_TTL_HOURS",
"workspace_trash_retention_days": "EVOSCIENTIST_WORKSPACE_TRASH_RETENTION_DAYS",
"channel_debug_tracing": "EVOSCIENTIST_CHANNEL_DEBUG_TRACING",
"ccproxy_port": "EVOSCIENTIST_CCPROXY_PORT",
"use_responses_api": "EVOSCIENTIST_USE_RESPONSES_API",
@@ -844,50 +926,26 @@ def apply_config_to_env(config: EvoScientistConfig) -> None:
Args:
config: Configuration to apply.
"""
if config.anthropic_api_key and not os.environ.get("ANTHROPIC_API_KEY"):
os.environ["ANTHROPIC_API_KEY"] = config.anthropic_api_key
if config.anthropic_base_url and not os.environ.get("ANTHROPIC_BASE_URL"):
os.environ["ANTHROPIC_BASE_URL"] = config.anthropic_base_url
if config.openai_api_key and not os.environ.get("OPENAI_API_KEY"):
os.environ["OPENAI_API_KEY"] = config.openai_api_key
if config.nvidia_api_key and not os.environ.get("NVIDIA_API_KEY"):
os.environ["NVIDIA_API_KEY"] = config.nvidia_api_key
if config.google_api_key and not os.environ.get("GOOGLE_API_KEY"):
os.environ["GOOGLE_API_KEY"] = config.google_api_key
if config.minimax_api_key and not os.environ.get("MINIMAX_API_KEY"):
os.environ["MINIMAX_API_KEY"] = config.minimax_api_key
if config.minimax_base_url and not os.environ.get("MINIMAX_BASE_URL"):
os.environ["MINIMAX_BASE_URL"] = config.minimax_base_url
if config.siliconflow_api_key and not os.environ.get("SILICONFLOW_API_KEY"):
os.environ["SILICONFLOW_API_KEY"] = config.siliconflow_api_key
if config.openrouter_api_key and not os.environ.get("OPENROUTER_API_KEY"):
os.environ["OPENROUTER_API_KEY"] = config.openrouter_api_key
if config.deepseek_api_key and not os.environ.get("DEEPSEEK_API_KEY"):
os.environ["DEEPSEEK_API_KEY"] = config.deepseek_api_key
if config.zhipu_api_key and not os.environ.get("ZHIPU_API_KEY"):
os.environ["ZHIPU_API_KEY"] = config.zhipu_api_key
if config.volcengine_api_key and not os.environ.get("VOLCENGINE_API_KEY"):
os.environ["VOLCENGINE_API_KEY"] = config.volcengine_api_key
if config.dashscope_api_key and not os.environ.get("DASHSCOPE_API_KEY"):
os.environ["DASHSCOPE_API_KEY"] = config.dashscope_api_key
if config.moonshot_api_key and not os.environ.get("MOONSHOT_API_KEY"):
os.environ["MOONSHOT_API_KEY"] = config.moonshot_api_key
if config.kimi_api_key and not os.environ.get("KIMI_API_KEY"):
os.environ["KIMI_API_KEY"] = config.kimi_api_key
if config.custom_openai_api_key and not os.environ.get("CUSTOM_OPENAI_API_KEY"):
os.environ["CUSTOM_OPENAI_API_KEY"] = config.custom_openai_api_key
if config.custom_openai_base_url and not os.environ.get("CUSTOM_OPENAI_BASE_URL"):
os.environ["CUSTOM_OPENAI_BASE_URL"] = config.custom_openai_base_url
if config.custom_anthropic_api_key and not os.environ.get(
"CUSTOM_ANTHROPIC_API_KEY"
):
os.environ["CUSTOM_ANTHROPIC_API_KEY"] = config.custom_anthropic_api_key
if config.custom_anthropic_base_url and not os.environ.get(
"CUSTOM_ANTHROPIC_BASE_URL"
):
os.environ["CUSTOM_ANTHROPIC_BASE_URL"] = config.custom_anthropic_base_url
if config.ollama_base_url and not os.environ.get("OLLAMA_BASE_URL"):
os.environ["OLLAMA_BASE_URL"] = config.ollama_base_url
_apply_config_env("ANTHROPIC_API_KEY", config.anthropic_api_key)
_apply_config_env("ANTHROPIC_BASE_URL", config.anthropic_base_url)
_apply_config_env("OPENAI_API_KEY", config.openai_api_key)
_apply_config_env("NVIDIA_API_KEY", config.nvidia_api_key)
_apply_config_env("GOOGLE_API_KEY", config.google_api_key)
_apply_config_env("MINIMAX_API_KEY", config.minimax_api_key)
_apply_config_env("MINIMAX_BASE_URL", config.minimax_base_url)
_apply_config_env("SILICONFLOW_API_KEY", config.siliconflow_api_key)
_apply_config_env("OPENROUTER_API_KEY", config.openrouter_api_key)
_apply_config_env("DEEPSEEK_API_KEY", config.deepseek_api_key)
_apply_config_env("ZHIPU_API_KEY", config.zhipu_api_key)
_apply_config_env("VOLCENGINE_API_KEY", config.volcengine_api_key)
_apply_config_env("DASHSCOPE_API_KEY", config.dashscope_api_key)
_apply_config_env("MOONSHOT_API_KEY", config.moonshot_api_key)
_apply_config_env("KIMI_API_KEY", config.kimi_api_key)
_apply_config_env("CUSTOM_OPENAI_API_KEY", config.custom_openai_api_key)
_apply_config_env("CUSTOM_OPENAI_BASE_URL", config.custom_openai_base_url)
_apply_config_env("CUSTOM_ANTHROPIC_API_KEY", config.custom_anthropic_api_key)
_apply_config_env("CUSTOM_ANTHROPIC_BASE_URL", config.custom_anthropic_base_url)
_apply_config_env("OLLAMA_BASE_URL", config.ollama_base_url)
if config.tavily_api_key and not os.environ.get("TAVILY_API_KEY"):
os.environ["TAVILY_API_KEY"] = config.tavily_api_key
if config.reasoning_effort and not os.environ.get("EVOSCIENTIST_REASONING_EFFORT"):
EvoScientist/cron/schedule.py
+97 -11
View File
@@ -12,7 +12,7 @@ multiple clients at one hand-started server they will share the same cron store.
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from langgraph_sdk.schema import Cron, Run
@@ -47,22 +47,84 @@ def is_available() -> bool:
return bool(is_langgraph_dev_running(base_url=_scheduler_url()))
def _scope_payload(
scope: Any | None, *, owner_id: str | None = None
) -> tuple[dict[str, Any], dict[str, Any]]:
if scope is None:
return {}, {}
metadata = {
"workspace_scope_id": scope.scope_id,
"workspace_scope_owner_id": owner_id or scope.owner_id,
"workspace_deployment_id": scope.deployment_id,
}
return metadata, {
"configurable": {
**metadata,
"workspace_scope_revision": scope.revision,
}
}
def create_schedule(
*, name: str, schedule: str, prompt: str, timezone: str | None = None
*,
name: str,
schedule: str,
prompt: str,
timezone: str | None = None,
scope: Any | None = None,
) -> Cron:
"""Create a recurring scheduled task on the scheduler graph."""
# Crons are stored in the langgraph-dev process's .langgraph_api store, not
# tagged by workspace. Isolation is process-level (see module docstring).
return _client().crons.create(
assistant_id=SCHEDULER_GRAPH_ID,
schedule=schedule,
input=messages_input(prompt),
metadata={"run_kind": SCHEDULED_RUN_KIND, "name": name, "prompt": prompt},
timezone=timezone or _default_timezone(),
owner = None
if scope is not None:
from ..scope_registry import get_scope_registry
owner = get_scope_registry().register_owner(
scope.deployment_id,
scope.scope_id,
owner_type="schedule",
parent_owner_id=scope.owner_id,
state="reserved",
)
scope_metadata, config = _scope_payload(
scope, owner_id=owner.owner_id if owner else None
)
try:
cron = _client().crons.create(
assistant_id=SCHEDULER_GRAPH_ID,
schedule=schedule,
input=messages_input(prompt),
metadata={
"run_kind": SCHEDULED_RUN_KIND,
"name": name,
"prompt": prompt,
**scope_metadata,
},
**({"config": config} if config else {}),
timezone=timezone or _default_timezone(),
)
except Exception:
if owner is not None:
get_scope_registry().bind_owner(
scope.deployment_id,
scope.scope_id,
owner.owner_id,
f"failed:{owner.owner_id}",
state="terminal",
)
raise
if owner is not None:
get_scope_registry().bind_owner(
scope.deployment_id,
scope.scope_id,
owner.owner_id,
str(cron["cron_id"]),
)
return cron
def list_schedules() -> list[Cron]:
def list_schedules(scope: Any | None = None) -> list[Cron]:
"""Return only EvoScientist scheduled tasks.
Filtered server-side by ``run_kind`` metadata (the cron backend matches by
@@ -71,10 +133,17 @@ def list_schedules() -> list[Cron]:
rather than ``assistant_id`` because the stored ``assistant_id`` is a resolved
UUID, not the ``scheduler`` graph name we create with.
"""
return _client().crons.search(
rows = _client().crons.search(
metadata={"run_kind": SCHEDULED_RUN_KIND},
limit=1000,
)
if scope is None:
return rows
return [
row
for row in rows
if (row.get("metadata") or {}).get("workspace_scope_id") == scope.scope_id
]
def delete_schedule(cron_id: str) -> None:
@@ -87,13 +156,28 @@ def set_enabled(cron_id: str, enabled: bool) -> Cron:
return _client().crons.update(cron_id, enabled=enabled)
def run_now(prompt: str) -> Run:
def run_now(prompt: str, scope: Any | None = None) -> Run:
"""Fire a one-off scheduler run immediately (for ``/schedule run``).
Output goes wherever the task's prompt specifies; there is no push notification.
"""
client = _client()
thread = client.threads.create(graph_id=SCHEDULER_GRAPH_ID)
owner = None
if scope is not None:
from ..scope_registry import get_scope_registry
owner = get_scope_registry().register_owner(
scope.deployment_id,
scope.scope_id,
owner_type="scheduler_run",
resource_id=str(thread["thread_id"]),
parent_owner_id=scope.owner_id,
state="active",
)
scope_metadata, config = _scope_payload(
scope, owner_id=owner.owner_id if owner else None
)
return client.runs.create(
thread_id=str(thread["thread_id"]),
assistant_id=SCHEDULER_GRAPH_ID,
@@ -102,5 +186,7 @@ def run_now(prompt: str) -> Run:
"run_kind": SCHEDULED_RUN_KIND,
"name": "manual-run",
"prompt": prompt,
**scope_metadata,
},
**({"config": config} if config else {}),
)
EvoScientist/deploy/server.py
+103 -7
View File
@@ -5,8 +5,8 @@ for consumption by external LangChain-compatible UIs (deep-agents-ui,
agent-chat-ui, LangSmith Studio) and SDK clients.
Differs from ``EvoSci`` / ``EvoSci serve``: no in-process CLI agent,
no session DB, no channel runtime, no TUI. The terminal only shows
startup progress, the Ready banner, and then blocks until Ctrl+C.
no session DB, no channel runtime, no TUI. The terminal shows startup
progress, the Ready banner, and the live Gateway log until Ctrl+C.
Mode dispatch happens via the ``EVOSCIENTIST_DEPLOY_MODE`` env var
injected by ``start_langgraph_dev``: ``full`` for the deploy subprocess
@@ -20,11 +20,13 @@ loads or skips MCP based on the value.
from __future__ import annotations
import atexit
import codecs
import os
import signal
import sys
import threading
from pathlib import Path
from typing import Any
from typing import Any, TextIO
import typer # type: ignore[import-untyped]
from rich.panel import Panel
@@ -32,6 +34,65 @@ from rich.text import Text
from ..cli._app import app
from ..stream.console import console
from ..usage import prepare_usage_environment
def _follow_gateway_log(
log_path: Path,
start_offset: int,
stop_event: threading.Event,
*,
output: TextIO | None = None,
poll_interval: float = 0.1,
) -> None:
"""Mirror appended Gateway log bytes to a text stream until stopped."""
sink = output if output is not None else sys.stdout
decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
offset = max(0, start_offset)
while True:
try:
with log_path.open("rb") as log_file:
log_file.seek(0, os.SEEK_END)
if log_file.tell() < offset:
# Be defensive if another process replaced/truncated the log.
offset = 0
decoder.reset()
log_file.seek(offset)
chunk = log_file.read()
except OSError:
chunk = b""
if chunk:
offset += len(chunk)
text = decoder.decode(chunk)
if text:
sink.write(text)
sink.flush()
if stop_event.is_set():
break
stop_event.wait(poll_interval)
remainder = decoder.decode(b"", final=True)
if remainder:
sink.write(remainder)
sink.flush()
def _start_gateway_log_follower(
log_path: Path,
start_offset: int,
) -> tuple[threading.Event, threading.Thread]:
stop_event = threading.Event()
thread = threading.Thread(
target=_follow_gateway_log,
args=(log_path, start_offset, stop_event),
name="evoscientist-gateway-log",
daemon=True,
)
thread.start()
return stop_event, thread
@app.command()
@@ -39,7 +100,10 @@ def deploy(
workdir: str | None = typer.Option(
None,
"--workdir",
help="Workspace directory (default: config.default_workdir or cwd)",
help=(
"Workspace directory (default: config.default_workdir or "
"~/.evoscientist/workspace)"
),
),
port: int | None = typer.Option(
None,
@@ -64,11 +128,16 @@ def deploy(
Connect any LangChain-compatible UI or SDK client to the printed
endpoint. Press Ctrl+C to stop.
"""
from ..config import apply_config_to_env, get_effective_config
from ..config import (
apply_config_to_env,
get_default_workspace_dir,
get_effective_config,
)
from ..langgraph_dev.manager import (
_DEFAULT_PORT,
RUNTIME,
_is_port_occupied,
current_log_start_offset,
is_langgraph_dev_running,
read_tunnel_url,
start_langgraph_dev,
@@ -88,17 +157,21 @@ def deploy(
_configure_logging()
apply_config_to_env(config)
# 2. Resolve workspace (CLI > config.default_workdir > cwd)
# 2. Resolve workspace (CLI > config.default_workdir > stable app default)
if workdir:
ws = os.path.abspath(os.path.expanduser(workdir))
elif config.default_workdir:
ws = os.path.abspath(os.path.expanduser(config.default_workdir))
else:
ws = os.getcwd()
ws = str(get_default_workspace_dir())
# Subprocess inherits this path via EVOSCIENTIST_WORKSPACE_DIR (set inside
# start_langgraph_dev). Ensure the dir exists; do NOT mutate the parent
# process's paths module state — the deploy parent has no in-process agent.
os.makedirs(ws, exist_ok=True)
if getattr(config, "workspace_isolation", "optional") != "legacy":
from ..scope_registry import get_scope_service_token
os.environ["EVOSCIENTIST_BACKEND_SERVICE_TOKEN"] = get_scope_service_token(ws)
# 3. Resolve port (explicit None check — don't treat --port 0 as "unset"),
# then validate range so misconfigurations fail fast with a clear message
@@ -138,6 +211,18 @@ def deploy(
)
raise typer.Exit(1)
# Standalone WebUI development runs the backend and Next.js in separate
# terminals. Prepare the same durable identity, sink token, and spool used
# by the integrated launcher so the independently started WebUI can accept
# usage events through the shared data directory.
webui_port = int(getattr(config, "webui_port", 4716))
try:
usage_env = prepare_usage_environment(ws, webui_port=webui_port)
except (OSError, ValueError) as exc:
console.print(f"[yellow]Token statistics disabled:[/yellow] {exc}")
else:
os.environ.update(usage_env)
# 5. Startup banner
_auth_label = _describe_auth(config)
console.print(
@@ -249,6 +334,15 @@ def deploy(
)
)
# Replay this session's startup output, then mirror new Gateway log lines
# while deploy owns the foreground terminal. The child continues writing
# directly to the file, so terminal backpressure cannot block the server.
console.print("[dim]Gateway logs (also saved to the path above):[/dim]")
log_stop_event, log_thread = _start_gateway_log_follower(
RUNTIME.log_file,
current_log_start_offset(),
)
# 10. Block on signal — mirror serve's dual-gate (threading.Event +
# explicit SIGINT/SIGTERM handlers) so SIGTERM (no default raise) also
# triggers clean shutdown.
@@ -268,6 +362,8 @@ def deploy(
except KeyboardInterrupt:
shutdown_event.set()
finally:
log_stop_event.set()
log_thread.join(timeout=2.0)
signal.signal(signal.SIGINT, _orig_sigint)
signal.signal(signal.SIGTERM, _orig_sigterm)
# stop_langgraph_dev + stop_ccproxy run via atexit during interpreter
EvoScientist/deploy/webui.py
+38 -4
View File
@@ -57,12 +57,13 @@ def run_webui(config: Any, workspace_dir: str | None = None) -> None:
config: Effective ``EvoScientistConfig`` (already env-applied upstream,
but re-applied here so this is safe to call standalone).
workspace_dir: Resolved workspace path; falls back to
``config.default_workdir`` then cwd.
``config.default_workdir`` then ``~/.evoscientist/workspace``.
Blocks until Ctrl+C / SIGTERM, or until the front-end process exits, then
tears down both subprocesses. Never returns a value.
"""
from ..config import apply_config_to_env
from ..config import apply_config_to_env, get_default_workspace_dir
from ..config.provider_admin import get_provider_admin_token
from ..langgraph_dev.manager import (
_DEFAULT_PORT,
RUNTIME,
@@ -75,7 +76,7 @@ def run_webui(config: Any, workspace_dir: str | None = None) -> None:
apply_config_to_env(config)
# 1. Resolve workspace (CLI-resolved value > config.default_workdir > cwd),
# 1. Resolve workspace (CLI > config.default_workdir > stable app default),
# mirroring `EvoSci deploy`. The langgraph dev subprocess inherits this via
# EVOSCIENTIST_WORKSPACE_DIR (set inside start_langgraph_dev).
if workspace_dir:
@@ -83,8 +84,14 @@ def run_webui(config: Any, workspace_dir: str | None = None) -> None:
elif getattr(config, "default_workdir", ""):
ws = os.path.abspath(os.path.expanduser(config.default_workdir))
else:
ws = os.getcwd()
ws = str(get_default_workspace_dir())
os.makedirs(ws, exist_ok=True)
scope_service_token = ""
if getattr(config, "workspace_isolation", "optional") != "legacy":
from ..scope_registry import get_scope_service_token
scope_service_token = get_scope_service_token(ws)
os.environ["EVOSCIENTIST_BACKEND_SERVICE_TOKEN"] = scope_service_token
# 2. Resolve ports: backend = langgraph dev (browser connects here),
# webui_port = the local Next.js server the browser actually opens.
@@ -130,6 +137,28 @@ def run_webui(config: Any, workspace_dir: str | None = None) -> None:
)
raise typer.Exit(1)
# Token tracking is enabled only for this integrated topology. Generate the
# stable deployment/workspace identities and the dedicated transport token
# before starting either child so both processes share the exact values.
from ..usage import prepare_usage_environment
try:
usage_env = prepare_usage_environment(ws, webui_port=webui_port)
except (OSError, ValueError) as exc:
console.print(f"[yellow]Token statistics disabled:[/yellow] {exc}")
usage_env = {}
os.environ.update(usage_env)
# The provider editor uses a server-to-server management token. Resolve it
# through the shared user-level token file so a separately started WebUI,
# a fresh backend, and a reused backend all authenticate with the same
# value. The browser never receives it; the Next.js proxy adds it.
provider_admin_token = get_provider_admin_token()
if not provider_admin_token:
console.print("[red]Could not create the provider management token.[/red]")
raise typer.Exit(1)
os.environ["EVOSCIENTIST_PROVIDER_ADMIN_TOKEN"] = provider_admin_token
# 4. Backend (langgraph dev): reuse an EvoSci server already on the port,
# else start a fresh deploy-mode one (full MCP + async). Refuse a foreign
# occupant — that's a configuration error, not something to silently share.
@@ -205,7 +234,12 @@ def run_webui(config: Any, workspace_dir: str | None = None) -> None:
# are scrubbed — the browser UI never needs LLM provider API keys.
webui_env = _scrubbed_env(
{
**usage_env,
"EVOSCIENTIST_LANGGRAPH_DEV_PORT": str(backend_port),
"EVOSCIENTIST_BACKEND_URL": f"http://127.0.0.1:{backend_port}",
"EVOSCIENTIST_PROVIDER_ADMIN_TOKEN": provider_admin_token,
"EVOSCIENTIST_BACKEND_SERVICE_TOKEN": scope_service_token,
"EVOSCIENTIST_WORKSPACE_DIR": ws,
"PORT": str(webui_port),
}
)
EvoScientist/langgraph_dev/http.py
+1375 -18
View File
File diff suppressed because it is too large Load Diff
EvoScientist/langgraph_dev/manager.py
+46 -10
View File
@@ -20,6 +20,7 @@ import shutil
import subprocess
import threading
import time
import uuid
from dataclasses import dataclass
from pathlib import Path
@@ -177,7 +178,9 @@ class WorkspaceMismatchError(RuntimeError):
"""
def _write_workspace_sidecar(workspace_dir: Path, pid: int) -> None:
def _write_workspace_sidecar(
workspace_dir: Path, pid: int, *, port: int = _DEFAULT_PORT
) -> None:
"""Record the workspace + pid of the langgraph dev we just started.
Atomic write via temp-file + ``os.replace``: without this, a concurrent
@@ -194,8 +197,20 @@ def _write_workspace_sidecar(workspace_dir: Path, pid: int) -> None:
try:
RUNTIME.pid_dir.mkdir(parents=True, exist_ok=True)
tmp = RUNTIME.workspace_sidecar.with_suffix(".json.tmp")
resolved_workspace = workspace_dir.resolve()
deployment_id = os.getenv("EVOSCIENTIST_DEPLOYMENT_ID") or str(
uuid.uuid5(uuid.NAMESPACE_URL, f"evoscientist:{resolved_workspace}")
)
tmp.write_text(
json.dumps({"workspace": str(workspace_dir), "pid": pid}), encoding="utf-8"
json.dumps(
{
"workspace": str(workspace_dir),
"pid": pid,
"deployment_id": deployment_id,
"api_url": _base_url(port),
}
),
encoding="utf-8",
)
os.replace(tmp, RUNTIME.workspace_sidecar)
except OSError as exc:
@@ -231,11 +246,14 @@ def _read_workspace_sidecar() -> dict | None:
return data
def _unlink_workspace_sidecar() -> None:
def _unlink_workspace_sidecar(
runtime: LanggraphRuntimePaths | None = None,
) -> None:
"""Best-effort sidecar removal — called alongside every ``RUNTIME.pid_file.unlink()``
so the workspace fingerprint never outlives the PID file it pairs with."""
runtime = runtime or RUNTIME
try:
RUNTIME.workspace_sidecar.unlink()
runtime.workspace_sidecar.unlink()
except OSError:
pass
@@ -270,6 +288,16 @@ _LOG_OFFSET_AT_START: int = 0
# langgraph dev log. Mirrors langgraph_api/tunneling/cloudflare.py.
_TUNNEL_URL_RE = re.compile(r"https://[A-Za-z0-9.-]+\.trycloudflare\.com")
def current_log_start_offset() -> int:
"""Return the byte offset where the current server session started.
Consumers that mirror the Gateway log can start here to avoid replaying
output from earlier deploy sessions that share the append-only log file.
"""
return _LOG_OFFSET_AT_START
# Whether async sub-agents are usable in this process.
#
# - CLI / serve parent process: starts False; flipped True after
@@ -743,7 +771,7 @@ def start_langgraph_dev(
except Exception:
pass
RUNTIME.pid_file.write_text(str(proc.pid), encoding="utf-8")
_write_workspace_sidecar(workspace_dir=workspace_dir, pid=proc.pid)
_write_workspace_sidecar(workspace_dir=workspace_dir, pid=proc.pid, port=port)
global _PROCESS_WORKSPACE
_PROCESS = proc
_PROCESS_WORKSPACE = workspace_dir
@@ -820,7 +848,11 @@ def read_tunnel_url(timeout: float = 35.0, poll_interval: float = 0.5) -> str |
return None
def stop_langgraph_dev(proc: subprocess.Popen | None = None) -> None:
def stop_langgraph_dev(
proc: subprocess.Popen | None = None,
*,
runtime: LanggraphRuntimePaths | None = None,
) -> None:
"""Gracefully stop a langgraph dev process.
Sends SIGTERM to the process group (langgraph dev spawns worker children),
@@ -831,6 +863,7 @@ def stop_langgraph_dev(proc: subprocess.Popen | None = None) -> None:
(which also hold ``_LOCK``) don't observe partially-cleared state.
"""
global _PROCESS, _PROCESS_WORKSPACE
runtime = runtime or RUNTIME
with _LOCK:
proc = proc if proc is not None else _PROCESS
if proc is None:
@@ -879,12 +912,12 @@ def stop_langgraph_dev(proc: subprocess.Popen | None = None) -> None:
if proc is _PROCESS:
_PROCESS = None
_PROCESS_WORKSPACE = None
if RUNTIME.pid_file.exists():
if runtime.pid_file.exists():
try:
RUNTIME.pid_file.unlink()
runtime.pid_file.unlink()
except OSError:
pass
_unlink_workspace_sidecar()
_unlink_workspace_sidecar(runtime)
# Note: ``.langgraph_api/`` is intentionally NOT removed — it holds
# langgraph dev's persisted async-task / scheduler / Store state that
@@ -1064,5 +1097,8 @@ def _ensure_langgraph_dev_locked(
return None
_ASYNC_SUBAGENTS_AVAILABLE = True
atexit.register(stop_langgraph_dev, proc)
# Capture the runtime paths alongside the process. Embedded callers and
# tests may replace the module-level RUNTIME before interpreter shutdown;
# consulting that later would unlink an unrelated live deployment's files.
atexit.register(stop_langgraph_dev, proc, runtime=RUNTIME)
return proc
EvoScientist/llm/models.py
+352 -23
View File
@@ -9,12 +9,28 @@ endpoints) and convenient short names for common models.
from __future__ import annotations
import asyncio
import hashlib
import json
import os
import re
import warnings
from typing import Any
from langchain.chat_models import init_chat_model
from ..config.provider_profiles import (
ProviderModel,
ProviderProfile,
ProviderProfileError,
get_builtin_provider_profile,
get_provider_profile,
get_provider_profile_revision,
get_provider_profiles_path,
list_configured_builtin_model_entries,
list_configured_model_entries,
resolve_provider_model,
)
from .context_window import apply_known_context_window
from .patches import (
_is_ccproxy_codex,
@@ -241,6 +257,135 @@ MODELS: dict[str, tuple[str, str]] = {
}
DEFAULT_MODEL = "claude-sonnet-4-6"
_STATIC_PROVIDER_IDS = {provider for _, _, provider in _MODEL_ENTRIES} | {"ollama"}
_MODEL_CATALOG_ID_PATTERN = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$")
def normalize_builtin_model_catalog(raw: Any) -> list[dict[str, Any]] | None:
"""Validate and normalize the optional config.yaml built-in model catalog."""
if raw is None:
return None
if not isinstance(raw, list):
raise ValueError("model_catalog must be a list or null.")
normalized: list[dict[str, Any]] = []
seen: set[tuple[str, str]] = set()
for index, item in enumerate(raw):
context = f"model_catalog[{index}]"
if not isinstance(item, dict):
raise ValueError(f"{context} must be an object.")
provider = item.get("provider")
alias = item.get("id")
name = item.get("name")
model_id = item.get("model_id")
if not isinstance(provider, str) or not provider.strip():
raise ValueError(f"{context}.provider is required.")
if not isinstance(alias, str) or not alias.strip():
raise ValueError(f"{context}.id is required.")
if not isinstance(name, str) or not name.strip():
raise ValueError(f"{context}.name is required.")
if not isinstance(model_id, str) or not model_id.strip():
raise ValueError(f"{context}.model_id is required.")
provider = provider.strip()
alias = alias.strip()
name = name.strip()
model_id = model_id.strip()
if provider not in _STATIC_PROVIDER_IDS:
raise ValueError(
f"{context}.provider must be a built-in provider, got {provider!r}."
)
if not _MODEL_CATALOG_ID_PATTERN.fullmatch(alias):
raise ValueError(
f"{context}.id must start with a lowercase letter or digit and "
"contain only lowercase letters, digits, dots, underscores, or hyphens."
)
if len(name) > 120:
raise ValueError(f"{context}.name must be at most 120 characters.")
if len(model_id) > 300:
raise ValueError(f"{context}.model_id must be at most 300 characters.")
key = (provider, alias)
if key in seen:
raise ValueError(
f"Model alias {alias!r} is duplicated for provider {provider!r}."
)
seen.add(key)
normalized.append(
{
"provider": provider,
"id": alias,
"name": name,
"model_id": model_id,
"enabled": bool(item.get("enabled", True)),
}
)
return normalized
def list_builtin_model_catalog_entries(
raw: Any,
) -> list[tuple[str, str, str]] | None:
"""Return enabled catalog entries, or ``None`` for legacy registry mode."""
catalog = normalize_builtin_model_catalog(raw)
if catalog is None:
return None
return [
(entry["id"], entry["model_id"], entry["provider"])
for entry in catalog
if entry["enabled"]
]
def _resolve_builtin_catalog_model(provider: str, model: str) -> str | None:
from ..config.settings import load_config
profile = get_builtin_provider_profile(provider)
if profile is not None:
if not profile.enabled:
return None
for candidate in profile.models:
if candidate.id == model:
return candidate.model_id if candidate.enabled else None
return None
try:
catalog = normalize_builtin_model_catalog(load_config().model_catalog)
except ValueError:
return None
if catalog is None:
return None
for entry in catalog:
if entry["provider"] == provider and entry["id"] == model:
return entry["model_id"] if entry["enabled"] else None
return None
def get_model_runtime_revision(provider: str | None) -> str | None:
"""Return the dynamic profile revision used in model cache keys."""
if not provider:
return None
if provider in _STATIC_PROVIDER_IDS:
managed_revision = get_provider_profile_revision(provider)
if managed_revision is not None:
return managed_revision
from ..config.settings import load_config
try:
catalog = normalize_builtin_model_catalog(load_config().model_catalog)
except ValueError:
return "invalid"
if catalog is None:
return None
provider_catalog = [entry for entry in catalog if entry["provider"] == provider]
encoded = json.dumps(
provider_catalog, sort_keys=True, separators=(",", ":")
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()[:16]
try:
return get_provider_profile_revision(provider)
except ProviderProfileError:
return None
def get_models_for_provider(provider: str) -> list[tuple[str, str]]:
@@ -252,7 +397,25 @@ def get_models_for_provider(provider: str) -> list[tuple[str, str]]:
Returns:
List of (short_name, model_id) tuples for the provider.
"""
return [(name, model_id) for name, model_id, p in _MODEL_ENTRIES if p == provider]
if provider in _STATIC_PROVIDER_IDS:
managed = get_builtin_provider_profile(provider)
if managed is not None:
if not managed.enabled:
return []
return [
(model.id, model.model_id) for model in managed.models if model.enabled
]
return [
(name, model_id)
for name, model_id, entry_provider in _MODEL_ENTRIES
if entry_provider == provider
]
profile = get_provider_profile(provider)
if profile is not None:
if not profile.enabled:
return []
return [(model.id, model.model_id) for model in profile.models if model.enabled]
return []
def _env_flag_enabled(name: str) -> bool:
@@ -365,6 +528,10 @@ def _apply_auto_config(
def get_chat_model(
model: str | None = None,
provider: str | None = None,
*,
profile_override: ProviderProfile | None = None,
model_override: ProviderModel | None = None,
skip_auto_config: bool = False,
**kwargs: Any,
) -> Any:
"""Get a chat model instance.
@@ -387,15 +554,42 @@ def get_chat_model(
"""
model = model or DEFAULT_MODEL
# Look up short name in registry (provider-aware)
model_id = None
if provider:
dynamic_profile = profile_override
dynamic_model = model_override
if dynamic_profile is not None:
provider = dynamic_profile.id
if dynamic_model is None:
dynamic_model = next(
(
candidate
for candidate in dynamic_profile.models
if candidate.id == model or candidate.model_id == model
),
None,
)
if dynamic_model is None:
raise ValueError(
f"Model {model!r} is not configured for provider {dynamic_profile.id!r}."
)
elif provider and provider not in _STATIC_PROVIDER_IDS:
try:
dynamic_resolution = resolve_provider_model(provider, model)
except ProviderProfileError as exc:
raise ValueError(str(exc)) from exc
if dynamic_resolution is not None:
dynamic_profile, dynamic_model = dynamic_resolution
# Look up short name in the configured catalog, then the legacy registry.
model_id = dynamic_model.model_id if dynamic_model is not None else None
if model_id is None and provider in _STATIC_PROVIDER_IDS:
model_id = _resolve_builtin_catalog_model(provider, model)
if provider and dynamic_profile is None and model_id is None:
# Try exact match with provider first
for name, mid, p in _MODEL_ENTRIES:
if name == model and p == provider:
model_id = mid
break
if model_id is None and model in MODELS:
if model_id is None and dynamic_profile is None and model in MODELS:
model_id, default_provider = MODELS[model]
provider = provider or default_provider
@@ -422,19 +616,82 @@ def get_chat_model(
)
_is_openai_proxy = False
_original_provider: str | None = None
if provider == "anthropic":
managed_builtin_profile = (
get_builtin_provider_profile(provider)
if provider in _STATIC_PROVIDER_IDS
else None
)
if managed_builtin_profile is not None:
if not managed_builtin_profile.enabled:
raise ValueError(f"Provider {provider!r} is disabled.")
from ..config.settings import get_effective_config
from .provider_operations import build_builtin_provider_profile
effective_profile = build_builtin_provider_profile(
get_effective_config(), provider
)
if effective_profile.api_key and provider != "ollama":
kwargs.setdefault("api_key", effective_profile.api_key)
if effective_profile.base_url:
kwargs.setdefault("base_url", effective_profile.base_url)
if dynamic_profile is not None:
adapter = dynamic_profile.adapter
if dynamic_profile.api_key and adapter != "ollama":
kwargs.setdefault("api_key", dynamic_profile.api_key)
if dynamic_profile.base_url:
kwargs.setdefault("base_url", dynamic_profile.base_url)
elif adapter == "grok":
kwargs.setdefault("base_url", "https://api.x.ai/v1")
if adapter in {"openai-compatible", "grok", "antigravity"}:
provider = "openai"
_original_provider = dynamic_profile.id
_is_third_party = True
elif adapter == "anthropic-compatible":
provider = "anthropic"
_original_provider = dynamic_profile.id
_is_third_party = True
else:
provider = adapter
_is_third_party = adapter in {
"openrouter",
"openai-compatible",
"anthropic-compatible",
"grok",
"antigravity",
} or (adapter in {"openai", "anthropic"} and bool(dynamic_profile.base_url))
if _is_third_party:
_original_provider = dynamic_profile.id
# Some OpenAI-compatible reseller gateways reject the SDK's default
# ``OpenAI/Python`` User-Agent with a generic 403. WebUI profiles with a
# custom Base URL are third-party endpoints, so use the same compatible
# User-Agent as the legacy ``custom-openai`` provider.
if provider == "openai" and dynamic_profile.base_url:
kwargs.setdefault("default_headers", {}).setdefault(
"User-Agent", "codex_cli_rs/0.0.0"
)
if provider == "openrouter" and not skip_auto_config:
effort = (
os.environ.get("EVOSCIENTIST_REASONING_EFFORT", "").strip() or "high"
)
kwargs.setdefault("reasoning", {"effort": effort, "summary": "auto"})
_patch_openrouter_strip_responses_reasoning()
elif provider == "anthropic":
base_url = os.environ.get("ANTHROPIC_BASE_URL", "")
if base_url:
kwargs["base_url"] = base_url
kwargs.setdefault("base_url", base_url)
api_key = os.environ.get("ANTHROPIC_API_KEY", "")
if api_key:
kwargs["api_key"] = api_key
kwargs.setdefault("api_key", api_key)
# Native OpenAI base_url override (e.g. ccproxy Codex at localhost:8000/codex/v1)
elif provider == "openai":
base_url = os.environ.get("OPENAI_BASE_URL", "")
if base_url:
kwargs["base_url"] = base_url
kwargs.setdefault("base_url", base_url)
_is_openai_proxy = _is_ccproxy_codex()
if _is_openai_proxy:
# Use Responses API for ccproxy: bypasses the format chain
@@ -450,14 +707,16 @@ def get_chat_model(
kwargs.pop("streaming", None) # remove if set elsewhere
api_key = os.environ.get("OPENAI_API_KEY", "")
if api_key:
kwargs["api_key"] = api_key
kwargs.setdefault("api_key", api_key)
# OpenAI-routed providers → route through OpenAI provider with base_url
elif provider in _OPENAI_ROUTED_PROVIDERS:
_original_provider = provider
base_url_default, api_key_env = _OPENAI_ROUTED_PROVIDERS[provider]
if provider == "custom-openai":
base_url = os.environ.get("CUSTOM_OPENAI_BASE_URL", "")
base_url = os.environ.get("CUSTOM_OPENAI_BASE_URL", "") or kwargs.get(
"base_url", ""
)
if not base_url:
raise ValueError(
"CUSTOM_OPENAI_BASE_URL environment variable is required when using "
@@ -468,10 +727,10 @@ def get_chat_model(
else:
base_url = base_url_default
if base_url:
kwargs["base_url"] = base_url
kwargs.setdefault("base_url", base_url)
api_key = os.environ.get(api_key_env, "")
if api_key:
kwargs["api_key"] = api_key
kwargs.setdefault("api_key", api_key)
# SiliconFlow: disable thinking — LangChain drops reasoning_content
# from history, causing error 20015 on multi-turn requests.
if provider == "siliconflow":
@@ -497,7 +756,7 @@ def get_chat_model(
_is_third_party = True
api_key = os.environ.get("OPENROUTER_API_KEY", "")
if api_key:
kwargs["api_key"] = api_key
kwargs.setdefault("api_key", api_key)
# Reasoning via `effort` + `summary: "auto"` so a readable reasoning
# summary is returned for display. OpenAI-Responses also emits encrypted
# reasoning items (`rs_*` id) that can't be replayed on multi-turn
@@ -513,7 +772,9 @@ def get_chat_model(
_original_provider = provider
base_url_default, api_key_env = _ANTHROPIC_ROUTED_PROVIDERS[provider]
if provider == "custom-anthropic":
base_url = os.environ.get("CUSTOM_ANTHROPIC_BASE_URL", "")
base_url = os.environ.get("CUSTOM_ANTHROPIC_BASE_URL", "") or kwargs.get(
"base_url", ""
)
if not base_url:
raise ValueError(
"CUSTOM_ANTHROPIC_BASE_URL environment variable is required when using "
@@ -526,10 +787,10 @@ def get_chat_model(
else:
base_url = base_url_default
if base_url:
kwargs["base_url"] = base_url
kwargs.setdefault("base_url", base_url)
api_key = os.environ.get(api_key_env, "")
if api_key:
kwargs["api_key"] = api_key
kwargs.setdefault("api_key", api_key)
# Kimi Coding Plan requires claude-code User-Agent header
if provider == "kimi-coding":
kwargs.setdefault("default_headers", {})["User-Agent"] = "claude-code/0.1.0"
@@ -538,10 +799,13 @@ def get_chat_model(
elif provider == "ollama":
base_url = os.environ.get("OLLAMA_BASE_URL", "")
if base_url:
kwargs["base_url"] = base_url
kwargs.setdefault("base_url", base_url)
_apply_auto_config(provider, model_id, _is_third_party, kwargs, _original_provider)
_apply_openrouter_anthropic_prompt_cache(provider, model_id, kwargs)
if not skip_auto_config:
_apply_auto_config(
provider, model_id, _is_third_party, kwargs, _original_provider
)
_apply_openrouter_anthropic_prompt_cache(provider, model_id, kwargs)
# User-level override for the OpenAI Responses API vs Chat Completions.
# When "false", force Chat Completions and drop reasoning (which triggers
@@ -556,7 +820,27 @@ def get_chat_model(
elif _responses_api_setting == "true":
kwargs["use_responses_api"] = True
# Usage tracking is an integrated-WebUI concern and is inert unless the
# launcher supplied the complete sink environment. Preserve the logical
# profile identity before compatible providers are represented by a native
# OpenAI/Anthropic adapter.
from ..usage import UsageModelIdentity, attach_usage_callback
_usage_profile_id = (
dynamic_profile.id
if dynamic_profile is not None
else (_original_provider or provider or "unknown")
)
_usage_identity = UsageModelIdentity(
provider_profile_id=_usage_profile_id,
provider_revision=get_model_runtime_revision(_usage_profile_id),
provider_adapter=provider or "unknown",
model_alias=model,
upstream_model_id=model_id,
)
chat_model = init_chat_model(model=model_id, model_provider=provider, **kwargs)
chat_model = attach_usage_callback(chat_model, _usage_identity)
# Flatten list content to strings for strict OpenAI-compatible providers
# (DeepSeek, SiliconFlow, OpenRouter, custom-openai, etc.) and
@@ -584,6 +868,24 @@ def get_chat_model(
return chat_model
def get_profile_chat_model(
profile: ProviderProfile,
model: ProviderModel,
*,
skip_auto_config: bool = False,
**kwargs: Any,
) -> Any:
"""Build a model from an unsaved or persisted provider profile."""
return get_chat_model(
model=model.id,
provider=profile.id,
profile_override=profile,
model_override=model,
skip_auto_config=skip_auto_config,
**kwargs,
)
def list_models() -> list[str]:
"""List all available model short names.
@@ -619,10 +921,37 @@ async def list_model_picker_entries(
ollama_base_url: str | None,
*,
include_custom_ollama: bool,
model_catalog: Any = None,
) -> list[tuple[str, str, str]]:
"""Return model picker entries, optionally including local Ollama models."""
entries = list_models_by_provider()
if ollama_base_url:
"""Return models from the WebUI registry, or legacy entries before migration."""
try:
registry_configured = await asyncio.to_thread(
get_provider_profiles_path().exists
)
configured_builtin_entries = (
await asyncio.to_thread(list_configured_builtin_model_entries)
if registry_configured
else list_builtin_model_catalog_entries(model_catalog)
)
except (ProviderProfileError, ValueError):
configured_builtin_entries = []
legacy_mode = configured_builtin_entries is None
entries = (
list_models_by_provider()
if configured_builtin_entries is None
else configured_builtin_entries
)
seen = {(name, provider) for name, _, provider in entries}
try:
configured_entries = await asyncio.to_thread(list_configured_model_entries)
except ProviderProfileError:
configured_entries = []
for name, model_id, provider in configured_entries:
key = (name, provider)
if key not in seen:
entries.append((name, model_id, provider))
seen.add(key)
if legacy_mode and ollama_base_url:
from .ollama_discovery import discover_ollama_models
for detected_name in await discover_ollama_models(
EvoScientist/llm/patches.py
+109 -5
View File
@@ -926,24 +926,128 @@ def _read_cfg_configurable() -> dict[str, str]:
def _merge_runs_config_kwargs(kwargs: dict) -> dict:
"""Merge the live model override into ``kwargs`` for ``runs.create``.
"""Merge model and usage correlation context into ``runs.create``.
Preserves any caller-supplied ``config.configurable`` keys. EvoScientist's
keys take precedence on conflict (callers shouldn't be passing model
overrides — the CLI is the source of truth).
"""
overrides = _read_cfg_configurable()
if not overrides:
return kwargs
usage_enabled = os.getenv("EVOSCIENTIST_USAGE_TRACKING", "").strip().lower() in {
"1",
"true",
"yes",
"on",
}
current_metadata: dict = {}
current_configurable: dict = {}
try:
from langgraph.config import get_config
# Scoped deployments need this context even when usage accounting is
# disabled: DeepAgents creates derived threads through this proxy.
current = get_config()
raw_metadata = current.get("metadata")
raw_configurable = current.get("configurable")
if isinstance(raw_metadata, dict):
current_metadata = raw_metadata
if isinstance(raw_configurable, dict):
current_configurable = raw_configurable
except (LookupError, RuntimeError):
pass
inherited_scope: dict[str, object] = {}
scope_keys = (
"workspace_scope_id",
"workspace_scope_owner_id",
"workspace_scope_revision",
"workspace_deployment_id",
)
if all(current_configurable.get(key) is not None for key in scope_keys):
inherited_scope = {key: current_configurable[key] for key in scope_keys}
target_thread_id = kwargs.get("thread_id")
source_thread_id = current_configurable.get("thread_id")
if (
isinstance(target_thread_id, str)
and target_thread_id
and target_thread_id != source_thread_id
):
try:
from EvoScientist.scope_registry import (
ScopeConflictError,
get_scope_registry,
)
registry = get_scope_registry()
try:
owner = registry.register_owner(
str(inherited_scope["workspace_deployment_id"]),
str(inherited_scope["workspace_scope_id"]),
owner_type="derived_run",
resource_id=target_thread_id,
parent_owner_id=str(inherited_scope["workspace_scope_owner_id"]),
state="active",
)
except ScopeConflictError:
owner = registry.get_owner_by_resource(
str(inherited_scope["workspace_deployment_id"]),
target_thread_id,
)
if owner.scope_id != inherited_scope["workspace_scope_id"]:
raise
inherited_scope["workspace_scope_owner_id"] = owner.owner_id
inherited_scope["thread_id"] = target_thread_id
except Exception:
# The required-mode backend factory rejects missing/invalid
# ownership at execution time. Optional mode keeps legacy
# integrations working when they cannot register a child.
if os.getenv("EVOSCIENTIST_WORKSPACE_ISOLATION", "optional").lower() == "required":
raise
existing = kwargs.get("config")
if not isinstance(existing, dict):
existing = {}
existing_configurable = existing.get("configurable")
if not isinstance(existing_configurable, dict):
existing_configurable = {}
merged_configurable = {**existing_configurable, **overrides}
merged_configurable = {**existing_configurable, **overrides, **inherited_scope}
kwargs = dict(kwargs)
kwargs["config"] = {**existing, "configurable": merged_configurable}
if overrides or "config" in kwargs:
kwargs["config"] = {**existing, "configurable": merged_configurable}
if not usage_enabled:
return kwargs
inherited_metadata = {
key: current_metadata[key]
for key in (
"usage_context_version",
"turn_id",
"source_session_id",
"source_agent",
"workspace_dir",
)
if current_metadata.get(key) is not None
}
inherited_metadata.setdefault(
"source_session_id",
current_metadata.get("thread_id")
or current_metadata.get("langgraph_thread_id")
or current_configurable.get("thread_id"),
)
if inherited_metadata.get("source_session_id") is None:
inherited_metadata.pop("source_session_id", None)
existing_metadata = kwargs.get("metadata")
if not isinstance(existing_metadata, dict):
existing_metadata = {}
merged_metadata = {**inherited_metadata, **existing_metadata}
run_kind = merged_metadata.get("run_kind")
if "usage_scope" not in merged_metadata:
if isinstance(run_kind, str) and run_kind.startswith("evomemory_"):
merged_metadata["usage_scope"] = "memory"
else:
merged_metadata["usage_scope"] = "async_subagent"
kwargs["metadata"] = merged_metadata
return kwargs
EvoScientist/llm/provider_operations.py
+519
View File
@@ -0,0 +1,519 @@
"""Provider model discovery and lightweight connectivity tests."""
from __future__ import annotations
import asyncio
import os
import time
from dataclasses import dataclass
from typing import Any
import httpx
from ..config.provider_profiles import (
ProviderModel,
ProviderProfile,
get_builtin_provider_profile,
normalize_provider_base_url,
)
from ..config.settings import is_config_applied_env
_DEFAULT_BASE_URLS = {
"openai": "https://api.openai.com/v1",
"anthropic": "https://api.anthropic.com",
"google-genai": "https://generativelanguage.googleapis.com/v1beta",
"grok": "https://api.x.ai/v1",
"openrouter": "https://openrouter.ai/api/v1",
"nvidia": "https://integrate.api.nvidia.com/v1",
"ollama": "http://127.0.0.1:11434",
}
_API_KEY_REQUIRED_ADAPTERS = {
"openai",
"anthropic",
"google-genai",
"grok",
"openrouter",
"nvidia",
}
_MAX_DISCOVERED_MODELS = 2_000
_MAX_PAGES = 100
_MAX_REQUEST_ATTEMPTS = 2
@dataclass(frozen=True)
class BuiltinProviderConnection:
adapter: str
api_key_field: str | None
base_url_field: str | None = None
default_base_url: str = ""
_BUILTIN_PROVIDER_CONNECTIONS = {
"anthropic": BuiltinProviderConnection(
"anthropic", "anthropic_api_key", "anthropic_base_url"
),
"openai": BuiltinProviderConnection("openai", "openai_api_key"),
"google-genai": BuiltinProviderConnection("google-genai", "google_api_key"),
"minimax": BuiltinProviderConnection(
"anthropic-compatible",
"minimax_api_key",
"minimax_base_url",
"https://api.minimaxi.com/anthropic",
),
"nvidia": BuiltinProviderConnection("nvidia", "nvidia_api_key"),
"siliconflow": BuiltinProviderConnection(
"openai-compatible",
"siliconflow_api_key",
default_base_url="https://api.siliconflow.cn/v1",
),
"openrouter": BuiltinProviderConnection("openrouter", "openrouter_api_key"),
"zhipu": BuiltinProviderConnection(
"openai-compatible",
"zhipu_api_key",
default_base_url="https://open.bigmodel.cn/api/paas/v4",
),
"zhipu-code": BuiltinProviderConnection(
"openai-compatible",
"zhipu_api_key",
default_base_url="https://open.bigmodel.cn/api/coding/paas/v4",
),
"volcengine": BuiltinProviderConnection(
"openai-compatible",
"volcengine_api_key",
default_base_url="https://ark.cn-beijing.volces.com/api/v3",
),
"dashscope": BuiltinProviderConnection(
"openai-compatible",
"dashscope_api_key",
default_base_url="https://dashscope.aliyuncs.com/compatible-mode/v1",
),
"dashscope-code": BuiltinProviderConnection(
"openai-compatible",
"dashscope_api_key",
default_base_url="https://coding.dashscope.aliyuncs.com/v1",
),
"deepseek": BuiltinProviderConnection(
"openai-compatible",
"deepseek_api_key",
default_base_url="https://api.deepseek.com",
),
"moonshot": BuiltinProviderConnection(
"openai-compatible",
"moonshot_api_key",
default_base_url="https://api.moonshot.cn/v1",
),
"kimi-coding": BuiltinProviderConnection(
"anthropic-compatible",
"kimi_api_key",
default_base_url="https://api.kimi.com/coding/",
),
"custom-openai": BuiltinProviderConnection(
"openai-compatible",
"custom_openai_api_key",
"custom_openai_base_url",
),
"custom-anthropic": BuiltinProviderConnection(
"anthropic-compatible",
"custom_anthropic_api_key",
"custom_anthropic_base_url",
),
"ollama": BuiltinProviderConnection("ollama", None, "ollama_base_url"),
}
_BUILTIN_API_KEY_ENVS = {
"anthropic": "ANTHROPIC_API_KEY",
"openai": "OPENAI_API_KEY",
"google-genai": "GOOGLE_API_KEY",
"minimax": "MINIMAX_API_KEY",
"nvidia": "NVIDIA_API_KEY",
"siliconflow": "SILICONFLOW_API_KEY",
"openrouter": "OPENROUTER_API_KEY",
"zhipu": "ZHIPU_API_KEY",
"zhipu-code": "ZHIPU_API_KEY",
"volcengine": "VOLCENGINE_API_KEY",
"dashscope": "DASHSCOPE_API_KEY",
"dashscope-code": "DASHSCOPE_API_KEY",
"deepseek": "DEEPSEEK_API_KEY",
"moonshot": "MOONSHOT_API_KEY",
"kimi-coding": "KIMI_API_KEY",
"custom-openai": "CUSTOM_OPENAI_API_KEY",
"custom-anthropic": "CUSTOM_ANTHROPIC_API_KEY",
}
_BUILTIN_BASE_URL_ENVS = {
"anthropic": "ANTHROPIC_BASE_URL",
"openai": "OPENAI_BASE_URL",
"minimax": "MINIMAX_BASE_URL",
"custom-openai": "CUSTOM_OPENAI_BASE_URL",
"custom-anthropic": "CUSTOM_ANTHROPIC_BASE_URL",
"ollama": "OLLAMA_BASE_URL",
}
class ProviderOperationError(RuntimeError):
"""Raised when provider discovery or testing cannot complete."""
def build_builtin_provider_profile(
config: Any,
provider_id: str,
*,
saved_profile: ProviderProfile | None = None,
load_saved: bool = True,
use_environment: bool = True,
base_url_override: str | None = None,
api_key_override: str | None = None,
clear_api_key: bool = False,
) -> ProviderProfile:
"""Resolve a built-in profile with env > registry > legacy config priority."""
spec = _BUILTIN_PROVIDER_CONNECTIONS.get(provider_id)
if spec is None:
raise ProviderOperationError(f"Unknown built-in provider {provider_id!r}.")
saved = get_builtin_provider_profile(provider_id) if load_saved else saved_profile
base_url = saved.base_url if saved is not None else spec.default_base_url
if saved is None and spec.base_url_field:
configured_base_url = getattr(config, spec.base_url_field, "")
if isinstance(configured_base_url, str) and configured_base_url.strip():
base_url = configured_base_url.strip()
base_url_env = _BUILTIN_BASE_URL_ENVS.get(provider_id)
if (
base_url_env
and use_environment
and os.environ.get(base_url_env, "").strip()
and (saved is None or not is_config_applied_env(base_url_env))
):
base_url = os.environ[base_url_env].strip()
if base_url_override is not None:
base_url = base_url_override.strip() or spec.default_base_url
api_key = saved.api_key if saved is not None else ""
if saved is None and spec.api_key_field:
configured_api_key = getattr(config, spec.api_key_field, "")
if isinstance(configured_api_key, str):
api_key = configured_api_key.strip()
api_key_env = _BUILTIN_API_KEY_ENVS.get(provider_id)
if (
api_key_env
and use_environment
and os.environ.get(api_key_env, "").strip()
and (saved is None or not is_config_applied_env(api_key_env))
):
api_key = os.environ[api_key_env].strip()
if clear_api_key:
api_key = ""
elif api_key_override is not None and api_key_override.strip():
api_key = api_key_override.strip()
return ProviderProfile(
id=provider_id,
name=provider_id,
adapter=spec.adapter,
base_url=base_url,
api_key=api_key,
enabled=saved.enabled if saved is not None else True,
models=saved.models if saved is not None else (),
auth_mode=(
saved.auth_mode
if saved is not None
else getattr(config, f"{provider_id}_auth_mode", "api_key")
if provider_id in {"anthropic", "openai"}
else "api_key"
),
)
def builtin_provider_ids() -> tuple[str, ...]:
return tuple(_BUILTIN_PROVIDER_CONNECTIONS)
def builtin_provider_adapter(provider_id: str) -> str:
spec = _BUILTIN_PROVIDER_CONNECTIONS.get(provider_id)
if spec is None:
raise ProviderOperationError(f"Unknown built-in provider {provider_id!r}.")
return spec.adapter
def builtin_provider_legacy_api_key(config: Any, provider_id: str) -> str:
"""Return only the config.yaml key, excluding environment overrides."""
spec = _BUILTIN_PROVIDER_CONNECTIONS.get(provider_id)
if spec is None or spec.api_key_field is None:
return ""
value = getattr(config, spec.api_key_field, "")
return value.strip() if isinstance(value, str) else ""
@dataclass(frozen=True)
class DiscoveredProviderModel:
model_id: str
name: str
@dataclass(frozen=True)
class ProviderModelTestResult:
latency_ms: int
response: str
def _base_url(profile: ProviderProfile) -> str:
base_url = profile.base_url or _DEFAULT_BASE_URLS.get(profile.adapter, "")
if not base_url:
raise ProviderOperationError(
f"Base URL is required for adapter {profile.adapter}."
)
return normalize_provider_base_url(profile.adapter, base_url)
def _models_url(profile: ProviderProfile) -> str:
base_url = _base_url(profile)
if profile.adapter in {"anthropic", "anthropic-compatible"}:
return (
f"{base_url}/models"
if base_url.endswith(("/v1", "/v1beta"))
else f"{base_url}/v1/models"
)
if profile.adapter == "ollama":
return f"{base_url}/api/tags"
return f"{base_url}/models"
def _headers(profile: ProviderProfile) -> dict[str, str]:
if profile.adapter in _API_KEY_REQUIRED_ADAPTERS and not profile.api_key:
raise ProviderOperationError(
f"API key is required for adapter {profile.adapter}."
)
if profile.adapter in {"anthropic", "anthropic-compatible"}:
headers = {"anthropic-version": "2023-06-01"}
if profile.api_key:
headers["x-api-key"] = profile.api_key
return headers
if profile.adapter == "google-genai":
return {"x-goog-api-key": profile.api_key}
if profile.adapter == "ollama" or not profile.api_key:
return {}
return {"Authorization": f"Bearer {profile.api_key}"}
def _provider_error_message(response: httpx.Response) -> str:
message = ""
try:
payload = response.json()
if isinstance(payload, dict):
error = payload.get("error")
if isinstance(error, dict):
raw_message = error.get("message")
if isinstance(raw_message, str):
message = raw_message.strip()
elif isinstance(error, str):
message = error.strip()
if not message:
raw_message = payload.get("message")
if isinstance(raw_message, str):
message = raw_message.strip()
except (ValueError, TypeError):
pass
suffix = f": {message[:500]}" if message else ""
return f"Provider returned HTTP {response.status_code}{suffix}"
def _model_name(raw: dict[str, Any], model_id: str) -> str:
for key in ("display_name", "displayName", "name"):
value = raw.get(key)
if isinstance(value, str) and value.strip():
return value.strip()[:120]
return model_id[:120]
def _parse_models(
profile: ProviderProfile, payload: Any
) -> list[DiscoveredProviderModel]:
if not isinstance(payload, dict):
raise ProviderOperationError("Provider returned an invalid model list.")
records = (
payload.get("models") if profile.adapter == "ollama" else payload.get("data")
)
if profile.adapter == "google-genai":
records = payload.get("models")
elif not isinstance(records, list):
records = payload.get("models")
if not isinstance(records, list):
raise ProviderOperationError("Provider response does not contain a model list.")
result: list[DiscoveredProviderModel] = []
for raw in records:
if not isinstance(raw, dict):
continue
model_id_raw = (
(raw.get("model") or raw.get("name"))
if profile.adapter == "ollama"
else raw.get("id")
)
if profile.adapter == "google-genai":
model_id_raw = raw.get("name")
methods = raw.get("supportedGenerationMethods")
if isinstance(methods, list) and "generateContent" not in methods:
continue
if not isinstance(model_id_raw, str) or not model_id_raw.strip():
continue
model_id = model_id_raw.strip()
if profile.adapter == "google-genai":
model_id = model_id.removeprefix("models/")
result.append(
DiscoveredProviderModel(
model_id=model_id[:300],
name=_model_name(raw, model_id),
)
)
return result
async def discover_provider_models(
profile: ProviderProfile,
*,
transport: httpx.AsyncBaseTransport | None = None,
) -> list[DiscoveredProviderModel]:
"""Fetch every model exposed by a provider's model-list endpoint."""
url = _models_url(profile)
headers = _headers(profile)
params: dict[str, str | int] = {}
discovered: list[DiscoveredProviderModel] = []
seen: set[str] = set()
timeout = httpx.Timeout(30.0, connect=10.0)
async with httpx.AsyncClient(timeout=timeout, transport=transport) as client:
for _ in range(_MAX_PAGES):
response: httpx.Response | None = None
last_request_error: httpx.HTTPError | None = None
for attempt in range(_MAX_REQUEST_ATTEMPTS):
try:
response = await client.get(url, headers=headers, params=params)
break
except (httpx.TimeoutException, httpx.NetworkError) as exc:
last_request_error = exc
if attempt + 1 < _MAX_REQUEST_ATTEMPTS:
await asyncio.sleep(0.25 * (attempt + 1))
if response is None:
if isinstance(last_request_error, httpx.TimeoutException):
raise ProviderOperationError(
f"Provider model discovery timed out after "
f"{_MAX_REQUEST_ATTEMPTS} attempts."
) from last_request_error
error_name = (
last_request_error.__class__.__name__
if last_request_error is not None
else "NetworkError"
)
raise ProviderOperationError(
f"Could not connect to provider after "
f"{_MAX_REQUEST_ATTEMPTS} attempts: {error_name}."
) from last_request_error
if response.status_code >= 400:
raise ProviderOperationError(_provider_error_message(response))
try:
payload = response.json()
except ValueError as exc:
content_type = (
response.headers.get("content-type", "unknown")
.split(";", 1)[0]
.strip()
or "unknown"
)
hint = (
" OpenAI-compatible Base URLs normally end in /v1."
if profile.adapter
in {
"openai",
"openai-compatible",
"grok",
"openrouter",
"nvidia",
"antigravity",
}
else ""
)
raise ProviderOperationError(
f"Provider returned a non-JSON model list from {url} "
f"(Content-Type: {content_type}).{hint}"
) from exc
for model in _parse_models(profile, payload):
if model.model_id in seen:
continue
seen.add(model.model_id)
discovered.append(model)
if len(discovered) >= _MAX_DISCOVERED_MODELS:
return sorted(discovered, key=lambda item: item.model_id.lower())
if profile.adapter in {"anthropic", "anthropic-compatible"}:
if not isinstance(payload, dict) or payload.get("has_more") is not True:
break
last_id = payload.get("last_id")
if not isinstance(last_id, str) or not last_id:
break
params = {"after_id": last_id, "limit": 100}
continue
if profile.adapter == "google-genai":
next_page = (
payload.get("nextPageToken") if isinstance(payload, dict) else None
)
if not isinstance(next_page, str) or not next_page:
break
params = {"pageToken": next_page, "pageSize": 1000}
continue
break
return sorted(discovered, key=lambda item: item.model_id.lower())
def _response_text(response: Any) -> str:
content = getattr(response, "content", response)
if isinstance(content, str):
return content.strip()
if isinstance(content, list):
parts: list[str] = []
for block in content:
if isinstance(block, str):
parts.append(block)
elif isinstance(block, dict):
text = block.get("text") or block.get("content")
if isinstance(text, str):
parts.append(text)
return "\n".join(parts).strip()
return str(content).strip()
def _safe_test_error(exc: Exception, profile: ProviderProfile) -> str:
message = str(exc).strip() or exc.__class__.__name__
if profile.api_key:
message = message.replace(profile.api_key, "[redacted]")
return message[:800]
async def test_provider_model(
profile: ProviderProfile,
model: ProviderModel,
) -> ProviderModelTestResult:
"""Build the configured model and run a minimal real inference request."""
from .models import get_profile_chat_model
started = time.perf_counter()
try:
chat_model = await asyncio.to_thread(
get_profile_chat_model,
profile,
model,
max_tokens=64,
skip_auto_config=True,
)
response = await asyncio.wait_for(
chat_model.ainvoke("Reply with exactly: OK"), timeout=45.0
)
except TimeoutError as exc:
raise ProviderOperationError("Model test timed out after 45 seconds.") from exc
except Exception as exc:
raise ProviderOperationError(_safe_test_error(exc, profile)) from exc
latency_ms = max(1, round((time.perf_counter() - started) * 1000))
return ProviderModelTestResult(
latency_ms=latency_ms,
response=_response_text(response)[:500],
)
EvoScientist/memory/launch.py
+20 -10
View File
@@ -90,7 +90,7 @@ def _worker_workspace_dir(workspace_dir: str | Path) -> str:
def _memory_worker_metadata(context: MemorySourceContext) -> dict[str, str]:
return {
metadata = {
"run_kind": f"evomemory_{context.source_type.value}_worker",
"source_session_id": context.session_id,
"source_agent": context.source_agent,
@@ -98,6 +98,13 @@ def _memory_worker_metadata(context: MemorySourceContext) -> dict[str, str]:
"trajectory_digest": context.trajectory_digest,
"workspace_dir": _worker_workspace_dir(context.workspace_dir),
}
from ..usage.callback import usage_tracking_requested
if usage_tracking_requested():
metadata["usage_scope"] = "memory"
if usage_tracking_requested() and context.turn_id:
metadata["turn_id"] = context.turn_id
return metadata
def _memory_worker_run_payload(
@@ -107,19 +114,22 @@ def _memory_worker_run_payload(
) -> BackgroundRunPayload:
"""Build the LangGraph SDK run payload for a memory worker."""
metadata = _memory_worker_metadata(context)
configurable = {
"thread_id": thread_id,
"evomemory_source_session_id": context.session_id,
"evomemory_source_agent": context.source_agent,
"evomemory_project_id": context.project_id,
"evomemory_trajectory_digest": context.trajectory_digest,
}
from ..usage.callback import usage_tracking_requested
if usage_tracking_requested() and context.turn_id:
configurable["evomemory_source_turn_id"] = context.turn_id
payload: BackgroundRunPayload = {
"assistant_id": _memory_worker_graph_id(context.source_type),
"input": messages_input(_memory_worker_user_prompt(context)),
"metadata": metadata,
"config": {
"configurable": {
"thread_id": thread_id,
"evomemory_source_session_id": context.session_id,
"evomemory_source_agent": context.source_agent,
"evomemory_project_id": context.project_id,
"evomemory_trajectory_digest": context.trajectory_digest,
}
},
"config": {"configurable": configurable},
}
return _runs_create_kwargs(payload)
EvoScientist/memory/source_context.py
+16
View File
@@ -40,6 +40,7 @@ class MemorySourceContext:
session_id: str
trajectory: list[CompactMessage]
trajectory_digest: str
turn_id: str | None = None
def _task_tool_call_ids(messages: list[BaseMessage]) -> set[str]:
@@ -203,6 +204,20 @@ def _runtime_thread_id(runtime: Runtime | None) -> str | None:
return None
def _active_turn_id() -> str | None:
"""Read the WebUI turn correlation value from the active run config."""
try:
from langgraph.config import get_config
metadata = get_config().get("metadata", {})
except (LookupError, RuntimeError):
return None
if not isinstance(metadata, dict):
return None
value = metadata.get("turn_id")
return value if isinstance(value, str) and value else None
def _short_hash(text: str) -> str:
"""Return the short hash fragment used in generated ids."""
return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16]
@@ -240,6 +255,7 @@ def build_memory_source_context(
project_id=project_id,
source_agent=source_agent,
session_id=session_id,
turn_id=_active_turn_id(),
trajectory=trajectory,
trajectory_digest=_trajectory_digest(trajectory),
)
EvoScientist/middleware/ask_user.py
+96 -23
View File
@@ -29,13 +29,46 @@ from langchain.agents.middleware.types import (
)
from langchain.tools import InjectedToolCallId
from langchain_core.messages import AIMessage, SystemMessage, ToolMessage
from langchain_core.tools import tool
from langchain_core.tools import BaseTool, tool
from langgraph.types import Command, interrupt
from pydantic import BeforeValidator, Field
from typing_extensions import TypedDict
logger = logging.getLogger(__name__)
_VALID_REVIEW_MODES = frozenset({"manual", "auto", "full"})
def _review_mode() -> str:
"""Return the per-run WebUI review mode, failing closed to Manual."""
try:
from langgraph.config import get_config
config = get_config()
except Exception:
return "manual"
if not isinstance(config, dict):
return "manual"
configurable = config.get("configurable") or {}
if not isinstance(configurable, dict):
return "manual"
mode = configurable.get("review_mode")
return mode if mode in _VALID_REVIEW_MODES else "manual"
def _tool_name(tool_value: BaseTool | dict[str, Any]) -> str | None:
if isinstance(tool_value, BaseTool):
return tool_value.name or None
name = tool_value.get("name")
if isinstance(name, str) and name:
return name
function = tool_value.get("function")
if isinstance(function, dict):
nested_name = function.get("name")
if isinstance(nested_name, str) and nested_name:
return nested_name
return None
# ---------------------------------------------------------------------------
# Data types
@@ -202,6 +235,16 @@ or available tools.
- Never ask more than once per decision point — respect the user's time
- After receiving answers, summarize what you understood before proceeding"""
FULL_APPROVE_SYSTEM_PROMPT = """\
You are running in Full approve mode. Do not wait for user clarification. When
information is missing, make reasonable, conservative assumptions, continue the
task, and report material assumptions in the final response."""
FULL_APPROVE_TOOL_MESSAGE = (
"Full approve mode: continue with reasonable assumptions and do not ask "
"the user again."
)
# ---------------------------------------------------------------------------
# Validation & parsing
@@ -344,6 +387,17 @@ def _parse_answers(
# ---------------------------------------------------------------------------
def _with_system_prompt(request: ModelRequest[ContextT], prompt: str) -> SystemMessage:
if request.system_message is not None:
content = [
*request.system_message.content_blocks,
{"type": "text", "text": f"\n\n{prompt}"},
]
else:
content = [{"type": "text", "text": prompt}]
return SystemMessage(content=cast("list[str | dict[str, str]]", content))
class AskUserMiddleware(AgentMiddleware[Any, ContextT, ResponseT]):
"""Middleware that provides an ``ask_user`` tool for interactive questioning.
@@ -369,6 +423,17 @@ class AskUserMiddleware(AgentMiddleware[Any, ContextT, ResponseT]):
tool_call_id: Annotated[str, InjectedToolCallId],
) -> Command[Any]:
"""Ask the user one or more questions."""
if _review_mode() == "full":
return Command(
update={
"messages": [
ToolMessage(
FULL_APPROVE_TOOL_MESSAGE,
tool_call_id=tool_call_id,
)
]
}
)
_validate_questions(questions)
ask_request = AskUserRequest(
type="ask_user",
@@ -386,18 +451,22 @@ class AskUserMiddleware(AgentMiddleware[Any, ContextT, ResponseT]):
request: ModelRequest[ContextT],
handler: Callable[[ModelRequest[ContextT]], ModelResponse[ResponseT]],
) -> ModelResponse[ResponseT] | AIMessage:
"""Inject the ask_user system prompt."""
if request.system_message is not None:
new_system_content = [
*request.system_message.content_blocks,
{"type": "text", "text": f"\n\n{self.system_prompt}"},
]
else:
new_system_content = [{"type": "text", "text": self.system_prompt}]
new_system_message = SystemMessage(
content=cast("list[str | dict[str, str]]", new_system_content)
"""Apply the interactive or unattended prompt and tool policy."""
if _review_mode() == "full":
tools = [tool for tool in request.tools if _tool_name(tool) != "ask_user"]
return handler(
request.override(
system_message=_with_system_prompt(
request, FULL_APPROVE_SYSTEM_PROMPT
),
tools=tools,
)
)
return handler(
request.override(
system_message=_with_system_prompt(request, self.system_prompt)
)
)
return handler(request.override(system_message=new_system_message))
async def awrap_model_call(
self,
@@ -406,15 +475,19 @@ class AskUserMiddleware(AgentMiddleware[Any, ContextT, ResponseT]):
[ModelRequest[ContextT]], Awaitable[ModelResponse[ResponseT]]
],
) -> ModelResponse[ResponseT] | AIMessage:
"""Inject the ask_user system prompt (async)."""
if request.system_message is not None:
new_system_content = [
*request.system_message.content_blocks,
{"type": "text", "text": f"\n\n{self.system_prompt}"},
]
else:
new_system_content = [{"type": "text", "text": self.system_prompt}]
new_system_message = SystemMessage(
content=cast("list[str | dict[str, str]]", new_system_content)
"""Apply the interactive or unattended prompt and tool policy (async)."""
if _review_mode() == "full":
tools = [tool for tool in request.tools if _tool_name(tool) != "ask_user"]
return await handler(
request.override(
system_message=_with_system_prompt(
request, FULL_APPROVE_SYSTEM_PROMPT
),
tools=tools,
)
)
return await handler(
request.override(
system_message=_with_system_prompt(request, self.system_prompt)
)
)
return await handler(request.override(system_message=new_system_message))
EvoScientist/middleware/configurable_model.py
+5 -2
View File
@@ -101,7 +101,7 @@ class ConfigurableModelMiddleware(AgentMiddleware):
def __init__(self) -> None:
super().__init__()
self._cache: dict[tuple[str, str | None], Any] = {}
self._cache: dict[tuple[str, str | None, str | None], Any] = {}
self._lock = threading.Lock()
# Track the last (model, provider) pair we INFO-logged so we only
# surface a banner on transition. Without this, every LLM call in a
@@ -130,7 +130,10 @@ class ConfigurableModelMiddleware(AgentMiddleware):
def _resolve(self, model: str, provider: str | None) -> Any:
"""Return a cached or freshly-built chat model for ``(model, provider)``."""
key = (model, provider)
from ..llm.models import get_model_runtime_revision
revision = get_model_runtime_revision(provider)
key = (model, provider, revision)
with self._lock:
cached = self._cache.get(key)
if cached is not None:
EvoScientist/middleware/scheduler.py
+41 -14
View File
@@ -13,12 +13,14 @@ from __future__ import annotations
import asyncio
import time
from collections.abc import Awaitable, Callable
from types import SimpleNamespace
from langchain.agents.middleware.types import (
AgentMiddleware,
ModelRequest,
ModelResponse,
)
from langchain.tools import ToolRuntime
from langchain_core.tools import tool
from .utils import append_to_system_message
@@ -47,8 +49,20 @@ manage them.
# ---------------------------------------------------------------------------
def _scope_for_runtime(runtime: ToolRuntime | None):
from ..workspace_scope import require_scoped_runtime
return require_scoped_runtime(runtime, kind="scheduler")
@tool
def schedule_task(name: str, cron: str, prompt: str, timezone: str = "") -> str:
def schedule_task(
name: str,
cron: str,
prompt: str,
timezone: str = "",
runtime: ToolRuntime = None,
) -> str:
"""Create a recurring scheduled task that runs unattended in the background.
Translate the user's natural-language timing into a standard 5-field cron
@@ -67,7 +81,11 @@ def schedule_task(name: str, cron: str, prompt: str, timezone: str = "") -> str:
return "Scheduler unavailable: the langgraph dev backend is not running."
try:
rec = crons.create_schedule(
name=name, schedule=cron, prompt=prompt, timezone=timezone or None
name=name,
schedule=cron,
prompt=prompt,
timezone=timezone or None,
scope=_scope_for_runtime(runtime),
)
except Exception as e:
return f"Error: {e}"
@@ -78,14 +96,14 @@ def schedule_task(name: str, cron: str, prompt: str, timezone: str = "") -> str:
@tool
def list_scheduled_tasks() -> str:
def list_scheduled_tasks(runtime: ToolRuntime = None) -> str:
"""List the user's recurring scheduled tasks (id, name, schedule, enabled)."""
from ..cron import schedule as crons
if not crons.is_available():
return "Scheduler unavailable: the langgraph dev backend is not running."
try:
rows = crons.list_schedules()
rows = crons.list_schedules(_scope_for_runtime(runtime))
except Exception as e:
return f"Error: {e}"
if not rows:
@@ -101,7 +119,7 @@ def list_scheduled_tasks() -> str:
@tool
def cancel_scheduled_task(cron_id: str) -> str:
def cancel_scheduled_task(cron_id: str, runtime: ToolRuntime = None) -> str:
"""Cancel (delete) a scheduled task. Pass the id (or its prefix) shown by list_scheduled_tasks."""
from ..cron import schedule as crons
@@ -111,7 +129,7 @@ def cancel_scheduled_task(cron_id: str) -> str:
# Empty prefix would match (and delete) the only cron — refuse it.
return "Provide the id (or a prefix) of the task to cancel."
try:
rows = crons.list_schedules()
rows = crons.list_schedules(_scope_for_runtime(runtime))
# B2: collect ALL prefix matches before acting to detect ambiguity.
matches = [
r for r in rows if str(r.get("cron_id", "")).startswith(requested_id)
@@ -142,11 +160,15 @@ class SchedulerMiddleware(AgentMiddleware):
def __init__(self) -> None:
super().__init__()
self._cache: str | None = None
self._cache_at: float = 0.0
self._cache: dict[str, tuple[float, str]] = {}
self.tools = [schedule_task, list_scheduled_tasks, cancel_scheduled_task]
def _schedules_block(self) -> str:
def _runtime_scope(self):
from langgraph.config import get_config
return _scope_for_runtime(SimpleNamespace(config=get_config()))
def _schedules_block(self, scope=None) -> str:
"""Build the dynamic ``<scheduled_tasks>`` block (empty if none / down)."""
from ..cron import schedule as crons
@@ -158,7 +180,7 @@ class SchedulerMiddleware(AgentMiddleware):
try:
if not crons.is_available():
return ""
rows = crons.list_schedules()
rows = crons.list_schedules(scope)
except Exception:
return ""
if not rows:
@@ -182,10 +204,15 @@ class SchedulerMiddleware(AgentMiddleware):
def _cached_schedules_block(self) -> str:
now = time.monotonic()
if self._cache is None or (now - self._cache_at) > _CACHE_TTL_SECONDS:
self._cache = self._schedules_block()
self._cache_at = now
return self._cache
try:
scope = self._runtime_scope()
except Exception:
return ""
key = f"{scope.scope_id}:{scope.revision}" if scope is not None else "legacy"
cached = self._cache.get(key)
if cached is None or (now - cached[0]) > _CACHE_TTL_SECONDS:
self._cache[key] = (now, self._schedules_block(scope))
return self._cache[key][1]
def _injection(self, schedules_block: str) -> str:
"""Static instructions, then the dynamic list (static→dynamic, like memory)."""
EvoScientist/middleware/tool_selector.py
+24 -1
View File
@@ -258,6 +258,29 @@ def create_tool_selector_middleware(
model = _ensure_chat_model()
safe_model = disable_thinking(model)
selector_model = safe_model
from EvoScientist.usage.callback import usage_tracking_enabled
if usage_tracking_enabled():
try:
selector_model = safe_model.model_copy(
update={
"metadata": {
**(safe_model.metadata or {}),
"usage_scope": "tool_selector",
}
}
)
if not isinstance(selector_model, BaseChatModel):
raise TypeError("selector model copy is not a BaseChatModel")
except Exception:
# The model factory preflights this path before enabling its callback.
# Keep the selector operational even for an unusual third-party model.
from EvoScientist.usage.spool import mark_tracking_degraded
mark_tracking_degraded("selector_model_copy_unsupported")
selector_model = safe_model
logger.exception("Could not attach Tool Selector usage scope")
system_prompt = (
"You are selecting tools for a scientific research agent. "
@@ -270,7 +293,7 @@ def create_tool_selector_middleware(
def selector_factory(always_include: list[str]) -> AgentMiddleware:
return LLMToolSelectorMiddleware(
model=safe_model,
model=selector_model,
system_prompt=system_prompt,
always_include=always_include,
)
EvoScientist/scope_registry.py
+1274
View File
File diff suppressed because it is too large Load Diff
EvoScientist/usage/__init__.py
+6
View File
@@ -0,0 +1,6 @@
"""Fail-open model usage capture for the integrated EvoScientist WebUI."""
from .callback import UsageModelIdentity, attach_usage_callback
from .identity import prepare_usage_environment
__all__ = ["UsageModelIdentity", "attach_usage_callback", "prepare_usage_environment"]
EvoScientist/usage/callback.py
+356
View File
@@ -0,0 +1,356 @@
"""LangChain terminal callback that emits one UsageEvent per real model call."""
from __future__ import annotations
import copy
import logging
import os
import threading
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any
from uuid import UUID
from langchain_core.callbacks import BaseCallbackHandler
from .schema import MAX_SAFE_TOKEN_INTEGER, UsageEventV1, UsageScope
from .spool import get_usage_spool, mark_tracking_degraded
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class UsageModelIdentity:
provider_profile_id: str
provider_revision: str | None
provider_adapter: str
model_alias: str
upstream_model_id: str
@dataclass
class _CallState:
started_at: datetime
parent_run_id: str | None
metadata: dict[str, Any] = field(default_factory=dict)
last_usage: dict[str, Any] | None = None
provider_request_id: str | None = None
def usage_tracking_requested() -> bool:
"""Return whether this process was launched in usage-tracking mode."""
enabled = os.getenv("EVOSCIENTIST_USAGE_TRACKING", "").strip().lower()
return enabled in {"1", "true", "yes", "on"}
def usage_tracking_enabled() -> bool:
"""Return whether the integrated launcher supplied a complete usage sink."""
return usage_tracking_requested() and all(
os.getenv(name, "").strip()
for name in (
"EVOSCIENTIST_USAGE_SINK_URL",
"EVOSCIENTIST_USAGE_SINK_TOKEN",
"EVOSCIENTIST_DEPLOYMENT_ID",
"EVOSCIENTIST_WORKSPACE_ID",
"EVOSCIENTIST_USAGE_SPOOL_DIR",
)
)
def _text(value: Any, *, limit: int = 256) -> str | None:
if value is None:
return None
rendered = "".join(
character
for character in str(value)
if ord(character) > 31 and ord(character) != 127
)
return rendered[:limit] if rendered else None
def _metadata_value(metadata: Mapping[str, Any], *keys: str) -> Any:
for key in keys:
value = metadata.get(key)
if value is not None:
return value
return None
def _scope(metadata: Mapping[str, Any]) -> UsageScope:
explicit = metadata.get("usage_scope")
allowed = {
"main", "sync_subagent", "async_subagent", "tool_selector", "summarizer",
"memory", "scheduler", "autoskills", "diagnostic", "skill_eval", "unattributed",
}
if explicit in allowed:
return explicit # type: ignore[return-value]
if metadata.get("lc_source") == "summarization":
return "summarizer"
run_kind = metadata.get("run_kind")
if run_kind == "scheduled_task":
return "scheduler"
if run_kind == "evomemory_autoskills":
return "autoskills"
if isinstance(run_kind, str) and run_kind.startswith("evomemory_"):
return "memory"
# deepagents attaches this stable metadata field to every compiled agent
# graph. Remote async runs carry an explicit usage_scope and have already
# returned above; remaining non-main names are synchronous subagents.
agent_name = metadata.get("lc_agent_name")
if agent_name == "EvoScientist":
return "main"
if isinstance(agent_name, str) and agent_name:
return "sync_subagent"
if metadata.get("async_task_id") or metadata.get("source_session_id"):
return "async_subagent"
if metadata.get("thread_id") or metadata.get("langgraph_thread_id"):
return "main"
return "unattributed"
def _clean_details(value: Any) -> dict[str, int]:
if not isinstance(value, Mapping):
return {}
result: dict[str, int] = {}
for key, count in value.items():
if (
isinstance(key, str)
and 0 < len(key) <= 128
and isinstance(count, int)
and not isinstance(count, bool)
and 0 <= count <= MAX_SAFE_TOKEN_INTEGER
):
result[key] = count
return result
def _normalize_usage(value: Any) -> dict[str, Any] | None:
if not isinstance(value, Mapping):
return None
input_tokens = value.get("input_tokens", value.get("prompt_tokens"))
output_tokens = value.get("output_tokens", value.get("completion_tokens"))
if not all(
isinstance(token, int)
and not isinstance(token, bool)
and 0 <= token <= MAX_SAFE_TOKEN_INTEGER
for token in (input_tokens, output_tokens)
):
return None
if input_tokens + output_tokens > MAX_SAFE_TOKEN_INTEGER:
return None
provider_total = value.get("total_tokens")
if not (
isinstance(provider_total, int)
and not isinstance(provider_total, bool)
and 0 <= provider_total <= MAX_SAFE_TOKEN_INTEGER
):
provider_total = None
return {
"input_tokens": input_tokens,
"output_tokens": output_tokens,
"provider_total_tokens": provider_total,
"input_token_details": _clean_details(value.get("input_token_details")),
"output_token_details": _clean_details(value.get("output_token_details")),
}
def _message_from_chunk(chunk: Any) -> Any:
return getattr(chunk, "message", chunk)
def _message_usage(message: Any) -> dict[str, Any] | None:
return _normalize_usage(getattr(message, "usage_metadata", None))
def _request_id(message: Any) -> str | None:
metadata = getattr(message, "response_metadata", None)
if not isinstance(metadata, Mapping):
return None
return _text(_metadata_value(metadata, "request_id", "id", "x_request_id"))
class UsageCaptureCallback(BaseCallbackHandler):
"""Capture final provider usage without influencing the model invocation."""
def __init__(self, identity: UsageModelIdentity) -> None:
self.identity = identity
self._lock = threading.Lock()
self._calls: dict[str, _CallState] = {}
def on_chat_model_start(
self,
serialized: dict[str, Any],
messages: list[list[Any]],
*,
run_id: UUID,
parent_run_id: UUID | None = None,
metadata: dict[str, Any] | None = None,
**kwargs: Any,
) -> None:
try:
with self._lock:
self._calls[str(run_id)] = _CallState(
started_at=datetime.now(UTC),
parent_run_id=str(parent_run_id) if parent_run_id else None,
metadata=dict(metadata or {}),
)
except Exception:
logger.exception("Usage callback could not initialize run state")
def on_llm_new_token(
self, token: Any, *, run_id: UUID, chunk: Any = None, **kwargs: Any
) -> None:
try:
message = _message_from_chunk(chunk)
usage = _message_usage(message)
if usage is None:
return
with self._lock:
state = self._calls.get(str(run_id))
if state:
state.last_usage = usage
state.provider_request_id = _request_id(message) or state.provider_request_id
except Exception:
logger.exception("Usage callback could not inspect a stream chunk")
def on_llm_end(self, response: Any, *, run_id: UUID, **kwargs: Any) -> None:
usage: dict[str, Any] | None = None
request_id: str | None = None
try:
for group in getattr(response, "generations", []) or []:
for generation in group or []:
message = getattr(generation, "message", generation)
usage = _message_usage(message) or usage
request_id = _request_id(message) or request_id
if usage is None:
output = getattr(response, "llm_output", None)
if isinstance(output, Mapping):
usage = _normalize_usage(output.get("token_usage") or output.get("usage"))
except Exception:
logger.exception("Usage callback could not inspect terminal usage")
try:
self._finish(str(run_id), usage=usage, request_id=request_id)
except Exception:
logger.exception("Usage callback could not emit terminal usage")
def on_llm_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None:
try:
self._finish(str(run_id), usage=None, request_id=None)
except Exception:
logger.exception("Usage callback could not emit unknown terminal usage")
def _finish(
self, run_id: str, *, usage: dict[str, Any] | None, request_id: str | None
) -> None:
with self._lock:
state = self._calls.pop(run_id, None)
if state is None:
return
usage = usage or state.last_usage
completed = datetime.now(UTC)
metadata = state.metadata
confirmed = usage is not None
event = UsageEventV1(
schema_version=1,
event_id=(
f"{os.environ['EVOSCIENTIST_DEPLOYMENT_ID']}:{run_id}:callback_final:1"
),
event_type="usage_observed",
source="callback_final",
authority_class="observed_final",
revision=1,
deployment_id=os.environ["EVOSCIENTIST_DEPLOYMENT_ID"],
workspace_id=os.environ["EVOSCIENTIST_WORKSPACE_ID"],
model_call_id=run_id,
parent_run_id=state.parent_run_id,
provider_request_id=request_id or state.provider_request_id,
thread_id=_text(
_metadata_value(metadata, "thread_id", "langgraph_thread_id")
),
source_session_id=_text(
_metadata_value(
metadata, "source_session_id", "evomemory_source_session_id"
)
),
turn_id=_text(
_metadata_value(metadata, "turn_id", "evomemory_source_turn_id")
),
workspace_dir=_text(
_metadata_value(metadata, "workspace_dir"), limit=4096
)
or _text(os.getenv("EVOSCIENTIST_WORKSPACE_DIR"), limit=4096),
scope=_scope(metadata),
source_agent=_text(
_metadata_value(metadata, "source_agent", "evomemory_source_agent")
),
provider_profile_id=_text(
self.identity.provider_profile_id, limit=512
)
or "unknown",
provider_revision=_text(self.identity.provider_revision),
provider_adapter=_text(self.identity.provider_adapter, limit=512)
or "unknown",
model_alias=_text(self.identity.model_alias, limit=512) or "unknown",
upstream_model_id=_text(
self.identity.upstream_model_id, limit=512
)
or "unknown",
usage_status="confirmed" if confirmed else "unknown",
input_tokens=usage["input_tokens"] if usage else None,
output_tokens=usage["output_tokens"] if usage else None,
provider_total_tokens=usage["provider_total_tokens"] if usage else None,
input_token_details=usage["input_token_details"] if usage else {},
output_token_details=usage["output_token_details"] if usage else {},
started_at=state.started_at,
observed_at=completed,
completed_at=completed,
)
get_usage_spool().enqueue(event)
def _merged_callbacks(existing: Any, callback: UsageCaptureCallback) -> Any:
if existing is None:
return [callback]
if isinstance(existing, (list, tuple)):
return [*existing, callback]
if hasattr(existing, "add_handler"):
manager = copy.copy(existing)
manager.add_handler(callback, inherit=True)
return manager
raise TypeError("unsupported callbacks value")
def attach_usage_callback(model: Any, identity: UsageModelIdentity) -> Any:
"""Return a callback-enabled model copy, or the original model on degradation."""
if not usage_tracking_enabled():
return model
try:
from langchain_core.language_models import BaseChatModel
if not isinstance(model, BaseChatModel):
raise TypeError("model is not a BaseChatModel")
callback = UsageCaptureCallback(identity)
updated = model.model_copy(
update={"callbacks": _merged_callbacks(model.callbacks, callback)}
)
if not isinstance(updated, BaseChatModel):
raise TypeError("callback model_copy did not return a BaseChatModel")
# Tool Selector depends on a second metadata-only model copy. Verify the
# contract before enabling tracking for this model so a selector call
# can never be silently counted as a main-agent call.
probe = updated.model_copy(
update={"metadata": {**(updated.metadata or {}), "usage_scope": "tool_selector"}}
)
if not isinstance(probe, BaseChatModel):
raise TypeError("selector metadata model_copy did not return a BaseChatModel")
# Start capability negotiation and heartbeat at backend startup rather
# than waiting for the first completed model call. This lets the UI
# distinguish a healthy empty database from unsupported tracking.
get_usage_spool()
return updated
except Exception:
mark_tracking_degraded("selector_model_copy_unsupported")
logger.exception("Usage callback injection failed; model behavior is unchanged")
return model
EvoScientist/usage/identity.py
+122
View File
@@ -0,0 +1,122 @@
"""Stable deployment/workspace identity and launcher environment setup."""
from __future__ import annotations
import hashlib
import ntpath
import os
import secrets
import time
import unicodedata
import uuid
from collections.abc import Callable
from pathlib import Path
def _fsync_directory(path: Path) -> None:
try:
descriptor = os.open(path, os.O_RDONLY)
except OSError:
return
try:
os.fsync(descriptor)
except OSError:
pass
finally:
os.close(descriptor)
def _load_or_create(path: Path, factory: Callable[[], str]) -> str:
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
value = factory()
try:
descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
except FileExistsError:
# Another launcher may have won O_EXCL but not completed its fsync yet.
for _ in range(100):
existing = path.read_text(encoding="utf-8").strip()
if existing:
return existing
time.sleep(0.01)
raise ValueError(f"Identity file is empty: {path}") from None
try:
os.write(descriptor, f"{value}\n".encode())
os.fsync(descriptor)
finally:
os.close(descriptor)
_fsync_directory(path.parent)
return value
def resolve_data_dir() -> Path:
configured = os.getenv("EVOSCIENTIST_DATA_DIR", "").strip()
if configured:
path = Path(configured)
if not path.is_absolute():
raise ValueError("EVOSCIENTIST_DATA_DIR must be absolute")
return path
return Path.home() / ".evoscientist"
def normalize_workspace_path_v1(
resolved_path: str, *, windows: bool | None = None
) -> str:
"""Normalize an already-real path using the frozen ws1 cross-platform rules."""
use_windows = os.name == "nt" if windows is None else windows
normalized = unicodedata.normalize("NFC", resolved_path)
if use_windows:
windows_path = ntpath.normcase(normalized)
_drive, tail = ntpath.splitdrive(windows_path)
normalized = windows_path.replace("\\", "/")
if tail in {"\\", "/"}:
return normalized.rstrip("/") + "/"
return normalized.rstrip("/")
if normalized == "/":
return normalized
return normalized.rstrip("/")
def workspace_identity_from_normalized(deployment_id: str, normalized_path: str) -> str:
"""Hash a normalized path; split out for cross-language contract fixtures."""
digest = hashlib.sha256(f"{deployment_id}\0{normalized_path}".encode()).hexdigest()
return f"ws1_{digest}"
def workspace_identity(deployment_id: str, workspace_dir: str | Path) -> str:
resolved = os.path.realpath(os.path.expanduser(str(workspace_dir)), strict=True)
normalized = normalize_workspace_path_v1(resolved)
return workspace_identity_from_normalized(deployment_id, normalized)
def prepare_usage_environment(
workspace_dir: str | Path, *, webui_port: int
) -> dict[str, str]:
"""Create stable secrets/identities and return the shared child environment."""
if not 1 <= webui_port <= 65535:
raise ValueError("webui_port must be an integer in [1, 65535]")
data_dir = resolve_data_dir().expanduser().resolve()
data_dir.mkdir(parents=True, exist_ok=True, mode=0o700)
deployment_id = _load_or_create(
data_dir / "deployment-id", lambda: str(uuid.uuid4())
)
# Reject corrupt identity files early instead of silently creating a second identity.
deployment_id = str(uuid.UUID(deployment_id))
sink_token = _load_or_create(
data_dir / "usage-sink-token", lambda: secrets.token_urlsafe(32)
)
workspace_id = workspace_identity(deployment_id, workspace_dir)
return {
"EVOSCIENTIST_DATA_DIR": str(data_dir),
"EVOSCIENTIST_USAGE_TRACKING": "true",
"EVOSCIENTIST_USAGE_SINK_URL": (
f"http://127.0.0.1:{webui_port}/api/usage/events"
),
"EVOSCIENTIST_USAGE_SINK_TOKEN": sink_token,
"EVOSCIENTIST_DEPLOYMENT_ID": deployment_id,
"EVOSCIENTIST_WORKSPACE_ID": workspace_id,
"EVOSCIENTIST_USAGE_SPOOL_DIR": str(data_dir / "usage-spool"),
}
EvoScientist/usage/schema.py
+170
View File
@@ -0,0 +1,170 @@
"""UsageEvent v1 model shared by the callback and durable spool."""
from __future__ import annotations
import re
from datetime import datetime
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, model_validator
MAX_SAFE_TOKEN_INTEGER = 9_007_199_254_740_991
_UTC_TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$")
UsageScope = Literal[
"main",
"sync_subagent",
"async_subagent",
"tool_selector",
"summarizer",
"memory",
"scheduler",
"autoskills",
"diagnostic",
"skill_eval",
"unattributed",
]
class UsageEventV1(BaseModel):
"""Validated terminal observation for exactly one LangChain model run."""
model_config = ConfigDict(extra="forbid")
schema_version: Literal[1]
event_id: str = Field(min_length=1, max_length=1024)
event_type: Literal["usage_observed"]
source: Literal["callback_final"]
authority_class: Literal["observed_final"]
revision: Literal[1]
deployment_id: str = Field(min_length=1, max_length=256)
workspace_id: str = Field(min_length=1, max_length=256)
model_call_id: str = Field(min_length=1, max_length=256)
parent_run_id: str | None = Field(min_length=1, max_length=256)
provider_request_id: str | None = Field(min_length=1, max_length=256)
thread_id: str | None = Field(min_length=1, max_length=256)
source_session_id: str | None = Field(min_length=1, max_length=256)
turn_id: str | None = Field(min_length=1, max_length=256)
workspace_dir: str | None = Field(min_length=1, max_length=4096)
scope: UsageScope
source_agent: str | None = Field(min_length=1, max_length=256)
provider_profile_id: str = Field(min_length=1, max_length=512)
provider_revision: str | None = Field(min_length=1, max_length=256)
provider_adapter: str = Field(min_length=1, max_length=512)
model_alias: str = Field(min_length=1, max_length=512)
upstream_model_id: str = Field(min_length=1, max_length=512)
usage_status: Literal["confirmed", "unknown"]
input_tokens: int | None = Field(ge=0, le=MAX_SAFE_TOKEN_INTEGER)
output_tokens: int | None = Field(ge=0, le=MAX_SAFE_TOKEN_INTEGER)
provider_total_tokens: int | None = Field(ge=0, le=MAX_SAFE_TOKEN_INTEGER)
input_token_details: dict[str, int]
output_token_details: dict[str, int]
started_at: datetime | None
observed_at: datetime
completed_at: datetime
@model_validator(mode="before")
@classmethod
def validate_json_types(cls, value: object) -> object:
if not isinstance(value, dict):
return value
integer_fields = (
"schema_version",
"revision",
"input_tokens",
"output_tokens",
"provider_total_tokens",
)
for name in integer_fields:
item = value.get(name)
if item is not None and (
not isinstance(item, int) or isinstance(item, bool)
):
raise ValueError(f"{name} must be an integer or null")
for name in ("input_token_details", "output_token_details"):
details = value.get(name)
if isinstance(details, dict) and any(
not isinstance(item, int) or isinstance(item, bool)
for item in details.values()
):
raise ValueError(f"{name} values must be integers")
for name in ("started_at", "observed_at", "completed_at"):
timestamp = value.get(name)
if timestamp is not None and not isinstance(timestamp, (str, datetime)):
raise ValueError(f"{name} must be an RFC 3339 UTC timestamp")
if isinstance(timestamp, str) and not _UTC_TIMESTAMP.fullmatch(timestamp):
raise ValueError(f"{name} must be an RFC 3339 UTC timestamp")
return value
@model_validator(mode="after")
def validate_contract(self) -> UsageEventV1:
expected = f"{self.deployment_id}:{self.model_call_id}:callback_final:1"
if self.event_id != expected:
raise ValueError("event_id does not match the v1 identity")
if self.usage_status == "unknown":
if any(
value is not None
for value in (
self.input_tokens,
self.output_tokens,
self.provider_total_tokens,
)
):
raise ValueError("unknown usage must have null token fields")
else:
if self.input_tokens is None or self.output_tokens is None:
raise ValueError("confirmed usage requires input and output tokens")
if self.input_tokens + self.output_tokens > MAX_SAFE_TOKEN_INTEGER:
raise ValueError("input plus output tokens exceeds safe integer")
for details in (self.input_token_details, self.output_token_details):
if any(
not key
or len(key) > 128
or any(
ord(character) <= 31 or ord(character) == 127 for character in key
)
or value < 0
or value > MAX_SAFE_TOKEN_INTEGER
for key, value in details.items()
):
raise ValueError("invalid token details")
for name in (
"event_id",
"deployment_id",
"workspace_id",
"model_call_id",
"parent_run_id",
"provider_request_id",
"thread_id",
"source_session_id",
"turn_id",
"workspace_dir",
"source_agent",
"provider_profile_id",
"provider_revision",
"provider_adapter",
"model_alias",
"upstream_model_id",
):
text = getattr(self, name)
if text is not None and any(
ord(character) <= 31 or ord(character) == 127 for character in text
):
raise ValueError(f"{name} contains a control character")
for timestamp in (self.started_at, self.observed_at, self.completed_at):
if timestamp is not None:
offset = timestamp.utcoffset()
if (
timestamp.tzinfo is None
or offset is None
or offset.total_seconds() != 0
):
raise ValueError("usage timestamps must be UTC")
if self.started_at and (
self.started_at > self.observed_at or self.started_at > self.completed_at
):
raise ValueError("started_at cannot follow terminal timestamps")
return self
EvoScientist/usage/spool.py
+540
View File
@@ -0,0 +1,540 @@
"""Local durable outbox and background sender for UsageEvent v1."""
from __future__ import annotations
import atexit
import hashlib
import json
import logging
import os
import random
import threading
import time
import uuid
from datetime import UTC, datetime
from pathlib import Path
from urllib.parse import urlsplit, urlunsplit
import httpx
from .schema import UsageEventV1
logger = logging.getLogger(__name__)
def _now() -> str:
return datetime.now(UTC).isoformat().replace("+00:00", "Z")
def _int_env(name: str, default: int, minimum: int = 1) -> int:
try:
return max(minimum, int(os.getenv(name, str(default))))
except ValueError:
return default
def _float_env(name: str, default: float) -> float:
try:
return max(0.05, float(os.getenv(name, str(default))))
except ValueError:
return default
def _fsync_dir(path: Path) -> None:
try:
descriptor = os.open(path, os.O_RDONLY)
except OSError:
return
try:
os.fsync(descriptor)
except OSError:
pass
finally:
os.close(descriptor)
def _endpoint(sink_url: str, suffix: str) -> str:
parts = urlsplit(sink_url)
base = parts.path.removesuffix("/api/usage/events")
return urlunsplit((parts.scheme, parts.netloc, f"{base}{suffix}", "", ""))
class UsageSpool:
"""Synchronous durable enqueue with an asynchronous at-least-once sender."""
def __init__(self) -> None:
self.sink_url = os.environ["EVOSCIENTIST_USAGE_SINK_URL"]
self.token = os.environ["EVOSCIENTIST_USAGE_SINK_TOKEN"]
self.deployment_id = os.environ["EVOSCIENTIST_DEPLOYMENT_ID"]
self.workspace_id = os.environ["EVOSCIENTIST_WORKSPACE_ID"]
self.root = Path(os.environ["EVOSCIENTIST_USAGE_SPOOL_DIR"])
if not self.root.is_absolute():
raise ValueError("EVOSCIENTIST_USAGE_SPOOL_DIR must be absolute")
self.tmp = self.root / "tmp"
self.pending = self.root / "pending"
self.inflight = self.root / "inflight"
self.quarantine = self.root / "quarantine"
for directory in (self.tmp, self.pending, self.inflight, self.quarantine):
directory.mkdir(parents=True, exist_ok=True, mode=0o700)
self.status_path = self.root / "status.json"
self.max_files = _int_env("EVOSCIENTIST_USAGE_SPOOL_MAX_FILES", 100_000)
self.max_bytes = _int_env("EVOSCIENTIST_USAGE_SPOOL_MAX_BYTES", 1_073_741_824)
self.max_event_bytes = _int_env("EVOSCIENTIST_USAGE_MAX_EVENT_BYTES", 262_144)
self.lease_seconds = _int_env("EVOSCIENTIST_USAGE_INFLIGHT_LEASE_SECONDS", 120)
self.heartbeat_interval = _float_env(
"EVOSCIENTIST_USAGE_HEARTBEAT_INTERVAL_SECONDS", 15
)
self.retry_initial = _float_env("EVOSCIENTIST_USAGE_RETRY_INITIAL_SECONDS", 1)
self.retry_max = _float_env("EVOSCIENTIST_USAGE_RETRY_MAX_SECONDS", 60)
self.connect_timeout = _float_env(
"EVOSCIENTIST_USAGE_HTTP_CONNECT_TIMEOUT_SECONDS", 1
)
self.request_timeout = _float_env("EVOSCIENTIST_USAGE_HTTP_TIMEOUT_SECONDS", 3)
self.unsupported_reprobe = _float_env(
"EVOSCIENTIST_USAGE_UNSUPPORTED_REPROBE_SECONDS", 300
)
self.schema_reprobe = _float_env(
"EVOSCIENTIST_USAGE_SCHEMA_REPROBE_SECONDS", 60
)
self.quarantine_retention_days = _int_env(
"EVOSCIENTIST_USAGE_QUARANTINE_RETENTION_DAYS", 90
)
self._state_lock = threading.Lock()
self._size_lock = threading.Lock()
self._cached_files = 0
self._cached_bytes = 0
self._size_cache_at = 0.0
self.first_loss_at: str | None = None
self.degraded_reason: str | None = None
self.last_error_code: str | None = None
self._load_status()
self._clean_expired_quarantine()
self._clean_stale_local_artifacts()
self._last_cleanup_at = time.monotonic()
self._stop = threading.Event()
self._wake = threading.Event()
self._thread = threading.Thread(
target=self._run, name="evoscientist-usage-sender", daemon=True
)
self._thread.start()
def _load_status(self) -> None:
try:
data = json.loads(self.status_path.read_text(encoding="utf-8"))
self.first_loss_at = data.get("first_loss_at")
self.degraded_reason = data.get("tracking_degraded_reason")
except (OSError, ValueError, TypeError):
pass
def _persist_status(self) -> None:
temporary = self.root / f"status.{os.getpid()}.{uuid.uuid4().hex}.tmp"
payload = json.dumps(
{
"first_loss_at": self.first_loss_at,
"tracking_degraded_reason": self.degraded_reason,
},
sort_keys=True,
separators=(",", ":"),
).encode()
try:
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
try:
os.write(descriptor, payload)
os.fsync(descriptor)
finally:
os.close(descriptor)
os.replace(temporary, self.status_path)
_fsync_dir(self.root)
except OSError:
logger.exception("Could not persist usage tracking degraded status")
try:
temporary.unlink(missing_ok=True)
except OSError:
pass
def mark_degraded(self, reason: str) -> None:
with self._state_lock:
if self.first_loss_at is None:
self.first_loss_at = _now()
if self.degraded_reason is None:
self.degraded_reason = reason[:512]
self._persist_status()
self._wake.set()
def _spool_size(self) -> tuple[int, int]:
with self._size_lock:
if time.monotonic() - self._size_cache_at < 1:
return self._cached_files, self._cached_bytes
count = 0
size = 0
for directory in (self.pending, self.inflight, self.quarantine):
try:
for item in os.scandir(directory):
if item.is_file() and item.name.endswith(".json"):
count += 1
try:
size += item.stat().st_size
except OSError:
pass
except OSError:
continue
with self._size_lock:
self._cached_files = count
self._cached_bytes = size
self._size_cache_at = time.monotonic()
return count, size
def _adjust_spool_size(self, count: int, size: int) -> None:
with self._size_lock:
self._cached_files = max(0, self._cached_files + count)
self._cached_bytes = max(0, self._cached_bytes + size)
def enqueue(self, event: UsageEventV1) -> None:
try:
payload = json.dumps(
event.model_dump(mode="json"),
ensure_ascii=False,
sort_keys=True,
separators=(",", ":"),
).encode("utf-8")
if len(payload) > self.max_event_bytes:
self.mark_degraded("event_too_large")
logger.error("Usage event %s exceeds spool event limit", event.event_id)
return
count, size = self._spool_size()
if count >= self.max_files or size + len(payload) > self.max_bytes:
self.mark_degraded("spool_soft_limit_reached")
logger.error("Usage spool soft limit reached; event was not persisted")
return
key = hashlib.sha256(event.event_id.encode()).hexdigest()
temporary = self.tmp / f"{key}.{os.getpid()}.{uuid.uuid4().hex}.tmp"
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
try:
os.write(descriptor, payload)
os.fsync(descriptor)
finally:
os.close(descriptor)
lock_path = self.root / f"{key}.lock"
deadline = time.monotonic() + 0.05
while True:
try:
lock_fd = os.open(
lock_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600
)
break
except FileExistsError:
if time.monotonic() < deadline:
time.sleep(0.001)
continue
os.replace(
temporary,
self.quarantine
/ f"{key}.lock-contention.{uuid.uuid4().hex}.json",
)
self._adjust_spool_size(1, len(payload))
_fsync_dir(self.quarantine)
logger.error(
"Usage event %s quarantined after spool lock contention",
event.event_id,
)
self.mark_degraded("spool_lock_contention")
self._wake.set()
return
try:
target = self.pending / f"{key}.json"
existing = target if target.exists() else self.inflight / target.name
if existing.exists():
if existing.read_bytes() != payload:
os.replace(
temporary,
self.quarantine / f"{key}.conflict.{uuid.uuid4().hex}.json",
)
self._adjust_spool_size(1, len(payload))
_fsync_dir(self.quarantine)
self.mark_degraded("local_event_payload_conflict")
else:
temporary.unlink(missing_ok=True)
else:
os.replace(temporary, target)
self._adjust_spool_size(1, len(payload))
_fsync_dir(self.pending)
finally:
os.close(lock_fd)
lock_path.unlink(missing_ok=True)
except Exception:
self.mark_degraded("spool_write_failed")
logger.exception("Usage capture could not persist a terminal event")
finally:
self._wake.set()
def _recover_stale_inflight(self) -> None:
cutoff = time.time() - self.lease_seconds
try:
items = list(self.inflight.glob("*.json"))
except OSError:
return
for item in items:
try:
if item.stat().st_mtime < cutoff:
os.replace(item, self.pending / item.name)
_fsync_dir(self.inflight)
_fsync_dir(self.pending)
except (FileNotFoundError, OSError):
continue
def _clean_expired_quarantine(self) -> None:
cutoff = time.time() - self.quarantine_retention_days * 86_400
removed = 0
for item in self.quarantine.glob("*.json"):
try:
if item.stat().st_mtime < cutoff:
item.unlink()
removed += 1
except (FileNotFoundError, OSError):
continue
if removed:
logger.warning("Removed %d expired usage quarantine events", removed)
_fsync_dir(self.quarantine)
def _clean_stale_local_artifacts(self) -> None:
cutoff = time.time() - self.lease_seconds
candidates = [*self.tmp.glob("*.tmp"), *self.root.glob("*.lock")]
for item in candidates:
try:
if item.stat().st_mtime < cutoff:
item.unlink()
except (FileNotFoundError, OSError):
continue
def _counts(self) -> tuple[int, int, int, int]:
def files(directory: Path) -> list[Path]:
try:
return list(directory.glob("*.json"))
except OSError:
return []
pending = files(self.pending)
inflight = files(self.inflight)
quarantine = files(self.quarantine)
size = 0
for item in pending + inflight + quarantine:
try:
size += item.stat().st_size
except OSError:
pass
return len(pending), len(inflight), len(quarantine), size
def _heartbeat(self, client: httpx.Client) -> None:
pending, inflight, quarantine, size = self._counts()
with self._state_lock:
body = {
"deployment_id": self.deployment_id,
"workspace_id": self.workspace_id,
"emitter_version": "2.0",
"schema_version": 1,
"sender_status": (
"degraded"
if self.first_loss_at or self.degraded_reason
else "healthy"
),
"spool_pending": pending,
"spool_inflight": inflight,
"spool_quarantined": quarantine,
"spool_bytes": size,
"first_loss_at": self.first_loss_at,
"tracking_degraded_reason": self.degraded_reason,
"last_error_code": self.last_error_code,
"sent_at": _now(),
}
try:
response = client.post(
_endpoint(self.sink_url, "/api/usage/sources/heartbeat"), json=body
)
if response.status_code >= 400:
self.last_error_code = f"heartbeat_http_{response.status_code}"
except httpx.HTTPError:
self.last_error_code = "heartbeat_unreachable"
def _capable(self, client: httpx.Client) -> bool:
try:
response = client.get(_endpoint(self.sink_url, "/api/usage/capabilities"))
if response.status_code != 200:
self.last_error_code = f"capabilities_http_{response.status_code}"
return False
versions = response.json().get("supported_schema_versions", [])
if 1 not in versions:
self.last_error_code = "schema_incompatible"
return False
self.last_error_code = None
return True
except (httpx.HTTPError, ValueError, TypeError):
self.last_error_code = "collector_unreachable"
return False
def _send_one(self, client: httpx.Client) -> bool:
pending: Path | None = None
lock_fd: int | None = None
lock_path: Path | None = None
try:
candidates = self.pending.glob("*.json")
for candidate in candidates:
candidate_lock = self.root / f"{candidate.stem}.lock"
try:
descriptor = os.open(
candidate_lock,
os.O_WRONLY | os.O_CREAT | os.O_EXCL,
0o600,
)
except FileExistsError:
continue
pending = candidate
lock_fd = descriptor
lock_path = candidate_lock
break
except OSError:
return False
if pending is None:
return False
inflight = self.inflight / pending.name
try:
os.replace(pending, inflight)
_fsync_dir(self.pending)
_fsync_dir(self.inflight)
except (FileNotFoundError, OSError):
return True
finally:
if lock_fd is not None:
os.close(lock_fd)
if lock_path is not None:
lock_path.unlink(missing_ok=True)
try:
response = client.post(
self.sink_url,
content=inflight.read_bytes(),
headers={"Content-Type": "application/json"},
)
if response.status_code == 200:
status = response.json().get("status")
if status in {"accepted", "duplicate"}:
event_size = inflight.stat().st_size
inflight.unlink(missing_ok=True)
self._adjust_spool_size(-1, -event_size)
_fsync_dir(self.inflight)
self.last_error_code = None
return True
if response.status_code in {400, 409, 413, 422}:
os.replace(inflight, self.quarantine / inflight.name)
_fsync_dir(self.inflight)
_fsync_dir(self.quarantine)
self.mark_degraded(f"collector_rejected_event_{response.status_code}")
self.last_error_code = f"event_http_{response.status_code}"
return True
self.last_error_code = f"event_http_{response.status_code}"
except (httpx.HTTPError, OSError, ValueError, TypeError):
self.last_error_code = "event_send_failed"
try:
os.replace(inflight, self.pending / inflight.name)
_fsync_dir(self.inflight)
_fsync_dir(self.pending)
except (FileNotFoundError, OSError):
pass
return False
def _run(self) -> None:
headers = {"Authorization": f"Bearer {self.token}"}
timeout = httpx.Timeout(self.request_timeout, connect=self.connect_timeout)
retry = self.retry_initial
last_heartbeat = 0.0
collector_capable = False
next_probe = 0.0
with httpx.Client(headers=headers, timeout=timeout) as client:
while not self._stop.is_set():
self._recover_stale_inflight()
now = time.monotonic()
if now - self._last_cleanup_at >= 3_600:
self._clean_expired_quarantine()
self._clean_stale_local_artifacts()
self._last_cleanup_at = now
if now - last_heartbeat >= self.heartbeat_interval:
self._heartbeat(client)
last_heartbeat = now
if not collector_capable and now >= next_probe:
collector_capable = self._capable(client)
if not collector_capable:
if self.last_error_code == "capabilities_http_404":
probe_delay = self.unsupported_reprobe
elif self.last_error_code == "schema_incompatible":
probe_delay = self.schema_reprobe
else:
probe_delay = min(self.retry_max, retry)
next_probe = time.monotonic() + probe_delay
if not collector_capable:
wait_for = max(0.05, next_probe - time.monotonic())
self._wake.wait(wait_for)
self._wake.clear()
retry = min(self.retry_max, retry * 2)
continue
progressed = self._send_one(client)
if progressed:
retry = self.retry_initial
continue
if self.last_error_code in {
"event_http_401",
"event_http_403",
"event_http_404",
"event_http_426",
}:
collector_capable = False
next_probe = time.monotonic() + (
self.unsupported_reprobe
if self.last_error_code == "event_http_404"
else self.schema_reprobe
)
self._wake.wait(
self.heartbeat_interval
if not any(self.pending.glob("*.json"))
else retry * random.uniform(0.8, 1.2)
)
self._wake.clear()
retry = min(self.retry_max, retry * 2)
def close(self) -> None:
self._stop.set()
self._wake.set()
if self._thread.is_alive():
self._thread.join(timeout=1)
_singleton_lock = threading.Lock()
_singleton: tuple[int, UsageSpool] | None = None
def get_usage_spool() -> UsageSpool:
global _singleton
pid = os.getpid()
with _singleton_lock:
if _singleton is None or _singleton[0] != pid:
_singleton = (pid, UsageSpool())
return _singleton[1]
def mark_tracking_degraded(reason: str) -> None:
if os.getenv("EVOSCIENTIST_USAGE_TRACKING", "").strip().lower() not in {
"1",
"true",
"yes",
"on",
}:
return
try:
get_usage_spool().mark_degraded(reason)
except Exception:
logger.exception("Could not mark usage tracking as degraded: %s", reason)
def _close_singleton() -> None:
if _singleton is not None:
_singleton[1].close()
atexit.register(_close_singleton)
EvoScientist/workspace_cutover.py
+417
View File
@@ -0,0 +1,417 @@
"""Administrative migration from a shared workspace to conversation scopes."""
from __future__ import annotations
import hashlib
import json
import os
import uuid
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from .scope_registry import (
ScopeRegistry,
deployment_id_for_workspace,
get_scope_registry,
)
from .workspace_scope import provision_conversation_scope, workspace_metadata
_TERMINAL_RUN_STATUSES = frozenset(
{"success", "error", "timeout", "cancelled", "interrupted"}
)
_VALID_SCOPE_STATES = frozenset({"draft", "active"})
def _now() -> str:
return datetime.now(UTC).isoformat()
def _write_report(workspace_root: Path, report: dict[str, Any]) -> Path:
reports = workspace_root / ".evoscientist" / "control" / "cutover-reports"
reports.mkdir(mode=0o700, parents=True, exist_ok=True)
body = dict(report)
digest = hashlib.sha256(
json.dumps(body, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
report["sha256"] = digest
encoded = json.dumps(report, sort_keys=True, indent=2).encode("utf-8") + b"\n"
target = reports / f"{report['operation_id']}.json"
for destination in (target, reports / "latest.json"):
temporary = destination.with_suffix(".json.tmp")
fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
with os.fdopen(fd, "wb") as file:
file.write(encoded)
os.replace(temporary, destination)
return target
def _active_run_ids(client: Any, thread_id: str) -> list[str]:
return [
str(run.get("run_id"))
for run in client.runs.list(thread_id=thread_id, limit=1000)
if str(run.get("status")) not in _TERMINAL_RUN_STATUSES
]
def _scope_owner_error(
registry: ScopeRegistry,
*,
deployment_id: str,
resource_id: str,
metadata: dict[str, Any],
allowed_owner_types: frozenset[str] | None = None,
) -> str | None:
"""Return a reason unless metadata names a live Registry-owned resource."""
scope_id = metadata.get("workspace_scope_id")
if not isinstance(scope_id, str) or not scope_id:
return "missing workspace_scope_id"
try:
scope = registry.get(deployment_id, scope_id)
owner = registry.get_owner_by_resource(deployment_id, resource_id)
except Exception as exc:
return str(exc)
if scope.state not in _VALID_SCOPE_STATES:
return f"workspace scope is {scope.state}"
if owner.scope_id != scope.scope_id:
return "resource owner belongs to another workspace scope"
if owner.state != "active":
return f"resource owner is {owner.state}"
if owner.owner_type in {"primary_thread", "primary_run"}:
return f"unexpected owner type {owner.owner_type}"
if allowed_owner_types is not None and owner.owner_type not in allowed_owner_types:
return f"unexpected owner type {owner.owner_type}"
metadata_deployment = metadata.get("workspace_deployment_id")
if (
metadata_deployment is not None
and metadata_deployment != ""
and metadata_deployment != deployment_id
):
return "metadata deployment does not match the active deployment"
metadata_owner = metadata.get("workspace_scope_owner_id")
if (
metadata_owner is not None
and metadata_owner != ""
and metadata_owner != owner.owner_id
):
return "metadata owner does not match the resource owner"
return None
def _quarantine_derived_thread(
client: Any,
*,
thread_id: str,
metadata: dict[str, Any],
operation_id: str,
reason: str,
report: dict[str, Any],
) -> None:
"""Interrupt a non-primary thread and remove any untrusted scope claims."""
try:
active_runs = _active_run_ids(client, thread_id)
for run_id in active_runs:
client.runs.cancel(thread_id, run_id, wait=True, action="interrupt")
remaining_active = _active_run_ids(client, thread_id)
if remaining_active:
report["active_runs"].append(
{"thread_id": thread_id, "run_ids": remaining_active}
)
report["unmanaged_derived_threads"].append(thread_id)
return
cleaned_metadata = {
key: value
for key, value in metadata.items()
if not key.startswith("workspace_")
}
client.threads.update(
thread_id,
metadata={
**cleaned_metadata,
"workspace_quarantine": {
"operation_id": operation_id,
"reason": reason,
"quarantined_at": _now(),
},
},
)
report["quarantined_derived_threads"].append(
{
"thread_id": thread_id,
"cancelled_run_ids": active_runs,
"reason": reason,
}
)
except Exception as exc:
report["unmanaged_derived_threads"].append(thread_id)
report["quarantine_failures"].append(
{"thread_id": thread_id, "error": str(exc)}
)
def verify_required_cutover(workspace_root: Path) -> None:
"""Reject strict mode unless the current deployment has a passing report."""
latest = (
workspace_root / ".evoscientist" / "control" / "cutover-reports" / "latest.json"
)
try:
report = json.loads(latest.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise RuntimeError(
"required workspace isolation needs a passing workspace-cutover report"
) from exc
digest = report.pop("sha256", None)
expected = hashlib.sha256(
json.dumps(report, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
if (
digest != expected
or report.get("status") != "passed"
or report.get("browser_sdk_gate") is not True
or report.get("deployment_id") != deployment_id_for_workspace(workspace_root)
):
raise RuntimeError("workspace-cutover report is missing, stale, or failed")
operation = get_scope_registry(workspace_root).get_operation(
str(report["deployment_id"]), str(report["operation_id"])
)
if (
operation.kind != "workspace-cutover"
or operation.state != "completed"
or operation.result_sha256 != digest
):
raise RuntimeError("workspace-cutover registry operation does not match report")
def run_workspace_cutover(
*,
workspace_root: Path,
client: Any,
assistant_id: str = "EvoScientist",
browser_sdk_gate: bool = False,
) -> dict[str, Any]:
"""Provision primary threads and quarantine legacy unscoped work."""
workspace_root = workspace_root.expanduser().resolve()
deployment_id = deployment_id_for_workspace(workspace_root)
registry: ScopeRegistry = get_scope_registry(workspace_root)
operation_id = str(uuid.uuid4())
registry.acquire_lock(
deployment_id, "workspace-lifecycle", operation_id, lease_seconds=300
)
try:
registry.acquire_lock(
deployment_id, "workspace-cutover", operation_id, lease_seconds=300
)
except Exception:
registry.release_lock(deployment_id, "workspace-lifecycle", operation_id)
raise
registry.begin_operation(deployment_id, operation_id, kind="workspace-cutover")
report: dict[str, Any] = {
"operation_id": operation_id,
"deployment_id": deployment_id,
"started_at": _now(),
"primary_threads": 0,
"scopes": 0,
"quarantined_crons": 0,
"quarantined_cron_records": [],
"quarantined_derived_threads": [],
"validated_derived_threads": [],
"validated_crons": [],
"invalid_scoped_derived_threads": [],
"invalid_scoped_crons": [],
"active_runs": [],
"unmanaged_derived_threads": [],
"quarantine_failures": [],
"metadata_mismatches": [],
"browser_sdk_gate": browser_sdk_gate,
"errors": [],
"status": "failed",
}
try:
threads: list[dict[str, Any]] = []
offset = 0
while True:
registry.renew_lock(
deployment_id, "workspace-cutover", operation_id, lease_seconds=300
)
registry.renew_lock(
deployment_id, "workspace-lifecycle", operation_id, lease_seconds=300
)
page = list(client.threads.search(limit=100, offset=offset))
threads.extend(page)
if len(page) < 100:
break
offset += len(page)
for thread in threads:
registry.renew_lock(
deployment_id, "workspace-cutover", operation_id, lease_seconds=300
)
registry.renew_lock(
deployment_id, "workspace-lifecycle", operation_id, lease_seconds=300
)
metadata = dict(thread.get("metadata") or {})
thread_id = str(thread.get("thread_id") or "")
is_primary = (
metadata.get("graph_id") == assistant_id
or metadata.get("agent_name") == assistant_id
)
if is_primary:
if not thread_id:
continue
active_runs = _active_run_ids(client, thread_id)
if active_runs:
report["active_runs"].append(
{"thread_id": thread_id, "run_ids": active_runs}
)
continue
record = provision_conversation_scope(
thread_id,
deployment_id=deployment_id,
workspace_root=workspace_root,
lock_operation_id=operation_id,
)
updated = client.threads.update(
thread_id, metadata={**metadata, **workspace_metadata(record)}
)
if isinstance(updated, dict):
updated_metadata = dict(updated.get("metadata") or {})
if updated_metadata.get("workspace_scope_id") != record.scope_id:
report["metadata_mismatches"].append(thread_id)
report["primary_threads"] += 1
report["scopes"] += 1
elif thread_id:
scope_id = metadata.get("workspace_scope_id")
if scope_id is None:
_quarantine_derived_thread(
client,
thread_id=thread_id,
metadata=metadata,
operation_id=operation_id,
reason="unscoped-derived-thread",
report=report,
)
continue
ownership_error = _scope_owner_error(
registry,
deployment_id=deployment_id,
resource_id=thread_id,
metadata=metadata,
)
if ownership_error is not None:
report["invalid_scoped_derived_threads"].append(
{"thread_id": thread_id, "error": ownership_error}
)
_quarantine_derived_thread(
client,
thread_id=thread_id,
metadata=metadata,
operation_id=operation_id,
reason="invalid-scoped-derived-thread",
report=report,
)
continue
report["validated_derived_threads"].append(thread_id)
active_runs = _active_run_ids(client, thread_id)
if active_runs:
report["active_runs"].append(
{"thread_id": thread_id, "run_ids": active_runs}
)
for cron in client.crons.search(limit=1000):
registry.renew_lock(
deployment_id, "workspace-cutover", operation_id, lease_seconds=300
)
registry.renew_lock(
deployment_id, "workspace-lifecycle", operation_id, lease_seconds=300
)
metadata = dict(cron.get("metadata") or {})
cron_id = str(cron.get("cron_id") or "")
scope_id = metadata.get("workspace_scope_id")
if scope_id is not None:
ownership_error = _scope_owner_error(
registry,
deployment_id=deployment_id,
resource_id=cron_id,
metadata=metadata,
allowed_owner_types=frozenset({"schedule"}),
)
if ownership_error is None:
report["validated_crons"].append(cron_id)
continue
client.crons.update(cron_id, enabled=False)
report["quarantined_crons"] += 1
report["invalid_scoped_crons"].append(
{"cron_id": cron_id, "error": ownership_error}
)
report["quarantined_cron_records"].append(
{"cron_id": cron_id, "reason": "invalid-scoped-cron"}
)
continue
if metadata.get("run_kind") != "scheduled_task":
continue
client.crons.update(cron_id, enabled=False)
report["quarantined_crons"] += 1
report["quarantined_cron_records"].append(
{"cron_id": cron_id, "reason": "unscoped-scheduled-task"}
)
report["status"] = (
"passed"
if (
report["browser_sdk_gate"]
and not report["unmanaged_derived_threads"]
and not report["quarantine_failures"]
and not report["active_runs"]
and not report["metadata_mismatches"]
)
else "failed"
)
report["completed_at"] = _now()
report["report_path"] = str(
workspace_root
/ ".evoscientist"
/ "control"
/ "cutover-reports"
/ f"{operation_id}.json"
)
_write_report(workspace_root, report)
registry.finish_operation(
deployment_id,
operation_id,
state="completed" if report["status"] == "passed" else "failed",
result_sha256=str(report["sha256"]),
last_error_code=None
if report["status"] == "passed"
else "cutover-gates-failed",
)
return report
except Exception as exc:
report["errors"].append(str(exc))
report["completed_at"] = _now()
report["report_path"] = str(
workspace_root
/ ".evoscientist"
/ "control"
/ "cutover-reports"
/ f"{operation_id}.json"
)
_write_report(workspace_root, report)
registry.finish_operation(
deployment_id,
operation_id,
state="failed",
result_sha256=str(report["sha256"]),
last_error_code="cutover-exception",
)
return report
finally:
try:
registry.release_lock(deployment_id, "workspace-cutover", operation_id)
except Exception:
# A lost/expired lease is already recorded as a failed report.
pass
try:
registry.release_lock(deployment_id, "workspace-lifecycle", operation_id)
except Exception:
pass
EvoScientist/workspace_maintenance.py
+168
View File
@@ -0,0 +1,168 @@
"""Idempotent lifecycle maintenance for conversation workspaces."""
from __future__ import annotations
import os
import shutil
import uuid
from datetime import UTC, datetime, timedelta
from pathlib import Path
from typing import Any
from .scope_registry import (
ScopeRecord,
deployment_id_for_workspace,
get_scope_registry,
)
_TERMINAL_RUN_STATES = frozenset(
{"success", "error", "timeout", "cancelled", "interrupted"}
)
def _now() -> datetime:
return datetime.now(UTC)
def _parse_timestamp(value: str) -> datetime:
parsed = datetime.fromisoformat(value)
return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC)
def _trash_root(workspace_root: Path) -> Path:
return workspace_root / ".evoscientist" / "trash"
def _conversation_root(workspace_root: Path, scope_id: str) -> Path:
return workspace_root / ".evoscientist" / "conversations" / scope_id
def _move_to_trash(workspace_root: Path, scope_id: str, now: datetime) -> None:
source = _conversation_root(workspace_root, scope_id)
target_root = _trash_root(workspace_root)
target_root.mkdir(mode=0o700, parents=True, exist_ok=True)
target = target_root / f"{scope_id}-{int(now.timestamp() * 1000)}"
try:
os.replace(source, target)
os.utime(target, (now.timestamp(), now.timestamp()))
except FileNotFoundError:
return
def _purge_trash(workspace_root: Path, cutoff: datetime) -> tuple[int, list[str]]:
root = _trash_root(workspace_root)
if not root.exists():
return 0, []
removed = 0
errors: list[str] = []
for candidate in root.iterdir():
try:
if candidate.is_symlink() or not candidate.is_dir():
continue
modified = datetime.fromtimestamp(candidate.stat().st_mtime, tz=UTC)
if modified >= cutoff:
continue
shutil.rmtree(candidate)
removed += 1
except OSError as exc:
errors.append(f"trash:{candidate.name}:{exc}")
return removed, errors
def _draft_can_be_deleted(scope: ScopeRecord, registry: Any, client: Any) -> bool:
owners = registry.owners(scope.deployment_id, scope.scope_id)
if any(owner.owner_type != "primary_thread" for owner in owners):
return False
try:
runs = list(client.runs.list(thread_id=scope.primary_thread_id, limit=1000))
if any(str(run.get("status")) not in _TERMINAL_RUN_STATES for run in runs):
return False
state = client.threads.get_state(scope.primary_thread_id)
values = state.get("values") if isinstance(state, dict) else None
messages = values.get("messages") if isinstance(values, dict) else None
return not messages
except Exception:
# Maintenance must not delete a draft whose content cannot be proven empty.
return False
def run_workspace_maintenance(
*,
workspace_root: Path,
client: Any,
draft_ttl_hours: int = 24,
trash_retention_days: int = 7,
now: datetime | None = None,
) -> dict[str, Any]:
"""Delete safely-drained stale drafts and purge aged trash directories."""
if draft_ttl_hours <= 0 or trash_retention_days <= 0:
raise ValueError("workspace maintenance retention values must be positive")
workspace_root = workspace_root.expanduser().resolve()
current = now or _now()
deployment_id = deployment_id_for_workspace(workspace_root)
registry = get_scope_registry(workspace_root)
if registry.active_lock(deployment_id, "workspace-cutover") is not None:
raise RuntimeError("workspace maintenance is blocked by workspace cutover")
operation_id = str(uuid.uuid4())
registry.acquire_lock(
deployment_id, "workspace-lifecycle", operation_id, lease_seconds=300
)
registry.begin_operation(deployment_id, operation_id, kind="workspace-maintenance")
report: dict[str, Any] = {
"operation_id": operation_id,
"deployment_id": deployment_id,
"started_at": current.isoformat(),
"deleted_drafts": [],
"skipped_drafts": [],
"purged_trash": 0,
"errors": [],
}
try:
draft_cutoff = current - timedelta(hours=draft_ttl_hours)
for scope in registry.list_scopes(deployment_id, state="draft"):
registry.renew_lock(
deployment_id, "workspace-lifecycle", operation_id, lease_seconds=300
)
if _parse_timestamp(scope.created_at) >= draft_cutoff:
continue
if not _draft_can_be_deleted(scope, registry, client):
report["skipped_drafts"].append(scope.primary_thread_id)
continue
deleting = registry.transition_scope(
deployment_id,
scope.scope_id,
expected_revision=scope.revision,
state="deleting",
)
_move_to_trash(workspace_root, deleting.scope_id, current)
try:
client.threads.delete(deleting.primary_thread_id)
except Exception as exc:
status = getattr(exc, "status", None)
if status != 404 and "not found" not in str(exc).lower():
raise
registry.transition_scope(
deployment_id,
deleting.scope_id,
expected_revision=deleting.revision,
state="deleted",
)
report["deleted_drafts"].append(scope.primary_thread_id)
purged, errors = _purge_trash(
workspace_root, current - timedelta(days=trash_retention_days)
)
report["purged_trash"] = purged
report["errors"].extend(errors)
except Exception as exc:
report["errors"].append(str(exc))
finally:
report["completed_at"] = _now().isoformat()
registry.finish_operation(
deployment_id,
operation_id,
state="completed" if not report["errors"] else "failed",
last_error_code=None if not report["errors"] else "maintenance-failed",
)
registry.release_lock(deployment_id, "workspace-lifecycle", operation_id)
return report
EvoScientist/workspace_scope.py
+541
View File
@@ -0,0 +1,541 @@
"""Conversation-scoped workspace resolution and DeepAgents backend factory."""
from __future__ import annotations
import os
import shutil
import subprocess
import threading
import uuid
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from deepagents.backends.protocol import (
EditResult,
ExecuteResponse,
FileDownloadResponse,
FileUploadResponse,
GlobResult,
GrepResult,
LsResult,
ReadResult,
SandboxBackendProtocol,
WriteResult,
)
from langchain.tools import ToolRuntime
from . import paths
from .scope_registry import (
ScopeAccessError,
ScopeRecord,
deployment_id_for_workspace,
get_scope_registry,
)
IsolationMode = str
def workspace_isolation_mode() -> IsolationMode:
value = os.getenv("EVOSCIENTIST_WORKSPACE_ISOLATION", "optional").strip().lower()
if value not in {"legacy", "optional", "required"}:
raise RuntimeError("EVOSCIENTIST_WORKSPACE_ISOLATION must be legacy, optional or required")
return value
def is_required() -> bool:
return workspace_isolation_mode() == "required"
def verify_required_executor() -> None:
"""Fail startup unless the pinned scope executor is locally usable."""
docker = shutil.which("docker")
if not docker:
raise RuntimeError("required workspace isolation needs the docker OCI runtime")
image = os.getenv("EVOSCIENTIST_STRICT_EXECUTOR_IMAGE", "").strip()
if "@sha256:" not in image:
raise RuntimeError("required workspace isolation needs an OCI image pinned by digest")
try:
probe = subprocess.run(
[docker, "image", "inspect", image],
check=False,
capture_output=True,
text=True,
timeout=10,
)
except (OSError, subprocess.TimeoutExpired) as exc:
raise RuntimeError("required workspace isolation cannot verify the OCI executor") from exc
if probe.returncode != 0:
raise RuntimeError(
f"required workspace isolation needs local OCI image {image!r}"
)
def current_deployment_id() -> str:
return deployment_id_for_workspace(paths.WORKSPACE_ROOT)
def conversation_root(scope_id: str, workspace_root: Path | None = None) -> Path:
scope = str(uuid.UUID(scope_id))
# The deploy process supplies an absolute workspace root. This helper is
# called from synchronous DeepAgents backend factories on the ASGI loop.
root = (workspace_root or paths.WORKSPACE_ROOT).expanduser()
return root / ".evoscientist" / "conversations" / scope
def conversation_files_dir(scope_id: str, workspace_root: Path | None = None) -> Path:
return conversation_root(scope_id, workspace_root) / "files"
@dataclass(frozen=True, slots=True)
class ScopeContext:
deployment_id: str
scope_id: str
owner_id: str
thread_id: str
revision: int
files_dir: Path
runtime_dir: Path
primary_thread_id: str
@dataclass(frozen=True, slots=True)
class _RuntimeScopeConfig:
"""Untrusted runtime identifiers parsed without filesystem or Registry I/O."""
scope_id: str
owner_id: str
thread_id: str
deployment_id: str | None
class ScopedContainerBackend:
"""Filesystem backend whose shell commands execute in a scope-only OCI container."""
def __init__(self, root_dir: Path, *, timeout: int) -> None:
from .backends import CustomSandboxBackend
# Reuse the hardened filesystem operations; only ``execute`` is
# replaced so no agent shell runs in the host process.
self._filesystem = CustomSandboxBackend(
root_dir=str(root_dir), virtual_mode=True, timeout=timeout, dangerous=False
)
self._root_dir = root_dir
self._timeout = timeout
def __getattr__(self, name: str) -> Any:
return getattr(self._filesystem, name)
def execute(self, command: str, *, timeout: int | None = None) -> Any:
from .backends import ExecuteResponse, prepare_sandbox_command
command, error = prepare_sandbox_command(
command, self._filesystem.cwd, virtual_mode=True, dangerous=False
)
if error:
return ExecuteResponse(output=error, exit_code=1, truncated=False)
image = os.getenv("EVOSCIENTIST_STRICT_EXECUTOR_IMAGE", "").strip()
if "@sha256:" not in image:
return ExecuteResponse(
output="Required workspace isolation needs an OCI image pinned by digest.",
exit_code=125,
truncated=False,
)
effective_timeout = max(1, min(timeout or self._timeout, 3600))
invocation = [
"docker",
"run",
"--rm",
"--network",
"none",
"--read-only",
"--tmpfs",
"/tmp:rw,noexec,nosuid,size=64m",
"--cap-drop",
"ALL",
"--security-opt",
"no-new-privileges",
"--pids-limit",
os.getenv("EVOSCIENTIST_STRICT_EXECUTOR_PIDS", "128"),
"--memory",
os.getenv("EVOSCIENTIST_STRICT_EXECUTOR_MEMORY", "1g"),
"--cpus",
os.getenv("EVOSCIENTIST_STRICT_EXECUTOR_CPUS", "1"),
"--mount",
f"type=bind,src={self._root_dir},dst=/workspace",
"--workdir",
"/workspace",
image,
"sh",
"-lc",
command,
]
try:
completed = subprocess.run(
invocation,
check=False,
capture_output=True,
text=True,
timeout=effective_timeout,
)
except FileNotFoundError:
return ExecuteResponse(
output="Required workspace isolation needs an OCI runtime (docker was not found).",
exit_code=127,
truncated=False,
)
except subprocess.TimeoutExpired as exc:
output = (exc.stdout or "") + (exc.stderr or "")
return ExecuteResponse(output=output, exit_code=124, truncated=False)
output = completed.stdout + completed.stderr
return ExecuteResponse(output=output, exit_code=completed.returncode, truncated=False)
def _configurable(runtime: ToolRuntime[Any, Any] | Any | None) -> dict[str, Any]:
"""Return the active runnable config, with a non-graph fallback.
``ToolRuntime`` deliberately does not expose ``RunnableConfig`` during a
graph execution. LangGraph keeps it in a context variable instead. The
fallback preserves direct callers and unit tests that supply a lightweight
runtime object outside a runnable context.
"""
config: Any = None
try:
from langgraph.config import get_config
config = get_config()
except (ImportError, LookupError, RuntimeError):
pass
if not isinstance(config, dict) and runtime is not None:
config = getattr(runtime, "config", None) or {}
if not isinstance(config, dict):
return {}
configurable = config.get("configurable") or {}
return dict(configurable) if isinstance(configurable, dict) else {}
def _required_string(configurable: dict[str, Any], key: str) -> str:
value = configurable.get(key)
if not isinstance(value, str) or not value:
raise ScopeAccessError(f"missing {key}")
return value
def _runtime_scope_config(
runtime: ToolRuntime[Any, Any] | Any | None,
*,
kind: str,
) -> _RuntimeScopeConfig | None:
"""Parse scope identifiers without treating config as an authorization grant."""
configurable = _configurable(runtime)
scope_id = configurable.get("workspace_scope_id")
owner_id = configurable.get("workspace_scope_owner_id")
thread_id = configurable.get("thread_id")
if scope_id is None and owner_id is None:
if workspace_isolation_mode() == "required":
raise ScopeAccessError(f"{kind} requires a workspace scope")
return None
if (
not isinstance(scope_id, str)
or not isinstance(owner_id, str)
or not isinstance(thread_id, str)
):
raise ScopeAccessError(f"{kind} has an incomplete workspace scope")
try:
canonical_scope_id = str(uuid.UUID(scope_id))
canonical_owner_id = str(uuid.UUID(owner_id))
except ValueError as exc:
raise ScopeAccessError(f"{kind} has an invalid workspace scope") from exc
deployment_id = configurable.get("workspace_deployment_id")
if deployment_id is not None and (
not isinstance(deployment_id, str) or not deployment_id
):
raise ScopeAccessError(f"{kind} has an invalid workspace deployment")
return _RuntimeScopeConfig(
scope_id=canonical_scope_id,
owner_id=canonical_owner_id,
thread_id=thread_id,
deployment_id=deployment_id,
)
def _validated_scope_directories(scope_id: str) -> tuple[Path, Path]:
"""Return canonical private directories after preventing symlink escape.
This function intentionally resolves paths and must run only from a
filesystem-operation worker, never from the runtime backend factory.
"""
conversations_dir = (
paths.WORKSPACE_ROOT.expanduser() / ".evoscientist" / "conversations"
).resolve(strict=True)
scope_root = (conversations_dir / scope_id).resolve(strict=True)
files_dir = (scope_root / "files").resolve(strict=True)
runtime_dir = (scope_root / "runtime").resolve(strict=True)
if (
scope_root.parent != conversations_dir
or files_dir.parent != scope_root
or runtime_dir.parent != scope_root
):
raise ScopeAccessError("workspace directory escapes its scope")
if not files_dir.is_dir() or not runtime_dir.is_dir():
raise ScopeAccessError("workspace directory is missing")
return files_dir, runtime_dir
def _resolve_scope_context(config: _RuntimeScopeConfig | None) -> ScopeContext | None:
"""Validate parsed scope identifiers against the active registry."""
if config is None:
return None
deployment_id = config.deployment_id or current_deployment_id()
registry = get_scope_registry(paths.WORKSPACE_ROOT)
if registry.active_lock(deployment_id, "workspace-cutover") is not None:
raise ScopeAccessError("workspace cutover is in progress")
record = registry.assert_runtime(
deployment_id, config.scope_id, config.thread_id, config.owner_id
)
files_dir, runtime_dir = _validated_scope_directories(record.scope_id)
return ScopeContext(
deployment_id=deployment_id,
scope_id=record.scope_id,
owner_id=config.owner_id,
thread_id=config.thread_id,
revision=record.revision,
files_dir=files_dir,
runtime_dir=runtime_dir,
primary_thread_id=record.primary_thread_id,
)
def require_scoped_runtime(
runtime: ToolRuntime[Any, Any] | Any | None,
*,
kind: str = "tool",
) -> ScopeContext | None:
"""Resolve and validate a runtime scope.
``optional`` retains legacy CLI compatibility when no scope has been
injected. ``required`` never falls back to ``WORKSPACE_ROOT``.
"""
config = _runtime_scope_config(runtime, kind=kind)
return _resolve_scope_context(config)
def provision_conversation_scope(
thread_id: str,
*,
deployment_id: str | None = None,
scope_id: str | None = None,
workspace_root: Path | None = None,
lock_operation_id: str | None = None,
) -> ScopeRecord:
"""Create the registry mapping and private directory for a primary thread."""
root = (workspace_root or paths.WORKSPACE_ROOT).expanduser()
deployment_id = deployment_id or deployment_id_for_workspace(root)
registry = get_scope_registry(root)
record = registry.provision(
deployment_id,
thread_id,
scope_id=scope_id,
lock_operation_id=lock_operation_id,
)
root = conversation_root(record.scope_id, root)
try:
(root / "files").mkdir(mode=0o700, parents=True, exist_ok=True)
(root / "runtime").mkdir(mode=0o700, parents=True, exist_ok=True)
for directory in (root, root / "files", root / "runtime"):
try:
directory.chmod(0o700)
except OSError:
pass
except OSError:
# Keep the durable reservation for the recovery job; it is safer than
# silently falling back to the shared deployment root.
raise
return record
def _build_backend(root_dir: Path, *, dangerous: bool) -> Any:
from deepagents.backends import CompositeBackend
from .backends import (
CustomSandboxBackend,
MemoryFilesystemBackend,
MergedSkillsBackend,
)
from .EvoScientist import SKILLS_DIR
cfg_timeout = int(os.getenv("EVOSCIENTIST_SANDBOX_EXECUTE_TIMEOUT", "300"))
ws_backend: Any
if is_required():
ws_backend = ScopedContainerBackend(root_dir, timeout=cfg_timeout)
else:
ws_backend = CustomSandboxBackend(
root_dir=str(root_dir),
virtual_mode=True,
timeout=cfg_timeout,
dangerous=dangerous,
)
return CompositeBackend(
default=ws_backend,
routes={
"/skills/": MergedSkillsBackend(
primary_dir=str(paths.USER_SKILLS_DIR),
global_dir=str(paths.GLOBAL_SKILLS_DIR),
secondary_dir=SKILLS_DIR,
),
"/memories/": MemoryFilesystemBackend(
root_dir=str(paths.MEMORIES_DIR), virtual_mode=True
),
},
)
class DeferredScopedBackend(SandboxBackendProtocol):
"""Resolve the scoped filesystem backend only from a worker thread.
DeepAgents invokes its deprecated backend factory from async middleware.
Its concrete filesystem backends synchronously call ``Path.resolve()`` in
their constructors, so doing that work in the factory makes every run fail
under LangGraph's blocking-call detector. This proxy itself is I/O-free;
the inherited async methods dispatch the synchronous operations to a
thread, where Registry validation and concrete backend construction occur.
"""
def __init__(
self,
config: _RuntimeScopeConfig,
*,
dangerous: bool,
) -> None:
self._config = config
self._dangerous = dangerous
self._backend: Any | None = None
self._backend_key: tuple[str, str, str, int] | None = None
self._lock = threading.RLock()
@property
def id(self) -> str:
# This is queried while composing the model request; do not initialize
# the real backend or touch the Registry here.
return f"scope-{self._config.scope_id[:8]}-{self._config.owner_id[:8]}"
def _delegate(self) -> Any:
"""Validate the current scope and return a concrete backend.
Every operation enters here, so a deleted scope or stale owner cannot
keep using a backend constructed before the lifecycle transition.
"""
# Async backend methods run this code in a worker thread. LangGraph's
# RunnableConfig context variable is not available there, so validate
# the immutable scope parsed by the factory on the graph thread.
context = _resolve_scope_context(self._config)
if context is None:
raise ScopeAccessError("scoped backend lost its workspace scope")
if is_required() and self._dangerous:
raise ScopeAccessError(
"dangerous_mode is incompatible with required isolation"
)
key = (context.scope_id, context.owner_id, context.thread_id, context.revision)
with self._lock:
if self._backend is None or self._backend_key != key:
self._backend = _build_backend(
context.files_dir, dangerous=self._dangerous
)
self._backend_key = key
return self._backend
def ls(self, path: str) -> LsResult:
return self._delegate().ls(path)
def read(self, file_path: str, offset: int = 0, limit: int = 2000) -> ReadResult:
return self._delegate().read(file_path, offset, limit)
def grep(
self, pattern: str, path: str | None = None, glob: str | None = None
) -> GrepResult:
return self._delegate().grep(pattern, path, glob)
def glob(self, pattern: str, path: str | None = None) -> GlobResult:
return self._delegate().glob(pattern, path)
def write(self, file_path: str, content: str) -> WriteResult:
return self._delegate().write(file_path, content)
def edit(
self,
file_path: str,
old_string: str,
new_string: str,
replace_all: bool = False,
) -> EditResult:
return self._delegate().edit(file_path, old_string, new_string, replace_all)
def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]:
return self._delegate().upload_files(files)
def download_files(self, paths: list[str]) -> list[FileDownloadResponse]:
return self._delegate().download_files(paths)
def execute(self, command: str, *, timeout: int | None = None) -> ExecuteResponse:
return self._delegate().execute(command, timeout=timeout)
def create_workspace_backend(
runtime: ToolRuntime[Any, Any],
*,
legacy_backend: Callable[[], Any],
dangerous: bool = False,
allow_unscoped_legacy: bool = True,
) -> Any:
"""Return a backend handle without blocking the Agent event loop."""
config = _runtime_scope_config(runtime, kind="filesystem backend")
if config is None:
if not allow_unscoped_legacy:
raise ScopeAccessError(
"deployed graph runs require a workspace scope"
)
return legacy_backend()
if is_required() and dangerous:
raise ScopeAccessError("dangerous_mode is incompatible with required isolation")
return DeferredScopedBackend(config, dangerous=dangerous)
def create_workspace_backend_factory(
legacy_backend: Callable[[], Any],
*,
dangerous: bool = False,
allow_unscoped_legacy: bool = True,
) -> Callable[[ToolRuntime[Any, Any]], Any]:
def factory(runtime: ToolRuntime[Any, Any]) -> Any:
return create_workspace_backend(
runtime,
legacy_backend=legacy_backend,
dangerous=dangerous,
allow_unscoped_legacy=allow_unscoped_legacy,
)
return factory
def workspace_metadata(record: ScopeRecord) -> dict[str, str | int]:
"""Metadata mirrored onto the LangGraph primary thread by trusted callers."""
return {
"workspace_schema_version": 1,
"workspace_scope_id": record.scope_id,
"workspace_status": record.state,
"workspace_scope_owner_id": record.primary_owner_id,
"workspace_scope_revision": record.revision,
"workspace_deployment_id": record.deployment_id,
}
README.md
+28
View File
@@ -428,6 +428,32 @@ Alternatively, copy the example `.env` file for project-level configuration:
cp .env.example .env # then fill in your keys
```
### WebUI workspace isolation
`EVOSCIENTIST_WORKSPACE_DIR` identifies the deployment workspace root. It is
not a directory selected by the browser. In isolated modes, each WebUI
conversation receives its own `files/` and `runtime/` directories below:
```text
<workspace>/.evoscientist/conversations/<scope-id>/
```
`EVOSCIENTIST_WORKSPACE_ISOLATION` is a deployment-level policy with three
values:
| Value | Behavior | Use it when |
| --- | --- | --- |
| `legacy` | All WebUI conversations use the shared deployment root. Files are visible across those conversations. | Only for an explicit compatibility rollback. |
| `optional` | New WebUI conversations are given isolated scope directories. Missing Registry, token, or scope rejects the operation; it does not fall back to the shared root. | Default for single-host deployments and migration. |
| `required` | Keeps per-conversation isolation and additionally rejects missing or invalid runtime ownership. It requires a completed cutover and a verified, pinned OCI executor. | Production hardening after migration is complete. |
Use `optional` unless you are intentionally rolling back or have completed the
`required` deployment checks. Changing this setting, the workspace root, or the
strict executor configuration requires restarting both the backend and WebUI.
It affects new operations only; do not use it to convert an existing
conversation between shared and isolated directories. Create a new conversation
and explicitly import files instead.
> ⚠️ Never commit `.env` files with real keys. It is already in `.gitignore`.
</details>
@@ -481,6 +507,8 @@ EvoSci config set webui_port 4800 # change the front-end port (must differ fr
Requires **Node.js 24 LTS** (for `npx`); the first launch downloads `@evoscientist/webui` and needs network. Note: the WebUI does not show your CLI/TUI chat history, and `-p` / `--resume` fall back to the classic CLI.
You may also run `EvoSci deploy --workdir <path>` and the WebUI separately. For processes owned by the same OS user, both sides automatically share the token-usage identity and private sink token through `~/.evoscientist`; no manual environment variables are required. The WebUI port must match the configured `webui_port` (default `4716`).
</details>
<details>
README.zh-CN.md
+23
View File
@@ -406,6 +406,27 @@ export TAVILY_API_KEY="tvly-..." # app.tavily.com
cp .env.example .env # 填入你的密钥
```
### WebUI 工作目录隔离
`EVOSCIENTIST_WORKSPACE_DIR` 指向部署工作目录根，而不是浏览器可选择的目录。在隔离模式下，
每个 WebUI 对话会在以下位置拥有独立的 `files/` 与 `runtime/` 目录：
```text
<workspace>/.evoscientist/conversations/<scope-id>/
```
`EVOSCIENTIST_WORKSPACE_ISOLATION` 是部署级策略，支持三个值：
| 值 | 行为 | 适用场景 |
| --- | --- | --- |
| `legacy` | 所有 WebUI 对话使用同一个部署公共根目录，文件会在这些对话之间可见。 | 仅用于明确的兼容性回退。 |
| `optional` | 新建 WebUI 对话获得独立 scope 目录；缺少 Registry、令牌或 scope 时请求失败，不会回退到公共目录。 | 默认值，适用于单机部署和迁移阶段。 |
| `required` | 保留会话隔离，并拒绝缺少或非法运行时归属的操作；需要完成全量 cutover 并验证固定 OCI 执行器。 | 迁移完成后的生产加固。 |
除非明确需要回退，或已经完成 `required` 的部署检查，否则应使用 `optional`。修改该值、
工作目录根或严格执行器配置后，必须同时重启后端和 WebUI。它不会转换已有对话的目录策略；
若需在共享与隔离之间转换，应新建对话并显式导入文件。
> ⚠️ 切勿将包含真实密钥的 `.env` 文件提交到版本库。该文件已在 `.gitignore` 中。
</details>
@@ -456,6 +477,8 @@ EvoSci config set webui_port 4800 # 修改前端端口（须与 langgraph dev
需要 **Node.js 24 LTS**（提供 `npx`）；首次启动会下载 `@evoscientist/webui`，需要联网。注意：WebUI 不会显示 CLI/TUI 的历史会话，且 `-p` / `--resume` 会回退到经典 CLI。
也可以分别运行 `EvoSci deploy --workdir <目录>` 和 WebUI。相同系统用户下，两端会通过 `~/.evoscientist` 自动共享 Token 统计身份和私有 sink token，无需手工设置环境变量。WebUI 端口必须与 `webui_port` 配置一致（默认 `4716`）。
</details>
<details>
pyproject.toml
+1
View File
@@ -34,6 +34,7 @@ dependencies = [
"langgraph-cli[inmem]>=0.4",
"langgraph-checkpoint-sqlite>=3.0",
"httpx>=0.28",
"pydantic>=2.10",
"psutil>=6.0",
"filelock>=3.16",
"lazy-loader>=0.5",
scripts/benchmark_usage_spool.py
+127
View File
@@ -0,0 +1,127 @@
"""Measure durable UsageEvent enqueue latency for the release platform matrix."""
from __future__ import annotations
import json
import os
import platform
import tempfile
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime
from pathlib import Path
from EvoScientist.usage.schema import UsageEventV1
from EvoScientist.usage.spool import UsageSpool
WORKER_COUNTS = (1, 4, 16)
SAMPLES_PER_WORKER = 32
P95_LIMIT_MS = 20.0
P99_LIMIT_MS = 100.0
class BenchmarkSpool(UsageSpool):
def _run(self) -> None:
self._stop.wait()
def _percentile(samples: list[float], percentile: float) -> float:
ordered = sorted(samples)
index = min(len(ordered) - 1, max(0, int(len(ordered) * percentile) - 1))
return ordered[index]
def _event(index: int) -> UsageEventV1:
model_call_id = f"benchmark-{index:08d}"
now = datetime.now(UTC)
deployment_id = "11111111-1111-4111-8111-111111111111"
return UsageEventV1(
schema_version=1,
event_id=f"{deployment_id}:{model_call_id}:callback_final:1",
event_type="usage_observed",
source="callback_final",
authority_class="observed_final",
revision=1,
deployment_id=deployment_id,
workspace_id="ws1_benchmark",
model_call_id=model_call_id,
parent_run_id=None,
provider_request_id=None,
thread_id="benchmark-thread",
source_session_id=None,
turn_id="benchmark-turn",
workspace_dir=None,
scope="main",
source_agent="EvoScientist",
provider_profile_id="benchmark",
provider_revision=None,
provider_adapter="openai",
model_alias="benchmark",
upstream_model_id="benchmark",
usage_status="confirmed",
input_tokens=1,
output_tokens=1,
provider_total_tokens=2,
input_token_details={},
output_token_details={},
started_at=now,
observed_at=now,
completed_at=now,
)
def _run_case(root: Path, workers: int) -> dict[str, float | int]:
spool_root = root / str(workers)
os.environ.update(
{
"EVOSCIENTIST_USAGE_SINK_URL": "http://127.0.0.1:1/api/usage/events",
"EVOSCIENTIST_USAGE_SINK_TOKEN": "benchmark",
"EVOSCIENTIST_DEPLOYMENT_ID": "11111111-1111-4111-8111-111111111111",
"EVOSCIENTIST_WORKSPACE_ID": "ws1_benchmark",
"EVOSCIENTIST_USAGE_SPOOL_DIR": str(spool_root),
"EVOSCIENTIST_USAGE_SPOOL_MAX_FILES": "100000",
}
)
spool = BenchmarkSpool()
def enqueue(index: int) -> float:
started = time.perf_counter_ns()
spool.enqueue(_event(workers * 1_000_000 + index))
return (time.perf_counter_ns() - started) / 1_000_000
try:
with ThreadPoolExecutor(max_workers=workers) as pool:
samples = list(pool.map(enqueue, range(workers * SAMPLES_PER_WORKER)))
finally:
spool.close()
return {
"workers": workers,
"samples": len(samples),
"p50_ms": round(_percentile(samples, 0.50), 3),
"p95_ms": round(_percentile(samples, 0.95), 3),
"p99_ms": round(_percentile(samples, 0.99), 3),
"max_ms": round(max(samples), 3),
}
def main() -> None:
with tempfile.TemporaryDirectory(prefix="evosci-usage-benchmark-") as temporary:
results = [_run_case(Path(temporary), workers) for workers in WORKER_COUNTS]
report = {
"platform": platform.platform(),
"python": platform.python_version(),
"limits_ms": {"p95": P95_LIMIT_MS, "p99": P99_LIMIT_MS},
"results": results,
}
print(json.dumps(report, indent=2, sort_keys=True))
failed = [
result
for result in results
if result["p95_ms"] > P95_LIMIT_MS or result["p99_ms"] > P99_LIMIT_MS
]
if failed:
raise SystemExit("usage spool latency exceeds the release threshold")
if __name__ == "__main__":
main()
scripts/workspace_cutover.py
+49
View File
@@ -0,0 +1,49 @@
#!/usr/bin/env python3
"""Run the durable conversation-workspace migration against a local backend."""
from __future__ import annotations
import argparse
import json
import subprocess
from pathlib import Path
from EvoScientist.langgraph_dev.sdk import get_langgraph_sync_client
from EvoScientist.workspace_cutover import run_workspace_cutover
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--workspace", required=True)
parser.add_argument("--api-url", default="http://127.0.0.1:6174")
parser.add_argument("--assistant-id", default="EvoScientist")
parser.add_argument(
"--webui-root",
default=str(Path(__file__).resolve().parents[2] / "EvoScientist-WebUI"),
)
args = parser.parse_args()
try:
gate = subprocess.run(
["npm", "run", "verify:workspace-isolation"],
cwd=args.webui_root,
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
timeout=120,
)
browser_sdk_gate = gate.returncode == 0
except (OSError, subprocess.TimeoutExpired):
browser_sdk_gate = False
report = run_workspace_cutover(
workspace_root=Path(args.workspace),
client=get_langgraph_sync_client(url=args.api_url),
assistant_id=args.assistant_id,
browser_sdk_gate=browser_sdk_gate,
)
print(json.dumps(report, indent=2, sort_keys=True))
return 0 if report["status"] == "passed" else 2
if __name__ == "__main__":
raise SystemExit(main())
scripts/workspace_maintenance.py
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env python3
"""Run periodic draft and conversation-trash cleanup for one deployment."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from EvoScientist.config import load_config
from EvoScientist.langgraph_dev.sdk import get_langgraph_sync_client
from EvoScientist.workspace_maintenance import run_workspace_maintenance
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--workspace", required=True)
parser.add_argument("--api-url", default="http://127.0.0.1:6174")
parser.add_argument("--draft-ttl-hours", type=int)
parser.add_argument("--trash-retention-days", type=int)
args = parser.parse_args()
config = load_config()
report = run_workspace_maintenance(
workspace_root=Path(args.workspace),
client=get_langgraph_sync_client(url=args.api_url),
draft_ttl_hours=args.draft_ttl_hours or config.draft_workspace_ttl_hours,
trash_retention_days=args.trash_retention_days
or config.workspace_trash_retention_days,
)
print(json.dumps(report, indent=2, sort_keys=True))
return 0 if not report["errors"] else 2
if __name__ == "__main__":
raise SystemExit(main())
tests/test_ask_user.py
+137
View File
@@ -1,9 +1,23 @@
"""Tests for the ask_user middleware, stream events, state, and UI helpers."""
from contextlib import contextmanager
from unittest.mock import MagicMock, patch
import pytest
@contextmanager
def _patched_runtime_config(config):
import langgraph.config as langgraph_config
if isinstance(config, Exception):
with patch.object(langgraph_config, "get_config", side_effect=config):
yield
else:
with patch.object(langgraph_config, "get_config", return_value=config):
yield
# ---------------------------------------------------------------------------
# Middleware data types
# ---------------------------------------------------------------------------
@@ -292,6 +306,129 @@ class TestAskUserMiddleware:
assert "resource" in ASK_USER_TOOL_DESCRIPTION.lower()
@pytest.mark.parametrize("mode", ["manual", "auto"])
def test_manual_and_auto_keep_ask_user(self, mode):
from langchain.agents.middleware.types import ModelRequest
from EvoScientist.middleware.ask_user import (
ASK_USER_SYSTEM_PROMPT,
AskUserMiddleware,
)
middleware = AskUserMiddleware()
request = ModelRequest(
model=MagicMock(),
messages=[],
tools=[middleware.tools[0], {"name": "execute"}],
)
handler = MagicMock(return_value="response")
with _patched_runtime_config({"configurable": {"review_mode": mode}}):
result = middleware.wrap_model_call(request, handler)
assert result == "response"
forwarded = handler.call_args.args[0]
assert [
tool.name if hasattr(tool, "name") else tool["name"]
for tool in forwarded.tools
] == [
"ask_user",
"execute",
]
assert ASK_USER_SYSTEM_PROMPT in forwarded.system_message.text
def test_full_removes_ask_user_and_adds_unattended_prompt(self):
from langchain.agents.middleware.types import ModelRequest
from EvoScientist.middleware.ask_user import (
FULL_APPROVE_SYSTEM_PROMPT,
AskUserMiddleware,
)
middleware = AskUserMiddleware()
request = ModelRequest(
model=MagicMock(),
messages=[],
tools=[
middleware.tools[0],
{"name": "execute"},
{"type": "function", "function": {"name": "ask_user"}},
],
)
handler = MagicMock(return_value="response")
with _patched_runtime_config({"configurable": {"review_mode": "full"}}):
result = middleware.wrap_model_call(request, handler)
assert result == "response"
forwarded = handler.call_args.args[0]
assert forwarded.tools == [{"name": "execute"}]
assert FULL_APPROVE_SYSTEM_PROMPT in forwarded.system_message.text
async def test_full_async_wrapper_matches_sync_behavior(self):
from langchain.agents.middleware.types import ModelRequest
from EvoScientist.middleware.ask_user import AskUserMiddleware
middleware = AskUserMiddleware()
request = ModelRequest(
model=MagicMock(),
messages=[],
tools=[middleware.tools[0], {"name": "execute"}],
)
async def handler(forwarded):
assert forwarded.tools == [{"name": "execute"}]
return "response"
with _patched_runtime_config({"configurable": {"review_mode": "full"}}):
result = await middleware.awrap_model_call(request, handler)
assert result == "response"
def test_full_tool_defense_does_not_interrupt(self):
from EvoScientist.middleware.ask_user import (
FULL_APPROVE_TOOL_MESSAGE,
AskUserMiddleware,
)
middleware = AskUserMiddleware()
with (
_patched_runtime_config({"configurable": {"review_mode": "full"}}),
patch("EvoScientist.middleware.ask_user.interrupt") as mock_interrupt,
):
result = middleware.tools[0].func(
questions=[],
tool_call_id="tool-call-1",
)
mock_interrupt.assert_not_called()
message = result.update["messages"][0]
assert message.content == FULL_APPROVE_TOOL_MESSAGE
assert message.tool_call_id == "tool-call-1"
class TestReviewMode:
@pytest.mark.parametrize(
("config", "expected"),
[
({"configurable": {"review_mode": "manual"}}, "manual"),
({"configurable": {"review_mode": "auto"}}, "auto"),
({"configurable": {"review_mode": "full"}}, "full"),
({"configurable": {"review_mode": "invalid"}}, "manual"),
({"configurable": "invalid"}, "manual"),
("invalid", "manual"),
],
)
def test_review_mode_parsing(self, config, expected):
from EvoScientist.middleware.ask_user import _review_mode
with _patched_runtime_config(config):
assert _review_mode() == expected
def test_outside_runtime_defaults_to_manual(self):
from EvoScientist.middleware.ask_user import _review_mode
with _patched_runtime_config(RuntimeError("outside runtime")):
assert _review_mode() == "manual"
# ---------------------------------------------------------------------------
# Stream event emitter
tests/test_cli_deploy.py
+122 -5
View File
@@ -1,7 +1,7 @@
"""Tests for ``EvoSci deploy`` command flow.
Verifies the orchestration:
- workspace resolution (CLI > config > cwd)
- workspace resolution (CLI > config > stable default)
- port resolution (CLI > config > default)
- port collision pre-flight
- ccproxy lifecycle (only if OAuth configured)
@@ -11,6 +11,9 @@ Verifies the orchestration:
from __future__ import annotations
import io
import threading
from pathlib import Path
from types import SimpleNamespace
from typing import Any
@@ -24,6 +27,7 @@ def _make_config(
*,
default_workdir: str = "",
langgraph_dev_port: int = 6174,
webui_port: int = 4716,
anthropic_auth_mode: str = "api_key",
openai_auth_mode: str = "api_key",
log_level: str = "warning",
@@ -34,6 +38,7 @@ def _make_config(
return SimpleNamespace(
default_workdir=default_workdir,
langgraph_dev_port=langgraph_dev_port,
webui_port=webui_port,
anthropic_auth_mode=anthropic_auth_mode,
openai_auth_mode=openai_auth_mode,
log_level=log_level,
@@ -76,6 +81,7 @@ def _run_deploy_once(
langgraph_dev_running: bool = True, # health-check passes after start
tunnel: bool = False,
tunnel_url: str | None = None,
default_workspace: str = "/tmp/evoscientist-default-workspace",
):
"""Run ``deploy()`` end-to-end with all external dependencies mocked.
Returns a ``captured`` dict with observation points."""
@@ -90,8 +96,23 @@ def _run_deploy_once(
"workspace_passed": None,
"port_passed": None,
"atexit_callbacks": [],
"log_follower_started": False,
"log_follower_stopped": False,
"usage_environment_prepared": False,
}
def _fake_prepare_usage_environment(workspace_dir, *, webui_port):
captured["usage_environment_prepared"] = True
captured["usage_workspace"] = str(workspace_dir)
captured["usage_webui_port"] = webui_port
return {}
monkeypatch.setattr(
deploy_server,
"prepare_usage_environment",
_fake_prepare_usage_environment,
)
def _fake_get_effective_config(cli_overrides=None):
captured["cli_overrides"] = dict(cli_overrides or {})
merged = vars(config).copy()
@@ -100,9 +121,32 @@ def _run_deploy_once(
monkeypatch.setattr(config_mod, "get_effective_config", _fake_get_effective_config)
monkeypatch.setattr(config_mod, "apply_config_to_env", lambda _cfg: None)
monkeypatch.setattr(
config_mod, "get_default_workspace_dir", lambda: Path(default_workspace)
)
monkeypatch.setattr(deploy_server, "console", _SilentConsole())
class _FakeFollowerStop:
def set(self):
captured["log_follower_stopped"] = True
class _FakeFollowerThread:
def join(self, timeout=None):
captured["log_follower_join_timeout"] = timeout
def _fake_start_gateway_log_follower(log_path, start_offset):
captured["log_follower_started"] = True
captured["log_follower_path"] = log_path
captured["log_follower_offset"] = start_offset
return _FakeFollowerStop(), _FakeFollowerThread()
monkeypatch.setattr(
deploy_server,
"_start_gateway_log_follower",
_fake_start_gateway_log_follower,
)
# Workspace setup mocks
from EvoScientist import paths as paths_mod
@@ -242,6 +286,74 @@ def test_deploy_starts_langgraph_dev_with_deploy_mode_true(monkeypatch, tmp_path
)
def test_deploy_prepares_usage_for_separately_started_webui(monkeypatch, tmp_path):
workspace = tmp_path / "workspace"
config = _make_config(
default_workdir=str(workspace),
webui_port=4816,
)
captured = _run_deploy_once(monkeypatch, config)
assert captured["usage_environment_prepared"] is True
assert captured["usage_workspace"] == str(workspace)
assert captured["usage_webui_port"] == 4816
def test_deploy_streams_and_stops_gateway_log_follower(monkeypatch, tmp_path):
config = _make_config(default_workdir=str(tmp_path))
captured = _run_deploy_once(monkeypatch, config)
assert captured["log_follower_started"] is True
assert captured["log_follower_stopped"] is True
assert captured["log_follower_join_timeout"] == 2.0
def test_follow_gateway_log_only_replays_current_session(tmp_path):
log_path = tmp_path / "langgraph_dev.log"
stale = "old session\n"
current = "Gateway ready\n模型调用完成\n"
log_path.write_bytes((stale + current).encode("utf-8"))
stop_event = threading.Event()
stop_event.set()
output = io.StringIO()
deploy_server._follow_gateway_log(
log_path,
len(stale.encode("utf-8")),
stop_event,
output=output,
)
assert output.getvalue() == current
def test_follow_gateway_log_streams_appended_output(tmp_path):
log_path = tmp_path / "langgraph_dev.log"
log_path.write_text("Gateway ready\n", encoding="utf-8")
output = io.StringIO()
class _AppendThenStop:
stopped = False
def is_set(self):
return self.stopped
def wait(self, _timeout):
with log_path.open("a", encoding="utf-8") as log_file:
log_file.write("new request\n")
self.stopped = True
deploy_server._follow_gateway_log(
log_path,
0,
_AppendThenStop(),
output=output,
)
assert output.getvalue() == "Gateway ready\nnew request\n"
def test_deploy_tunnel_default_off(monkeypatch, tmp_path):
"""Without ``--tunnel``, start_langgraph_dev is called with tunnel=False."""
config = _make_config(default_workdir=str(tmp_path))
@@ -280,12 +392,17 @@ def test_deploy_workdir_config_beats_cwd(monkeypatch, tmp_path):
assert captured["workspace_passed"] == str(cfg_ws)
def test_deploy_workdir_falls_back_to_cwd(monkeypatch, tmp_path):
def test_deploy_workdir_falls_back_to_stable_default(monkeypatch, tmp_path):
config = _make_config(default_workdir="")
cwd = str(tmp_path / "cwd")
captured = _run_deploy_once(monkeypatch, config, cwd=cwd)
default_workspace = str(tmp_path / "default-workspace")
captured = _run_deploy_once(
monkeypatch,
config,
cwd=str(tmp_path / "cwd"),
default_workspace=default_workspace,
)
assert captured["workspace_passed"] == cwd
assert captured["workspace_passed"] == default_workspace
def test_deploy_port_cli_arg_beats_config(monkeypatch, tmp_path):
tests/test_config.py
+26
View File
@@ -17,6 +17,7 @@ from EvoScientist.config import (
get_config_dir,
get_config_path,
get_config_value,
get_default_workspace_dir,
get_effective_config,
list_config,
load_config,
@@ -192,6 +193,26 @@ class TestEvoScientistConfig:
assert config.dangerous_mode is True
assert config.auto_approve is True
def test_workspace_isolation_defaults_to_optional(self):
config = EvoScientistConfig()
assert config.workspace_isolation == "optional"
def test_required_isolation_needs_a_pinned_executor_image(self):
with pytest.raises(ValueError, match="pinned by digest"):
EvoScientistConfig(workspace_isolation="required")
config = EvoScientistConfig(
workspace_isolation="required",
strict_executor_image="registry.example/runtime@sha256:" + "a" * 64,
)
assert config.strict_executor_image.endswith("a" * 64)
def test_workspace_maintenance_retention_values_must_be_positive(self):
with pytest.raises(ValueError, match="draft_workspace_ttl_hours"):
EvoScientistConfig(draft_workspace_ttl_hours=0)
with pytest.raises(ValueError, match="workspace_trash_retention_days"):
EvoScientistConfig(workspace_trash_retention_days=0)
# =============================================================================
# Test config path functions
@@ -216,6 +237,11 @@ class TestConfigPaths:
config_path = get_config_path()
assert config_path == temp_config_dir / "config.yaml"
def test_default_workspace_dir_is_stable_under_home(self, monkeypatch, tmp_path):
monkeypatch.setattr(Path, "home", lambda: tmp_path)
assert get_default_workspace_dir() == tmp_path / ".evoscientist" / "workspace"
# =============================================================================
# Test load/save/reset
tests/test_cron_schedule.py
+4 -4
View File
@@ -201,10 +201,10 @@ requirements = [
monkeypatch.setattr(manager, "RUNTIME", runtime)
workspace = tmp_path / "workspace-écho"
manager._write_workspace_sidecar(workspace, 12345)
assert manager._read_workspace_sidecar() == {
"workspace": str(workspace),
"pid": 12345,
}
sidecar = manager._read_workspace_sidecar()
assert sidecar is not None
assert sidecar["workspace"] == str(workspace)
assert sidecar["pid"] == 12345
runtime.pid_file.write_text(str(os.getpid()), encoding="utf-8")
tests/test_langgraph_dev_http.py
+746 -8
View File
@@ -7,15 +7,31 @@ from __future__ import annotations
from unittest.mock import patch
import pytest
from langchain_core.messages import AIMessage, HumanMessage
from starlette.testclient import TestClient
from EvoScientist.config import EvoScientistConfig
from EvoScientist.config import EvoScientistConfig, load_config, save_config
from EvoScientist.config.provider_profiles import (
load_provider_profiles,
replace_provider_profiles,
)
from EvoScientist.langgraph_dev.http import app
from EvoScientist.llm.provider_operations import (
DiscoveredProviderModel,
ProviderModelTestResult,
ProviderOperationError,
)
client = TestClient(app)
@pytest.fixture(autouse=True)
def isolate_xdg_config(tmp_path, monkeypatch):
"""Keep HTTP tests independent from the developer's saved providers."""
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
def test_get_models_returns_entries_and_default():
mock_cfg = EvoScientistConfig(
model="claude-sonnet-4-6", provider="custom-anthropic"
@@ -45,6 +61,695 @@ def test_get_models_returns_entries_and_default():
assert entry["provider"]
def test_get_models_only_returns_enabled_catalog_entries_when_configured():
mock_cfg = EvoScientistConfig(
model="chat-main",
provider="openai",
model_catalog=[
{
"provider": "openai",
"id": "chat-main",
"name": "Chat Main",
"model_id": "gpt-upstream",
"enabled": True,
},
{
"provider": "anthropic",
"id": "hidden-model",
"name": "Hidden",
"model_id": "claude-hidden",
"enabled": False,
},
],
)
with patch(
"EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg
):
body = client.get("/api/models").json()
assert body["entries"] == [
{"name": "chat-main", "model_id": "gpt-upstream", "provider": "openai"}
]
def test_get_models_empty_catalog_hides_all_builtin_models():
mock_cfg = EvoScientistConfig(model_catalog=[])
with patch(
"EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg
):
body = client.get("/api/models").json()
assert body["entries"] == []
def test_provider_profiles_api_requires_admin_token_header(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
response = client.get("/api/provider-profiles")
assert response.status_code == 403
def test_provider_profiles_api_round_trip_redacts_secret(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
payload = {
"providers": [
{
"id": "lab-openai",
"name": "Lab OpenAI",
"adapter": "openai-compatible",
"base_url": "https://llm.example.test/v1",
"api_key": "provider-secret",
"enabled": True,
"models": [
{
"id": "lab-model",
"name": "Lab Model",
"model_id": "vendor/model",
"enabled": True,
}
],
}
]
}
put_response = client.put("/api/provider-profiles", headers=headers, json=payload)
assert put_response.status_code == 200
assert "provider-secret" not in put_response.text
assert put_response.json()["providers"][0]["api_key_configured"] is True
assert "openai" in put_response.json()["reserved_provider_ids"]
get_response = client.get("/api/provider-profiles", headers=headers)
assert get_response.status_code == 200
assert get_response.json()["providers"][0]["models"][0]["id"] == "lab-model"
assert "provider-secret" not in get_response.text
def test_llm_config_api_requires_admin_token_header(monkeypatch):
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
response = client.get("/api/config")
assert response.status_code == 403
def test_llm_config_api_redacts_secrets_and_reports_env_overrides(monkeypatch):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
monkeypatch.setenv("OPENAI_API_KEY", "environment-secret")
save_config(
EvoScientistConfig(
provider="openai",
model="gpt-5.4",
openai_api_key="file-secret-1234",
default_workdir="/tmp/research",
)
)
response = client.get(
"/api/config",
headers={"X-EvoScientist-Admin-Token": "admin-secret"},
)
assert response.status_code == 200
body = response.json()
assert body["values"]["provider"] == "openai"
assert body["values"]["model"] == "gpt-5.4"
assert "openai_api_key" not in body["values"]
assert body["secrets"]["openai_api_key"] == {
"configured": True,
"hint": "...1234",
}
assert body["env_overrides"]["openai_api_key"] == "OPENAI_API_KEY"
assert "file-secret-1234" not in response.text
assert "environment-secret" not in response.text
assert "default_workdir" not in body["values"]
assert body["model_catalog"] is None
assert body["builtin_model_candidates"]
openai = next(
provider for provider in body["builtin_providers"] if provider["id"] == "openai"
)
assert openai["managed"] is False
assert openai["api_key_configured"] is True
assert "environment-secret" not in response.text
def test_llm_config_api_saves_builtin_registry_without_rewriting_legacy_secret(
monkeypatch,
):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
save_config(
EvoScientistConfig(
provider="openai",
model="chat-main",
openai_api_key="legacy-secret",
default_workdir="/tmp/research",
)
)
loaded = client.get("/api/config", headers=headers).json()
openai = next(
provider
for provider in loaded["builtin_providers"]
if provider["id"] == "openai"
)
openai.update(
{
"managed": True,
"base_url": "https://proxy.example.test/v1",
"models": [
{
"id": "chat-main",
"name": "Chat Main",
"model_id": "gpt-upstream",
"enabled": True,
}
],
}
)
response = client.patch(
"/api/config",
headers=headers,
json={
"revision": loaded["revision"],
"values": loaded["values"],
"secrets": {},
"clear_secrets": [],
"builtin_providers": [openai],
},
)
assert response.status_code == 200
body = response.json()
assert body["restart_required"] is False
assert "builtin_providers" in body["changed_fields"]
saved_profile = load_provider_profiles().builtins[0]
assert saved_profile.id == "openai"
assert saved_profile.api_key == "legacy-secret"
assert saved_profile.base_url == "https://proxy.example.test/v1"
saved_config = load_config()
assert saved_config.openai_api_key == "legacy-secret"
assert saved_config.default_workdir == "/tmp/research"
assert saved_config.model_catalog is None
models = client.get("/api/models").json()["entries"]
assert models == [
{"name": "chat-main", "model_id": "gpt-upstream", "provider": "openai"}
]
def test_first_builtin_registry_save_migrates_legacy_default_model(monkeypatch):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://environment.example.test")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
save_config(EvoScientistConfig(provider="anthropic", model="claude-sonnet-4-6"))
loaded = client.get("/api/config", headers=headers).json()
ollama = next(
provider
for provider in loaded["builtin_providers"]
if provider["id"] == "ollama"
)
ollama.update(
{
"managed": True,
"base_url": "http://127.0.0.1:11434",
"models": [],
}
)
response = client.patch(
"/api/config",
headers=headers,
json={
"revision": loaded["revision"],
"values": loaded["values"],
"secrets": {},
"clear_secrets": [],
"builtin_providers": [ollama],
},
)
assert response.status_code == 200
builtins = {profile.id: profile for profile in load_provider_profiles().builtins}
assert set(builtins) == {"anthropic", "ollama"}
assert builtins["anthropic"].models[0].id == "claude-sonnet-4-6"
assert builtins["anthropic"].base_url == ""
def test_llm_config_api_persists_model_catalog(monkeypatch):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
save_config(EvoScientistConfig(provider="openai", model="chat-main"))
revision = client.get("/api/config", headers=headers).json()["revision"]
catalog = [
{
"provider": "openai",
"id": "chat-main",
"name": "Chat Main",
"model_id": "gpt-upstream",
"enabled": True,
}
]
response = client.patch(
"/api/config",
headers=headers,
json={
"revision": revision,
"values": {},
"secrets": {},
"clear_secrets": [],
"model_catalog": catalog,
},
)
assert response.status_code == 200
assert response.json()["model_catalog"] == catalog
assert response.json()["restart_required"] is False
assert load_config().model_catalog == catalog
def test_llm_config_api_rejects_default_outside_model_catalog(monkeypatch):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
save_config(EvoScientistConfig(provider="openai", model="gpt-default"))
revision = client.get("/api/config", headers=headers).json()["revision"]
response = client.patch(
"/api/config",
headers=headers,
json={
"revision": revision,
"values": {},
"secrets": {},
"clear_secrets": [],
"model_catalog": [
{
"provider": "openai",
"id": "different-model",
"name": "Different",
"model_id": "gpt-different",
"enabled": True,
}
],
},
)
assert response.status_code == 400
assert "Default model" in response.json()["error"]
assert load_config().model_catalog is None
def test_llm_config_api_patch_preserves_unrelated_fields_and_secret_by_default(
monkeypatch,
):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
save_config(
EvoScientistConfig(
openai_api_key="saved-openai-key",
anthropic_api_key="saved-anthropic-key",
default_workdir="/tmp/research",
)
)
revision = client.get("/api/config", headers=headers).json()["revision"]
response = client.patch(
"/api/config",
headers=headers,
json={
"revision": revision,
"values": {
"provider": "openai",
"model": "gpt-5.4",
"ollama_base_url": "http://127.0.0.1:11434",
},
"secrets": {"openai_api_key": "replacement-openai-key"},
"clear_secrets": ["anthropic_api_key"],
},
)
assert response.status_code == 200
body = response.json()
assert body["restart_required"] is True
assert set(body["changed_fields"]) == {
"anthropic_api_key",
"model",
"ollama_base_url",
"openai_api_key",
"provider",
}
assert "replacement-openai-key" not in response.text
saved = load_config()
assert saved.provider == "openai"
assert saved.model == "gpt-5.4"
assert saved.ollama_base_url == "http://127.0.0.1:11434"
assert saved.openai_api_key == "replacement-openai-key"
assert saved.anthropic_api_key == ""
assert saved.default_workdir == "/tmp/research"
def test_llm_config_api_rejects_stale_revision(monkeypatch):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
save_config(EvoScientistConfig(model="claude-sonnet-4-6"))
revision = client.get("/api/config", headers=headers).json()["revision"]
save_config(EvoScientistConfig(model="gpt-5.4", provider="openai"))
response = client.patch(
"/api/config",
headers=headers,
json={
"revision": revision,
"values": {"model": "claude-opus-4-8"},
},
)
assert response.status_code == 409
assert "Reload and try again" in response.json()["error"]
assert load_config().model == "gpt-5.4"
@pytest.mark.parametrize(
("payload", "message"),
[
(
{"values": {"openai_auth_mode": "password"}},
"openai_auth_mode must be 'api_key' or 'oauth'",
),
(
{"values": {"default_workdir": "/tmp/other"}},
"Unsupported config fields: default_workdir",
),
(
{"values": {"ollama_base_url": "localhost:11434"}},
"ollama_base_url must use http:// or https://",
),
],
)
def test_llm_config_api_validates_updates(monkeypatch, payload, message):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
revision = client.get("/api/config", headers=headers).json()["revision"]
response = client.patch(
"/api/config",
headers=headers,
json={"revision": revision, **payload},
)
assert response.status_code == 400
assert message in response.json()["error"]
def test_provider_actions_api_requires_admin_token_header(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
response = client.post("/api/provider-actions", json={})
assert response.status_code == 403
def test_provider_actions_lists_models_with_saved_api_key(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
provider = {
"id": "lab-openai",
"name": "Lab OpenAI",
"adapter": "openai-compatible",
"base_url": "https://llm.example.test/v1",
"api_key": "provider-secret",
"enabled": True,
"models": [],
}
assert (
client.put(
"/api/provider-profiles",
headers=headers,
json={"providers": [provider]},
).status_code
== 200
)
provider["api_key"] = ""
async def fake_discover(profile):
assert profile.api_key == "provider-secret"
return [DiscoveredProviderModel("vendor/model", "Vendor Model")]
with patch(
"EvoScientist.langgraph_dev.http.discover_provider_models",
new=fake_discover,
):
response = client.post(
"/api/provider-actions",
headers=headers,
json={"action": "list_models", "provider": provider},
)
assert response.status_code == 200
assert response.json() == {
"models": [{"model_id": "vendor/model", "name": "Vendor Model"}]
}
def test_provider_actions_tests_model(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
async def fake_test(profile, model):
assert profile.adapter == "grok"
assert model.model_id == "grok-4"
return ProviderModelTestResult(latency_ms=123, response="OK")
with patch("EvoScientist.langgraph_dev.http.test_provider_model", new=fake_test):
response = client.post(
"/api/provider-actions",
headers=headers,
json={
"action": "test_model",
"provider": {
"id": "lab-grok",
"name": "Lab Grok",
"adapter": "grok",
"base_url": "",
"api_key": "xai-secret",
"enabled": True,
"models": [],
},
"model": {
"id": "grok",
"name": "Grok",
"model_id": "grok-4",
},
},
)
assert response.status_code == 200
assert response.json() == {"ok": True, "latency_ms": 123, "response": "OK"}
def test_provider_actions_maps_provider_failures_to_bad_gateway(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
async def fake_discover(_profile):
raise ProviderOperationError("Provider returned HTTP 401")
with patch(
"EvoScientist.langgraph_dev.http.discover_provider_models",
new=fake_discover,
):
response = client.post(
"/api/provider-actions",
headers={"X-EvoScientist-Admin-Token": "admin-secret"},
json={
"action": "list_models",
"provider": {
"id": "lab-openai",
"name": "Lab OpenAI",
"adapter": "openai-compatible",
"base_url": "https://llm.example.test/v1",
"api_key": "bad-key",
"enabled": True,
"models": [],
},
},
)
assert response.status_code == 502
assert response.json() == {"error": "Provider returned HTTP 401"}
def test_llm_config_action_discovers_models_with_effective_builtin_secret(
monkeypatch,
):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
monkeypatch.setenv("OPENAI_API_KEY", "environment-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
async def fake_discover(profile):
assert profile.id == "openai"
assert profile.adapter == "openai"
assert profile.api_key == "environment-secret"
return [DiscoveredProviderModel("gpt-discovered", "GPT Discovered")]
with patch(
"EvoScientist.langgraph_dev.http.discover_provider_models",
new=fake_discover,
):
response = client.post(
"/api/config",
headers=headers,
json={"action": "list_models", "provider": {"id": "openai"}},
)
assert response.status_code == 200
assert response.json() == {
"models": [{"model_id": "gpt-discovered", "name": "GPT Discovered"}]
}
def test_llm_config_action_tests_builtin_model_with_draft_connection(monkeypatch):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
async def fake_test(profile, model):
assert profile.id == "custom-openai"
assert profile.adapter == "openai-compatible"
assert profile.base_url == "https://proxy.example.test/v1"
assert profile.api_key == "draft-secret"
assert model.model_id == "vendor/model"
return ProviderModelTestResult(latency_ms=42, response="OK")
with patch("EvoScientist.langgraph_dev.http.test_provider_model", new=fake_test):
response = client.post(
"/api/config",
headers=headers,
json={
"action": "test_model",
"provider": {
"id": "custom-openai",
"base_url": "https://proxy.example.test/v1",
"api_key": "draft-secret",
},
"model": {
"id": "chat-main",
"name": "Chat Main",
"model_id": "vendor/model",
},
},
)
assert response.status_code == 200
assert response.json() == {"ok": True, "latency_ms": 42, "response": "OK"}
def test_default_model_api_requires_admin_token_header(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
response = client.put(
"/api/default-model",
json={"model": "gpt-5.4", "provider": "openai"},
)
assert response.status_code == 403
def test_default_model_api_persists_pair_and_preserves_config(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
save_config(EvoScientistConfig(default_workdir="/tmp/research"))
response = client.put(
"/api/default-model",
headers={"X-EvoScientist-Admin-Token": "admin-secret"},
json={"model": "gpt-5.4", "provider": "openai"},
)
assert response.status_code == 200
assert response.json() == {"default": {"name": "gpt-5.4", "provider": "openai"}}
saved = load_config()
assert saved.model == "gpt-5.4"
assert saved.provider == "openai"
assert saved.default_workdir == "/tmp/research"
def test_default_model_api_accepts_configured_dynamic_model(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
headers = {"X-EvoScientist-Admin-Token": "admin-secret"}
provider_payload = {
"providers": [
{
"id": "lab-openai",
"name": "Lab OpenAI",
"adapter": "openai-compatible",
"base_url": "https://llm.example.test/v1",
"api_key": "provider-secret",
"enabled": True,
"models": [
{
"id": "lab-model",
"name": "Lab Model",
"model_id": "vendor/model",
"enabled": True,
}
],
}
]
}
assert (
client.put(
"/api/provider-profiles", headers=headers, json=provider_payload
).status_code
== 200
)
response = client.put(
"/api/default-model",
headers=headers,
json={"model": "lab-model", "provider": "lab-openai"},
)
assert response.status_code == 200
saved = load_config()
assert (saved.model, saved.provider) == ("lab-model", "lab-openai")
def test_default_model_api_rejects_unconfigured_pair(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
response = client.put(
"/api/default-model",
headers={"X-EvoScientist-Admin-Token": "admin-secret"},
json={"model": "missing-model", "provider": "missing-provider"},
)
assert response.status_code == 400
assert "is not configured" in response.json()["error"]
def test_default_model_api_rejects_invalid_json(monkeypatch):
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret")
response = client.put(
"/api/default-model",
headers={
"X-EvoScientist-Admin-Token": "admin-secret",
"Content-Type": "application/json",
},
content="{invalid",
)
assert response.status_code == 400
assert response.json() == {"error": "Request body must be valid JSON."}
def test_entries_preserve_registry_order():
"""The picker uses position-in-list to rank providers per short name —
the JSON must preserve the order returned by ``list_models_by_provider``.
@@ -67,22 +772,55 @@ def test_entries_preserve_registry_order():
assert resp.json()["entries"] == expected
def test_default_passes_through_arbitrary_config_pair():
"""If config.yaml names a (name, provider) pair that isn't in the
registry (typo, retired model), still report it as default — the
picker labels it as the active selection regardless.
"""
def test_unavailable_default_falls_back_to_first_picker_entry():
mock_cfg = EvoScientistConfig(model="some-retired-name", provider="some-provider")
with patch(
"EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg
):
resp = client.get("/api/models")
first = resp.json()["entries"][0]
assert resp.json()["default"] == {
"name": "some-retired-name",
"provider": "some-provider",
"name": first["name"],
"provider": first["provider"],
}
def test_custom_registry_default_does_not_reuse_stale_builtin_provider():
replace_provider_profiles(
{
"providers": [
{
"id": "open",
"name": "Open proxy",
"adapter": "openai",
"base_url": "https://proxy.example.test/v1",
"api_key": "provider-secret",
"enabled": True,
"models": [
{
"id": "gpt-5.5",
"name": "GPT 5.5",
"model_id": "gpt-5.5",
"enabled": True,
}
],
}
]
}
)
mock_cfg = EvoScientistConfig(model="gpt-5.5", provider="openai")
with patch(
"EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg
):
body = client.get("/api/models").json()
assert body["entries"] == [
{"name": "gpt-5.5", "model_id": "gpt-5.5", "provider": "open"}
]
assert body["default"] == {"name": "gpt-5.5", "provider": "open"}
def test_ollama_models_appended_when_base_url_configured():
"""Mirrors the TUI ``/model`` picker: when ``ollama_base_url`` is set,
locally-pulled Ollama models are appended after the static registry
tests/test_langgraph_dev_workspace_sidecar.py
+5 -1
View File
@@ -112,7 +112,11 @@ def test_read_workspace_sidecar_round_trip(tmp_path, monkeypatch, runtime_paths)
workspace = tmp_path / "x" / "y"
manager._write_workspace_sidecar(workspace_dir=workspace, pid=42)
data = manager._read_workspace_sidecar()
assert data == {"workspace": str(workspace), "pid": 42}
assert data is not None
assert data["workspace"] == str(workspace)
assert data["pid"] == 42
assert data["api_url"] == "http://localhost:6174"
assert isinstance(data["deployment_id"], str)
def test_workspace_mismatch_error_is_runtime_error_subclass():
tests/test_langgraph_manager.py
+5
View File
@@ -294,6 +294,7 @@ class TestEnsureLanggraphDev:
with (
patch.object(manager, "is_langgraph_dev_running", return_value=False),
patch.object(manager, "start_langgraph_dev", return_value=proc) as start,
patch.object(manager.atexit, "register") as register_exit,
patch.object(
manager,
"RUNTIME",
@@ -303,10 +304,14 @@ class TestEnsureLanggraphDev:
),
),
):
runtime = manager.RUNTIME
result = manager.ensure_langgraph_dev(cfg, workspace_dir=tmp_path)
assert result is proc
start.assert_called_once()
register_exit.assert_called_once_with(
manager.stop_langgraph_dev, proc, runtime=runtime
)
assert manager.is_async_subagents_available() is True
def test_skips_when_async_and_memory_workers_disabled(
tests/test_model_command.py
+4
View File
@@ -740,6 +740,10 @@ class TestModelCommandOllamaPicker:
"""Verify Ollama discovery augments the picker entries and the sentinel
is always present when Ollama is configured."""
@pytest.fixture(autouse=True)
def _isolate_provider_registry(self, tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
def _make_ctx_and_cfg(self, *, ollama_base_url: str | None):
cfg = SimpleNamespace(
model="claude-sonnet-4-6",
tests/test_observation_memory.py
+29
View File
@@ -121,6 +121,7 @@ def _memory_source_context(
project_id: str = "P-project",
source_agent: str = "EvoScientist",
session_id: str = "thread-1",
turn_id: str | None = None,
trajectory: list[source_context.CompactMessage] | None = None,
) -> source_context.MemorySourceContext:
context_trajectory = trajectory or [{"role": "human", "content": "hi"}]
@@ -131,6 +132,7 @@ def _memory_source_context(
project_id=project_id,
source_agent=source_agent,
session_id=session_id,
turn_id=turn_id,
trajectory=context_trajectory,
trajectory_digest=source_context._trajectory_digest(context_trajectory),
)
@@ -1781,6 +1783,7 @@ def test_subagent_summary_writer_uses_worker_metadata(tmp_path, monkeypatch):
def test_memory_worker_run_payload_use_server_thread_id_and_source_metadata(
monkeypatch,
):
monkeypatch.setenv("EVOSCIENTIST_USAGE_TRACKING", "true")
monkeypatch.setattr(
memory_launch,
"_worker_workspace_dir",
@@ -1805,6 +1808,7 @@ def test_memory_worker_run_payload_use_server_thread_id_and_source_metadata(
assert kwargs["assistant_id"] == memory_launch.SUBAGENT_MEMORY_WORKER_GRAPH_ID
assert kwargs["metadata"] == {
"run_kind": "evomemory_subagent_worker",
"usage_scope": "memory",
"source_session_id": "thread-1",
"source_agent": "writing-agent",
"project_id": "P-project",
@@ -1825,6 +1829,29 @@ def test_memory_worker_run_payload_use_server_thread_id_and_source_metadata(
}
def test_memory_worker_propagates_optional_usage_turn(monkeypatch):
monkeypatch.setenv("EVOSCIENTIST_USAGE_TRACKING", "true")
monkeypatch.setattr(
memory_launch,
"_worker_workspace_dir",
lambda _workspace_dir: "/tmp/ws",
)
context = _memory_source_context(
memory_dir="/memories",
workspace_dir="/active/workspace",
turn_id="human-message-fixture",
)
kwargs = memory_launch._memory_worker_run_payload(
context=context,
thread_id="worker-thread",
)
assert kwargs["metadata"]["turn_id"] == "human-message-fixture"
assert (
kwargs["config"]["configurable"]["evomemory_source_turn_id"]
== "human-message-fixture"
)
def test_memory_worker_finish_launches_linker_for_new_observations(
tmp_path,
):
@@ -2440,6 +2467,7 @@ def test_memory_worker_skips_when_langgraph_dev_unavailable(tmp_path, monkeypatc
def test_memory_worker_marks_active_status(tmp_path, monkeypatch):
monkeypatch.setenv("EVOSCIENTIST_USAGE_TRACKING", "true")
monkeypatch.setattr(
background_runs, "default_background_run_url", lambda: "http://x"
)
@@ -2480,6 +2508,7 @@ def test_memory_worker_marks_active_status(tmp_path, monkeypatch):
assert worker_activity.memory_worker_status().is_running is True
expected_metadata = {
"run_kind": "evomemory_turn_worker",
"usage_scope": "memory",
"source_session_id": "thread-1",
"source_agent": "EvoScientist",
"project_id": "P-project",
tests/test_provider_admin.py
+42
View File
@@ -0,0 +1,42 @@
from __future__ import annotations
import os
from EvoScientist.config.provider_admin import (
get_provider_admin_token,
get_provider_admin_token_path,
)
def test_explicit_provider_admin_token_wins(monkeypatch, tmp_path):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "explicit-token")
assert get_provider_admin_token() == "explicit-token"
assert not get_provider_admin_token_path().exists()
def test_provider_admin_token_is_created_and_reused(monkeypatch, tmp_path):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
first = get_provider_admin_token()
second = get_provider_admin_token()
path = get_provider_admin_token_path()
assert first
assert first == second
assert path.read_text(encoding="utf-8").strip() == first
if os.name != "nt":
assert path.parent.stat().st_mode & 0o777 == 0o700
assert path.stat().st_mode & 0o777 == 0o600
def test_provider_admin_token_reads_existing_file(monkeypatch, tmp_path):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False)
path = get_provider_admin_token_path()
path.parent.mkdir(parents=True)
path.write_text("shared-token\n", encoding="utf-8")
assert get_provider_admin_token() == "shared-token"
tests/test_provider_operations.py
+280
View File
@@ -0,0 +1,280 @@
"""Tests for provider-specific model discovery and connectivity checks."""
from __future__ import annotations
from types import SimpleNamespace
from unittest.mock import patch
import httpx
import pytest
from EvoScientist.config.provider_profiles import ProviderModel, ProviderProfile
from EvoScientist.llm.provider_operations import (
ProviderOperationError,
discover_provider_models,
)
from EvoScientist.llm.provider_operations import (
test_provider_model as run_provider_model_test,
)
def _profile(
adapter: str,
*,
base_url: str = "",
api_key: str = "provider-secret",
) -> ProviderProfile:
return ProviderProfile(
id="test-provider",
name="Test provider",
adapter=adapter,
base_url=base_url,
api_key=api_key,
enabled=True,
models=(),
)
@pytest.mark.asyncio
async def test_openai_discovery_uses_bearer_token_and_models_endpoint():
def handler(request: httpx.Request) -> httpx.Response:
assert request.url == "https://api.openai.com/v1/models"
assert request.headers["Authorization"] == "Bearer provider-secret"
return httpx.Response(
200,
json={
"data": [
{"id": "gpt-z"},
{"id": "gpt-a", "name": "GPT A"},
]
},
)
models = await discover_provider_models(
_profile("openai"), transport=httpx.MockTransport(handler)
)
assert [(model.model_id, model.name) for model in models] == [
("gpt-a", "GPT A"),
("gpt-z", "gpt-z"),
]
@pytest.mark.asyncio
async def test_anthropic_discovery_uses_native_headers_and_pagination():
requests: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
assert request.url.path == "/v1/models"
assert request.headers["x-api-key"] == "provider-secret"
assert request.headers["anthropic-version"] == "2023-06-01"
if request.url.params.get("after_id"):
return httpx.Response(
200,
json={
"data": [{"id": "claude-opus", "display_name": "Claude Opus"}],
"has_more": False,
},
)
return httpx.Response(
200,
json={
"data": [{"id": "claude-sonnet"}],
"has_more": True,
"last_id": "claude-sonnet",
},
)
models = await discover_provider_models(
_profile("anthropic"), transport=httpx.MockTransport(handler)
)
assert len(requests) == 2
assert requests[1].url.params["after_id"] == "claude-sonnet"
assert [model.model_id for model in models] == ["claude-opus", "claude-sonnet"]
@pytest.mark.asyncio
async def test_google_discovery_filters_non_generation_models_and_paginates():
def handler(request: httpx.Request) -> httpx.Response:
assert request.url.path == "/v1beta/models"
assert request.headers["x-goog-api-key"] == "provider-secret"
if request.url.params.get("pageToken"):
return httpx.Response(
200,
json={
"models": [
{
"name": "models/gemini-2.5-pro",
"displayName": "Gemini 2.5 Pro",
"supportedGenerationMethods": ["generateContent"],
}
]
},
)
return httpx.Response(
200,
json={
"models": [
{
"name": "models/text-embedding-004",
"supportedGenerationMethods": ["embedContent"],
},
{
"name": "models/gemini-2.5-flash",
"supportedGenerationMethods": ["generateContent"],
},
],
"nextPageToken": "next-page",
},
)
models = await discover_provider_models(
_profile("google-genai"), transport=httpx.MockTransport(handler)
)
assert [(model.model_id, model.name) for model in models] == [
("gemini-2.5-flash", "models/gemini-2.5-flash"),
("gemini-2.5-pro", "Gemini 2.5 Pro"),
]
@pytest.mark.asyncio
async def test_grok_uses_xai_default_and_antigravity_uses_configured_proxy():
requests: list[httpx.Request] = []
def handler(request: httpx.Request) -> httpx.Response:
requests.append(request)
return httpx.Response(200, json={"data": [{"id": "grok-4"}]})
transport = httpx.MockTransport(handler)
await discover_provider_models(_profile("grok"), transport=transport)
await discover_provider_models(
_profile("antigravity", base_url="http://127.0.0.1:8317/v1"),
transport=transport,
)
assert str(requests[0].url) == "https://api.x.ai/v1/models"
assert str(requests[1].url) == "http://127.0.0.1:8317/v1/models"
@pytest.mark.asyncio
async def test_ollama_discovery_uses_tags_without_api_key():
def handler(request: httpx.Request) -> httpx.Response:
assert request.url == "http://127.0.0.1:11434/api/tags"
assert "Authorization" not in request.headers
return httpx.Response(
200,
json={"models": [{"name": "qwen3:8b"}, {"model": "llama3.3:latest"}]},
)
models = await discover_provider_models(
_profile("ollama", api_key=""), transport=httpx.MockTransport(handler)
)
assert [model.model_id for model in models] == ["llama3.3:latest", "qwen3:8b"]
@pytest.mark.asyncio
async def test_discovery_requires_native_provider_api_key():
with pytest.raises(ProviderOperationError, match="API key is required"):
await discover_provider_models(_profile("openai", api_key=""))
@pytest.mark.asyncio
async def test_provider_http_error_surfaces_message_without_response_body_dump():
def handler(_request: httpx.Request) -> httpx.Response:
return httpx.Response(
401,
json={"error": {"message": "Invalid credential"}, "secret": "hidden"},
)
with pytest.raises(
ProviderOperationError, match="Provider returned HTTP 401: Invalid credential"
):
await discover_provider_models(
_profile("openai"), transport=httpx.MockTransport(handler)
)
@pytest.mark.asyncio
async def test_bare_openai_origin_uses_v1_and_reports_non_json_content_type():
def handler(request: httpx.Request) -> httpx.Response:
assert request.url == "https://api.example.test/v1/models"
return httpx.Response(
200,
text="<!doctype html>",
headers={"content-type": "text/html; charset=utf-8"},
)
with pytest.raises(ProviderOperationError) as raised:
await discover_provider_models(
_profile("openai", base_url="https://api.example.test"),
transport=httpx.MockTransport(handler),
)
message = str(raised.value)
assert "https://api.example.test/v1/models" in message
assert "Content-Type: text/html" in message
assert "normally end in /v1" in message
@pytest.mark.asyncio
async def test_discovery_retries_a_transient_connection_failure():
attempts = 0
def handler(request: httpx.Request) -> httpx.Response:
nonlocal attempts
attempts += 1
if attempts == 1:
raise httpx.ConnectTimeout("temporary timeout", request=request)
return httpx.Response(200, json={"data": [{"id": "gpt-recovered"}]})
models = await discover_provider_models(
_profile("openai"), transport=httpx.MockTransport(handler)
)
assert attempts == 2
assert [model.model_id for model in models] == ["gpt-recovered"]
@pytest.mark.asyncio
async def test_model_test_builds_draft_model_and_returns_text():
class FakeChatModel:
async def ainvoke(self, prompt: str):
assert prompt == "Reply with exactly: OK"
return SimpleNamespace(content=[{"type": "text", "text": "OK"}])
profile = _profile("grok")
model = ProviderModel(
id="grok-test", name="Grok test", model_id="grok-4", enabled=True
)
with patch(
"EvoScientist.llm.models.get_profile_chat_model", return_value=FakeChatModel()
) as build_model:
result = await run_provider_model_test(profile, model)
assert result.response == "OK"
assert result.latency_ms >= 1
build_model.assert_called_once_with(
profile,
model,
max_tokens=64,
skip_auto_config=True,
)
@pytest.mark.asyncio
async def test_model_test_redacts_api_key_from_errors():
profile = _profile("openai", api_key="secret-value")
model = ProviderModel(id="test", name="Test", model_id="gpt-test")
with patch(
"EvoScientist.llm.models.get_profile_chat_model",
side_effect=ValueError("request rejected for secret-value"),
):
with pytest.raises(ProviderOperationError) as raised:
await run_provider_model_test(profile, model)
assert "secret-value" not in str(raised.value)
assert "[redacted]" in str(raised.value)
tests/test_provider_profiles.py
+453
View File
@@ -0,0 +1,453 @@
"""Tests for runtime-editable provider profiles."""
from __future__ import annotations
import os
from unittest.mock import patch
import pytest
from EvoScientist.config import EvoScientistConfig, apply_config_to_env, save_config
from EvoScientist.config.provider_profiles import (
ProviderProfileError,
get_builtin_provider_profile,
get_provider_profile_revision,
get_provider_profiles_path,
list_configured_model_entries,
load_provider_profiles,
provider_profiles_public,
replace_provider_profiles,
resolve_provider_profile_draft,
)
from EvoScientist.llm.models import get_chat_model, list_model_picker_entries
@pytest.fixture
def provider_config_dir(tmp_path, monkeypatch):
monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path))
return tmp_path / "evoscientist"
def _document(api_key: str = "secret-key") -> dict:
return {
"providers": [
{
"id": "lab-openai",
"name": "Lab OpenAI",
"adapter": "openai-compatible",
"base_url": "https://llm.example.test/v1/",
"api_key": api_key,
"enabled": True,
"models": [
{
"id": "research-model",
"name": "Research Model",
"model_id": "vendor/research-1",
"enabled": True,
},
{
"id": "disabled-model",
"name": "Disabled Model",
"model_id": "vendor/disabled",
"enabled": False,
},
],
}
]
}
def _builtin_document(api_key: str = "builtin-secret") -> dict:
return {
"builtins": [
{
"id": "openai",
"name": "OpenAI",
"adapter": "openai",
"base_url": "https://proxy.example.test/v1",
"api_key": api_key,
"auth_mode": "api_key",
"enabled": True,
"models": [
{
"id": "chat-main",
"name": "Chat Main",
"model_id": "gpt-upstream",
"enabled": True,
}
],
}
]
}
def test_replace_round_trip_and_redacts_api_key(provider_config_dir):
saved = replace_provider_profiles(_document())
assert saved.providers[0].base_url == "https://llm.example.test/v1"
assert load_provider_profiles() == saved
public = provider_profiles_public(saved)
assert "api_key" not in public["providers"][0]
assert public["providers"][0]["api_key_configured"] is True
assert public["providers"][0]["api_key_hint"] == "sec...-key"
assert "openai" in public["reserved_provider_ids"]
path = get_provider_profiles_path()
assert path == provider_config_dir / "providers.yaml"
if os.name != "nt":
assert path.parent.stat().st_mode & 0o777 == 0o700
assert path.stat().st_mode & 0o777 == 0o600
def test_v1_document_loads_and_saves_as_v2(provider_config_dir):
path = get_provider_profiles_path()
path.parent.mkdir(parents=True)
path.write_text(
"version: 1\nproviders:\n"
" - id: lab-openai\n"
" name: Lab OpenAI\n"
" adapter: openai-compatible\n"
" base_url: https://llm.example.test/v1\n"
" api_key: old-secret\n"
" enabled: true\n"
" models: []\n",
encoding="utf-8",
)
loaded = load_provider_profiles()
assert loaded.version == 2
assert loaded.builtins == ()
assert loaded.providers[0].id == "lab-openai"
replace_provider_profiles({"providers": _document()["providers"]})
assert path.read_text(encoding="utf-8").startswith("version: 2\nbuiltins:")
def test_builtin_profiles_round_trip_and_redact_secret(provider_config_dir):
saved = replace_provider_profiles(_builtin_document())
assert saved.builtins[0].id == "openai"
assert get_builtin_provider_profile("openai") == saved.builtins[0]
public = provider_profiles_public(saved)
assert public["builtins"][0]["api_key_configured"] is True
assert "builtin-secret" not in str(public)
def test_builtin_private_revision_changes_with_connection_secret(
provider_config_dir,
):
replace_provider_profiles(_builtin_document("first-secret"))
first = get_provider_profile_revision("openai")
replace_provider_profiles(_builtin_document("second-secret"))
second = get_provider_profile_revision("openai")
assert first
assert second
assert first != second
def test_custom_update_preserves_builtin_profiles(provider_config_dir):
replace_provider_profiles(_builtin_document())
saved = replace_provider_profiles(_document())
assert saved.builtins[0].id == "openai"
assert saved.providers[0].id == "lab-openai"
def test_builtin_update_preserves_custom_profiles(provider_config_dir):
replace_provider_profiles(_document())
saved = replace_provider_profiles(_builtin_document())
assert saved.builtins[0].id == "openai"
assert saved.providers[0].id == "lab-openai"
def test_empty_api_key_keeps_existing_secret(provider_config_dir):
replace_provider_profiles(_document("first-secret"))
update = _document("")
update["providers"][0]["name"] = "Renamed"
saved = replace_provider_profiles(update)
assert saved.providers[0].name == "Renamed"
assert saved.providers[0].api_key == "first-secret"
def test_public_revision_does_not_fingerprint_api_key(provider_config_dir):
first = provider_profiles_public(
replace_provider_profiles(_document("abc-first-1234"))
)["revision"]
second = provider_profiles_public(
replace_provider_profiles(_document("abc-second-1234"))
)["revision"]
assert first == second
def test_clear_api_key_is_explicit(provider_config_dir):
replace_provider_profiles(_document("first-secret"))
update = _document("")
update["providers"][0]["clear_api_key"] = True
saved = replace_provider_profiles(update)
assert saved.providers[0].api_key == ""
def test_draft_reuses_saved_api_key_without_exposing_it(provider_config_dir):
replace_provider_profiles(_document("saved-secret"))
draft = _document("")["providers"][0]
draft["models"] = []
resolved = resolve_provider_profile_draft(draft)
assert resolved.api_key == "saved-secret"
assert resolved.models == ()
def test_draft_can_explicitly_clear_saved_api_key(provider_config_dir):
replace_provider_profiles(_document("saved-secret"))
draft = _document("")["providers"][0]
draft.update({"models": [], "clear_api_key": True})
resolved = resolve_provider_profile_draft(draft)
assert resolved.api_key == ""
def test_reserved_provider_id_is_rejected(provider_config_dir):
raw = _document()
raw["providers"][0]["id"] = "openai"
with pytest.raises(ProviderProfileError, match="openai-custom"):
replace_provider_profiles(raw)
@pytest.mark.parametrize(
("adapter", "expected"),
[
("openai", "https://llm.example.test/v1"),
("openai-compatible", "https://llm.example.test/v1"),
("openrouter", "https://llm.example.test/api/v1"),
("google-genai", "https://llm.example.test/v1beta"),
("anthropic", "https://llm.example.test"),
],
)
def test_bare_provider_origin_gets_adapter_api_path(
provider_config_dir, adapter, expected
):
raw = _document()
raw["providers"][0].update(
{
"id": "lab-provider",
"adapter": adapter,
"base_url": "https://llm.example.test",
}
)
saved = replace_provider_profiles(raw)
assert saved.providers[0].base_url == expected
def test_dynamic_models_are_listed_and_disabled_models_are_hidden(
provider_config_dir,
):
replace_provider_profiles(_document())
assert list_configured_model_entries() == [
("research-model", "vendor/research-1", "lab-openai")
]
@pytest.mark.asyncio
async def test_custom_registry_replaces_static_picker_entries(provider_config_dir):
replace_provider_profiles(_document())
entries = await list_model_picker_entries(None, include_custom_ollama=False)
assert entries == [("research-model", "vendor/research-1", "lab-openai")]
@pytest.mark.asyncio
async def test_empty_registry_hides_static_picker_entries(provider_config_dir):
replace_provider_profiles({"providers": []})
entries = await list_model_picker_entries(None, include_custom_ollama=False)
assert entries == []
@pytest.mark.asyncio
async def test_managed_builtins_replace_static_picker_entries(provider_config_dir):
replace_provider_profiles(_builtin_document())
entries = await list_model_picker_entries(None, include_custom_ollama=False)
assert entries == [("chat-main", "gpt-upstream", "openai")]
def test_dynamic_profile_routes_through_selected_adapter(provider_config_dir):
replace_provider_profiles(_document())
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("research-model", provider="lab-openai")
_, kwargs = init_model.call_args
assert kwargs["model"] == "vendor/research-1"
assert kwargs["model_provider"] == "openai"
assert kwargs["base_url"] == "https://llm.example.test/v1"
assert kwargs["api_key"] == "secret-key"
assert kwargs["default_headers"]["User-Agent"] == "codex_cli_rs/0.0.0"
def test_dynamic_native_openai_adapter_uses_gateway_compatible_user_agent(
provider_config_dir,
):
raw = _document()
raw["providers"][0]["adapter"] = "openai"
replace_provider_profiles(raw)
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("research-model", provider="lab-openai")
_, kwargs = init_model.call_args
assert kwargs["model_provider"] == "openai"
assert kwargs["default_headers"]["User-Agent"] == "codex_cli_rs/0.0.0"
def test_builtin_catalog_alias_routes_to_upstream_model(provider_config_dir):
save_config(
EvoScientistConfig(
provider="openai",
model="chat-main",
model_catalog=[
{
"provider": "openai",
"id": "chat-main",
"name": "Chat Main",
"model_id": "gpt-upstream",
"enabled": True,
}
],
)
)
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("chat-main", provider="openai")
_, kwargs = init_model.call_args
assert kwargs["model"] == "gpt-upstream"
assert kwargs["model_provider"] == "openai"
def test_managed_builtin_routes_connection_without_restart(
provider_config_dir, monkeypatch
):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
replace_provider_profiles(_builtin_document())
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("chat-main", provider="openai")
_, kwargs = init_model.call_args
assert kwargs["model"] == "gpt-upstream"
assert kwargs["model_provider"] == "openai"
assert kwargs["base_url"] == "https://proxy.example.test/v1"
assert kwargs["api_key"] == "builtin-secret"
def test_environment_overrides_managed_builtin_connection(
provider_config_dir, monkeypatch
):
replace_provider_profiles(_builtin_document())
monkeypatch.setenv("OPENAI_API_KEY", "environment-secret")
monkeypatch.setenv("OPENAI_BASE_URL", "https://env.example.test/v1")
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("chat-main", provider="openai")
_, kwargs = init_model.call_args
assert kwargs["base_url"] == "https://env.example.test/v1"
assert kwargs["api_key"] == "environment-secret"
def test_managed_builtin_overrides_legacy_config_injected_environment(
provider_config_dir, monkeypatch
):
monkeypatch.delenv("OPENAI_API_KEY", raising=False)
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
apply_config_to_env(
EvoScientistConfig(
openai_api_key="legacy-secret",
)
)
replace_provider_profiles(_builtin_document("registry-secret"))
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("chat-main", provider="openai")
_, kwargs = init_model.call_args
assert kwargs["api_key"] == "registry-secret"
def test_dynamic_ollama_profile_does_not_pass_api_key(provider_config_dir):
raw = _document("unused-secret")
raw["providers"][0].update(
{
"id": "local-ollama",
"adapter": "ollama",
"base_url": "http://127.0.0.1:11434",
}
)
replace_provider_profiles(raw)
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("research-model", provider="local-ollama")
_, kwargs = init_model.call_args
assert kwargs["model_provider"] == "ollama"
assert kwargs["base_url"] == "http://127.0.0.1:11434"
assert "api_key" not in kwargs
@pytest.mark.parametrize(
("adapter", "base_url"),
[
("grok", ""),
("antigravity", "http://127.0.0.1:8317/v1"),
],
)
def test_openai_protocol_adapters_route_through_openai(
provider_config_dir, adapter, base_url
):
raw = _document()
raw["providers"][0].update(
{
"id": f"lab-{adapter}",
"adapter": adapter,
"base_url": base_url,
}
)
replace_provider_profiles(raw)
with patch("EvoScientist.llm.models.init_chat_model") as init_model:
get_chat_model("research-model", provider=f"lab-{adapter}")
_, kwargs = init_model.call_args
assert kwargs["model_provider"] == "openai"
assert kwargs["base_url"] == (base_url or "https://api.x.ai/v1")
def test_revision_changes_when_credentials_change(provider_config_dir):
replace_provider_profiles(_document("first-secret"))
first = get_provider_profile_revision("lab-openai")
replace_provider_profiles(_document("second-secret"))
second = get_provider_profile_revision("lab-openai")
assert first
assert second
assert first != second
tests/test_scheduler.py
+2 -2
View File
@@ -19,7 +19,7 @@ def test_schedules_block_lists_active_crons(monkeypatch):
monkeypatch.setattr(
crons,
"list_schedules",
lambda: [
lambda scope=None: [
{
"cron_id": "abc12345-xyz",
"schedule": "*/10 * * * *",
@@ -48,7 +48,7 @@ def test_schedules_block_empty_on_error(monkeypatch):
monkeypatch.setattr(crons, "is_available", lambda: True)
def _boom():
def _boom(scope=None):
raise RuntimeError("backend died")
monkeypatch.setattr(crons, "list_schedules", _boom)
tests/test_scheduler_tools.py
+14
View File
@@ -3,6 +3,20 @@
from unittest.mock import patch
def test_scheduler_tool_schemas_are_json_serializable():
from langchain_core.utils.function_calling import convert_to_openai_tool
from EvoScientist.middleware.scheduler import (
cancel_scheduled_task,
list_scheduled_tasks,
schedule_task,
)
for tool in (schedule_task, list_scheduled_tasks, cancel_scheduled_task):
schema = convert_to_openai_tool(tool)
assert "runtime" not in schema["function"]["parameters"].get("properties", {})
def test_schedule_task_translates_and_creates():
from EvoScientist.middleware.scheduler import schedule_task
tests/test_usage_tracking.py
+552
View File
@@ -0,0 +1,552 @@
from __future__ import annotations
import json
import os
import time
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from uuid import uuid4
import pytest
from langchain_core.language_models.fake_chat_models import FakeListChatModel
from langchain_core.messages import AIMessage, AIMessageChunk
from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, LLMResult
from pydantic import ValidationError
from EvoScientist.usage.callback import (
UsageCaptureCallback,
UsageModelIdentity,
_scope,
attach_usage_callback,
)
from EvoScientist.usage.identity import (
_load_or_create,
normalize_workspace_path_v1,
prepare_usage_environment,
workspace_identity,
workspace_identity_from_normalized,
)
from EvoScientist.usage.schema import UsageEventV1
from EvoScientist.usage.spool import UsageSpool
IDENTITY = UsageModelIdentity(
provider_profile_id="profile-a",
provider_revision="revision-a",
provider_adapter="openai",
model_alias="chat-main",
upstream_model_id="upstream-a",
)
FIXTURES = Path(
os.getenv(
"EVOSCIENTIST_USAGE_FIXTURES",
str(
Path(__file__).parents[2]
/ "EvoScientist-WebUI"
/ "docs"
/ "schemas"
/ "fixtures"
),
)
)
def _fixture_event() -> UsageEventV1:
fixture = FIXTURES / "accepted" / "confirmed.json"
return UsageEventV1.model_validate_json(fixture.read_text(encoding="utf-8"))
def _usage_environment(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
values = {
"EVOSCIENTIST_USAGE_TRACKING": "true",
"EVOSCIENTIST_USAGE_SINK_URL": "http://127.0.0.1:1/api/usage/events",
"EVOSCIENTIST_USAGE_SINK_TOKEN": "test-token",
"EVOSCIENTIST_DEPLOYMENT_ID": "11111111-1111-4111-8111-111111111111",
"EVOSCIENTIST_WORKSPACE_ID": "ws1_fixture",
"EVOSCIENTIST_USAGE_SPOOL_DIR": str(tmp_path / "spool"),
"EVOSCIENTIST_WORKSPACE_DIR": str(tmp_path),
}
for key, value in values.items():
monkeypatch.setenv(key, value)
def test_prepare_usage_environment_rejects_invalid_webui_port(tmp_path: Path) -> None:
with pytest.raises(ValueError, match="webui_port"):
prepare_usage_environment(tmp_path, webui_port=0)
@pytest.mark.parametrize("name", ["confirmed", "unknown"])
def test_shared_accepted_fixtures_match_python_contract(name: str) -> None:
fixture = FIXTURES / "accepted" / f"{name}.json"
event = UsageEventV1.model_validate_json(fixture.read_text(encoding="utf-8"))
assert event.usage_status == name
assert event.model_dump(mode="json")["completed_at"].endswith("Z")
def test_shared_rejected_fixtures_match_python_contract() -> None:
rejected = json.loads(
(FIXTURES / "rejected" / "unsafe-token-sum.json").read_text(encoding="utf-8")
)
base = json.loads(
(FIXTURES / "rejected" / rejected["base_fixture"]).read_text(encoding="utf-8")
)
base.update(rejected["patch"])
with pytest.raises(ValidationError, match="input plus output tokens"):
UsageEventV1.model_validate(base)
def test_unknown_usage_rejects_numeric_tokens() -> None:
data = _fixture_event().model_dump()
data.update(
usage_status="unknown",
input_tokens=0,
output_tokens=None,
provider_total_tokens=None,
)
with pytest.raises(ValidationError, match="unknown usage"):
UsageEventV1.model_validate(data)
def test_python_contract_rejects_missing_and_coerced_fields() -> None:
data = _fixture_event().model_dump(mode="json")
data.pop("turn_id")
with pytest.raises(ValidationError, match="turn_id"):
UsageEventV1.model_validate(data)
data = _fixture_event().model_dump(mode="json")
data["input_tokens"] = "1000"
with pytest.raises(ValidationError, match="input_tokens"):
UsageEventV1.model_validate(data)
data = _fixture_event().model_dump(mode="json")
data.update(started_at=None, observed_at=0, completed_at=0)
with pytest.raises(ValidationError, match="observed_at"):
UsageEventV1.model_validate(data)
def test_model_copy_preflight_attaches_callback_without_mutating_source(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
_usage_environment(monkeypatch, tmp_path)
source = FakeListChatModel(responses=["ok"])
with patch("EvoScientist.usage.callback.get_usage_spool") as get_spool:
tracked = attach_usage_callback(source, IDENTITY)
get_spool.assert_called_once_with()
assert tracked is not source
assert not source.callbacks
assert any(
isinstance(item, UsageCaptureCallback) for item in tracked.callbacks or []
)
selector = tracked.model_copy(
update={
"metadata": {**(tracked.metadata or {}), "usage_scope": "tool_selector"}
}
)
assert selector.metadata["usage_scope"] == "tool_selector"
def test_async_run_inherits_usage_context_only_when_tracking_enabled(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from EvoScientist.llm import patches
monkeypatch.setenv("EVOSCIENTIST_USAGE_TRACKING", "true")
monkeypatch.setattr(patches, "_read_cfg_configurable", lambda: {})
active = {
"metadata": {
"turn_id": "turn-a",
"thread_id": "thread-a",
"source_agent": "EvoScientist",
},
"configurable": {"thread_id": "thread-a"},
}
with patch("langgraph.config.get_config", return_value=active):
merged = patches._merge_runs_config_kwargs({"metadata": {"name": "writer"}})
assert merged["metadata"] == {
"turn_id": "turn-a",
"source_agent": "EvoScientist",
"source_session_id": "thread-a",
"name": "writer",
"usage_scope": "async_subagent",
}
def test_callback_emits_one_confirmed_terminal_event(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
_usage_environment(monkeypatch, tmp_path)
spool = MagicMock()
callback = UsageCaptureCallback(IDENTITY)
run_id = uuid4()
callback.on_chat_model_start(
{},
[],
run_id=run_id,
metadata={
"thread_id": "thread-a",
"turn_id": "turn-a",
"usage_scope": "tool_selector",
},
)
message = AIMessage(
content="ok",
usage_metadata={"input_tokens": 12, "output_tokens": 3, "total_tokens": 15},
response_metadata={"request_id": "provider-request"},
)
result = LLMResult(generations=[[ChatGeneration(message=message)]])
with patch("EvoScientist.usage.callback.get_usage_spool", return_value=spool):
callback.on_llm_end(result, run_id=run_id)
callback.on_llm_end(result, run_id=run_id)
spool.enqueue.assert_called_once()
event = spool.enqueue.call_args.args[0]
assert event.model_call_id == str(run_id)
assert event.scope == "tool_selector"
assert event.turn_id == "turn-a"
assert event.provider_request_id == "provider-request"
assert event.input_tokens == 12
assert event.output_tokens == 3
def test_callback_error_emits_unknown(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
_usage_environment(monkeypatch, tmp_path)
spool = MagicMock()
callback = UsageCaptureCallback(IDENTITY)
run_id = uuid4()
callback.on_chat_model_start(
{}, [], run_id=run_id, metadata={"run_kind": "scheduled_task"}
)
with patch("EvoScientist.usage.callback.get_usage_spool", return_value=spool):
callback.on_llm_error(RuntimeError("provider failed"), run_id=run_id)
event = spool.enqueue.call_args.args[0]
assert event.usage_status == "unknown"
assert event.input_tokens is None
assert event.scope == "scheduler"
def test_stream_usage_survives_terminal_error(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
_usage_environment(monkeypatch, tmp_path)
spool = MagicMock()
callback = UsageCaptureCallback(IDENTITY)
run_id = uuid4()
callback.on_chat_model_start({}, [], run_id=run_id, metadata={})
chunk = ChatGenerationChunk(
message=AIMessageChunk(
content="",
usage_metadata={"input_tokens": 8, "output_tokens": 2, "total_tokens": 10},
)
)
callback.on_llm_new_token("", run_id=run_id, chunk=chunk)
with patch("EvoScientist.usage.callback.get_usage_spool", return_value=spool):
callback.on_llm_error(RuntimeError("stream interrupted"), run_id=run_id)
event = spool.enqueue.call_args.args[0]
assert event.usage_status == "confirmed"
assert event.input_tokens == 8
assert event.output_tokens == 2
def test_callback_uses_deepagents_name_for_sync_subagent_scope(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
_usage_environment(monkeypatch, tmp_path)
spool = MagicMock()
callback = UsageCaptureCallback(IDENTITY)
run_id = uuid4()
callback.on_chat_model_start(
{}, [], run_id=run_id, metadata={"lc_agent_name": "writing-agent"}
)
with patch("EvoScientist.usage.callback.get_usage_spool", return_value=spool):
callback.on_llm_error(RuntimeError("provider failed"), run_id=run_id)
assert spool.enqueue.call_args.args[0].scope == "sync_subagent"
@pytest.mark.parametrize(
("metadata", "expected"),
[
({"usage_scope": "tool_selector"}, "tool_selector"),
({"usage_scope": "diagnostic"}, "diagnostic"),
({"usage_scope": "skill_eval"}, "skill_eval"),
({"lc_source": "summarization"}, "summarizer"),
({"run_kind": "scheduled_task"}, "scheduler"),
({"run_kind": "evomemory_autoskills"}, "autoskills"),
({"run_kind": "evomemory_turn_worker"}, "memory"),
({"lc_agent_name": "EvoScientist"}, "main"),
({"lc_agent_name": "writing-agent"}, "sync_subagent"),
({"source_session_id": "origin-thread"}, "async_subagent"),
({"thread_id": "thread-a"}, "main"),
({}, "unattributed"),
],
)
def test_scope_mapping_contract(metadata: dict, expected: str) -> None:
assert _scope(metadata) == expected
def test_provider_compatibility_replays_all_terminal_cases(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
_usage_environment(monkeypatch, tmp_path)
compatibility = json.loads(
(FIXTURES / "providers" / "compatibility.json").read_text(encoding="utf-8")
)
for provider in compatibility["providers"]:
for case in compatibility["cases"]:
identity = UsageModelIdentity(
provider_profile_id=provider["name"],
provider_revision="fixture",
provider_adapter=provider["adapter"],
model_alias="fixture",
upstream_model_id="fixture",
)
callback = UsageCaptureCallback(identity)
spool = MagicMock()
run_id = uuid4()
callback.on_chat_model_start({}, [], run_id=run_id, metadata={})
request_field = provider["request_id_field"].split(".")[-1]
if case.get("stream_usage"):
chunk = ChatGenerationChunk(
message=AIMessageChunk(
content="",
usage_metadata=case["stream_usage"],
response_metadata={request_field: "request-fixture"},
)
)
callback.on_llm_new_token("", run_id=run_id, chunk=chunk)
with patch(
"EvoScientist.usage.callback.get_usage_spool", return_value=spool
):
if case["terminal"] == "error":
callback.on_llm_error(RuntimeError("fixture error"), run_id=run_id)
else:
message = AIMessage(
content="ok",
usage_metadata=case.get("usage"),
response_metadata={request_field: "request-fixture"},
)
callback.on_llm_end(
LLMResult(generations=[[ChatGeneration(message=message)]]),
run_id=run_id,
)
event = spool.enqueue.call_args.args[0]
assert event.usage_status == case["expected_status"]
if case["expected_status"] == "confirmed":
assert event.input_tokens is not None
assert event.output_tokens is not None
def test_workspace_identity_is_stable_and_deployment_scoped(tmp_path: Path) -> None:
first = workspace_identity("11111111-1111-4111-8111-111111111111", tmp_path)
second = workspace_identity("11111111-1111-4111-8111-111111111111", tmp_path)
other = workspace_identity("22222222-2222-4222-8222-222222222222", tmp_path)
assert first == second
assert first.startswith("ws1_")
assert first != other
def test_identity_file_creation_is_atomic_across_launchers(tmp_path: Path) -> None:
identity_path = tmp_path / "deployment-id"
with ThreadPoolExecutor(max_workers=16) as pool:
values = list(
pool.map(
lambda index: _load_or_create(
identity_path, lambda: f"launcher-{index}"
),
range(64),
)
)
assert len(set(values)) == 1
assert identity_path.read_text(encoding="utf-8").strip() == values[0]
if os.name != "nt":
assert identity_path.stat().st_mode & 0o777 == 0o600
def test_callback_remains_fail_open_when_spool_write_fails(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
_usage_environment(monkeypatch, tmp_path)
callback = UsageCaptureCallback(IDENTITY)
run_id = uuid4()
callback.on_chat_model_start({}, [], run_id=run_id, metadata={})
message = AIMessage(
content="ok",
usage_metadata={"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
)
result = LLMResult(generations=[[ChatGeneration(message=message)]])
spool = MagicMock()
spool.enqueue.side_effect = OSError("disk full")
with patch("EvoScientist.usage.callback.get_usage_spool", return_value=spool):
callback.on_llm_end(result, run_id=run_id)
def test_workspace_identity_normalization_contract(tmp_path: Path) -> None:
deployment = "11111111-1111-4111-8111-111111111111"
assert (
normalize_workspace_path_v1("/tmp/research/", windows=False) == "/tmp/research"
)
assert normalize_workspace_path_v1("/", windows=False) == "/"
assert normalize_workspace_path_v1("/tmp/Cafe\u0301", windows=False) == "/tmp/Café"
assert normalize_workspace_path_v1("C:\\Research\\", windows=True) == "c:/research"
assert normalize_workspace_path_v1("C:\\", windows=True) == "c:/"
assert (
normalize_workspace_path_v1("\\\\Server\\Share\\Research\\", windows=True)
== "//server/share/research"
)
normalized = normalize_workspace_path_v1("C:\\Research\\", windows=True)
assert workspace_identity_from_normalized(deployment, normalized).startswith("ws1_")
real = tmp_path / "real"
real.mkdir()
link = tmp_path / "link"
link.symlink_to(real, target_is_directory=True)
assert workspace_identity(deployment, real) == workspace_identity(deployment, link)
posix = json.loads(
(FIXTURES / "identity" / "workspace-posix.json").read_text(encoding="utf-8")
)
assert (
workspace_identity_from_normalized(deployment, posix["normalized_path"])
== posix["expected_workspace_id"]
)
windows = json.loads(
(FIXTURES / "identity" / "workspace-windows.json").read_text(encoding="utf-8")
)
for case in windows["cases"]:
normalized_case = normalize_workspace_path_v1(case["input"], windows=True)
assert normalized_case == case["normalized_path"]
assert (
workspace_identity_from_normalized(deployment, normalized_case)
== case["expected_workspace_id"]
)
unicode_case = json.loads(
(FIXTURES / "identity" / "workspace-unicode.json").read_text(encoding="utf-8")
)
normalized_unicode = normalize_workspace_path_v1(
unicode_case["input"], windows=False
)
assert normalized_unicode == unicode_case["normalized_path"]
assert (
workspace_identity_from_normalized(deployment, normalized_unicode)
== unicode_case["expected_workspace_id"]
)
root_case = json.loads(
(FIXTURES / "identity" / "workspace-root-and-symlink.json").read_text(
encoding="utf-8"
)
)["posix_root"]
assert (
workspace_identity_from_normalized(deployment, root_case["normalized_path"])
== root_case["expected_workspace_id"]
)
def test_spool_enqueue_is_durable_without_waiting_for_http(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
_usage_environment(monkeypatch, tmp_path)
monkeypatch.setattr(UsageSpool, "_run", lambda self: self._stop.wait())
spool = UsageSpool()
try:
spool.enqueue(_fixture_event())
pending = list(spool.pending.glob("*.json"))
assert len(pending) == 1
assert json.loads(pending[0].read_text(encoding="utf-8"))["schema_version"] == 1
finally:
spool.close()
def _event_with_id(event: UsageEventV1, model_call_id: str) -> UsageEventV1:
return event.model_copy(
update={
"model_call_id": model_call_id,
"event_id": (f"{event.deployment_id}:{model_call_id}:callback_final:1"),
}
)
def test_spool_multi_worker_same_event_is_idempotent(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
_usage_environment(monkeypatch, tmp_path)
monkeypatch.setattr(UsageSpool, "_run", lambda self: self._stop.wait())
first = UsageSpool()
second = UsageSpool()
try:
event = _fixture_event()
with ThreadPoolExecutor(max_workers=2) as pool:
list(pool.map(lambda spool: spool.enqueue(event), (first, second)))
assert len(list(first.pending.glob("*.json"))) == 1
assert list(first.quarantine.glob("*.json")) == []
finally:
first.close()
second.close()
def test_spool_recovers_stale_inflight_and_rejects_at_soft_limit(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
_usage_environment(monkeypatch, tmp_path)
monkeypatch.setenv("EVOSCIENTIST_USAGE_INFLIGHT_LEASE_SECONDS", "1")
monkeypatch.setenv("EVOSCIENTIST_USAGE_SPOOL_MAX_FILES", "1")
monkeypatch.setattr(UsageSpool, "_run", lambda self: self._stop.wait())
spool = UsageSpool()
try:
event = _fixture_event()
spool.enqueue(event)
pending = next(spool.pending.glob("*.json"))
inflight = spool.inflight / pending.name
pending.replace(inflight)
old = time.time() - 5
os.utime(inflight, (old, old))
spool._recover_stale_inflight()
assert (spool.pending / inflight.name).exists()
second = _event_with_id(event, "77777777-7777-4777-8777-777777777777")
spool.enqueue(second)
assert len(list(spool.pending.glob("*.json"))) == 1
assert spool.first_loss_at is not None
assert spool.degraded_reason == "spool_soft_limit_reached"
finally:
spool.close()
@pytest.mark.parametrize(
("status_code", "response_status", "expected_location", "progressed"),
[
(200, "accepted", "deleted", True),
(200, "duplicate", "deleted", True),
(409, "conflict", "quarantine", True),
(422, "rejected", "quarantine", True),
(401, "unauthorized", "pending", False),
(429, "rate_limited", "pending", False),
(500, "error", "pending", False),
],
)
def test_sender_response_state_machine(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
status_code: int,
response_status: str,
expected_location: str,
progressed: bool,
) -> None:
_usage_environment(monkeypatch, tmp_path)
monkeypatch.setattr(UsageSpool, "_run", lambda self: self._stop.wait())
spool = UsageSpool()
response = SimpleNamespace(
status_code=status_code,
json=lambda: {"status": response_status},
)
client = SimpleNamespace(post=lambda *args, **kwargs: response)
try:
spool.enqueue(_fixture_event())
assert spool._send_one(client) is progressed
if expected_location == "deleted":
assert list(spool.pending.glob("*.json")) == []
assert list(spool.inflight.glob("*.json")) == []
else:
directory = getattr(spool, expected_location)
assert len(list(directory.glob("*.json"))) == 1
finally:
spool.close()
tests/test_workspace_cutover.py
+223
View File
@@ -0,0 +1,223 @@
"""Tests for the required-mode migration gate."""
from __future__ import annotations
from types import SimpleNamespace
import pytest
from EvoScientist.scope_registry import deployment_id_for_workspace, get_scope_registry
from EvoScientist.workspace_cutover import (
run_workspace_cutover,
verify_required_cutover,
)
class _Threads:
def __init__(self, threads: list[dict] | None = None) -> None:
self.updated: list[tuple[str, dict]] = []
self._threads = threads or [
{
"thread_id": "primary-thread",
"metadata": {"graph_id": "EvoScientist"},
}
]
def search(self, *, limit: int, offset: int = 0):
assert limit == 100
assert offset == 0
return self._threads
def update(self, thread_id: str, *, metadata: dict):
self.updated.append((thread_id, metadata))
class _Crons:
def __init__(self, crons: list[dict] | None = None) -> None:
self._crons = crons or []
self.updated: list[tuple[str, bool]] = []
def search(self, *, limit: int):
assert limit == 1000
return self._crons
def update(self, cron_id: str, *, enabled: bool):
self.updated.append((cron_id, enabled))
class _Runs:
def __init__(self, runs: dict[str, list[dict]] | None = None) -> None:
self.runs = runs or {}
self.cancelled: list[tuple[str, str]] = []
def list(self, *, thread_id: str, limit: int):
assert limit == 1000
return self.runs.get(thread_id, [])
def cancel(self, thread_id: str, run_id: str, *, wait: bool, action: str):
assert wait is True
assert action == "interrupt"
self.cancelled.append((thread_id, run_id))
for run in self.runs.get(thread_id, []):
if run["run_id"] == run_id:
run["status"] = "interrupted"
def test_cutover_writes_a_verified_passing_report(tmp_path, monkeypatch):
import EvoScientist.scope_registry as registry_module
registry_module._registry_cache.clear()
monkeypatch.delenv("EVOSCIENTIST_DEPLOYMENT_ID", raising=False)
client = SimpleNamespace(threads=_Threads(), crons=_Crons(), runs=_Runs())
report = run_workspace_cutover(
workspace_root=tmp_path, client=client, browser_sdk_gate=True
)
assert report["status"] == "passed"
assert client.threads.updated[0][1]["workspace_scope_id"]
verify_required_cutover(tmp_path)
def test_required_cutover_rejects_missing_report(tmp_path):
with pytest.raises(RuntimeError, match="workspace-cutover"):
verify_required_cutover(tmp_path)
def test_cutover_quarantines_unscoped_derived_threads(tmp_path, monkeypatch):
import EvoScientist.scope_registry as registry_module
registry_module._registry_cache.clear()
monkeypatch.delenv("EVOSCIENTIST_DEPLOYMENT_ID", raising=False)
threads = _Threads(
[
{"thread_id": "primary-thread", "metadata": {"graph_id": "EvoScientist"}},
{"thread_id": "legacy-child", "metadata": {"agent_name": "worker"}},
]
)
runs = _Runs({"legacy-child": [{"run_id": "run-child", "status": "running"}]})
report = run_workspace_cutover(
workspace_root=tmp_path,
client=SimpleNamespace(threads=threads, crons=_Crons(), runs=runs),
browser_sdk_gate=True,
)
assert report["status"] == "passed"
assert runs.cancelled == [("legacy-child", "run-child")]
assert report["quarantined_derived_threads"] == [
{
"thread_id": "legacy-child",
"cancelled_run_ids": ["run-child"],
"reason": "unscoped-derived-thread",
}
]
derived_update = next(
update for update in threads.updated if update[0] == "legacy-child"
)
assert (
derived_update[1]["workspace_quarantine"]["reason"] == "unscoped-derived-thread"
)
def test_cutover_accepts_registered_scoped_derived_thread(tmp_path, monkeypatch):
import EvoScientist.scope_registry as registry_module
registry_module._registry_cache.clear()
monkeypatch.delenv("EVOSCIENTIST_DEPLOYMENT_ID", raising=False)
deployment_id = deployment_id_for_workspace(tmp_path)
registry = get_scope_registry(tmp_path)
scope = registry.provision(deployment_id, "primary-thread")
owner = registry.register_owner(
deployment_id,
scope.scope_id,
owner_type="async_thread",
resource_id="registered-child",
parent_owner_id=scope.primary_owner_id,
state="active",
)
threads = _Threads(
[
{"thread_id": "primary-thread", "metadata": {"graph_id": "EvoScientist"}},
{
"thread_id": "registered-child",
"metadata": {
"workspace_scope_id": scope.scope_id,
"workspace_scope_owner_id": owner.owner_id,
"workspace_deployment_id": deployment_id,
},
},
]
)
report = run_workspace_cutover(
workspace_root=tmp_path,
client=SimpleNamespace(threads=threads, crons=_Crons(), runs=_Runs()),
browser_sdk_gate=True,
)
assert report["status"] == "passed"
assert report["validated_derived_threads"] == ["registered-child"]
assert report["quarantined_derived_threads"] == []
def test_cutover_quarantines_scoped_thread_without_registry_owner(
tmp_path, monkeypatch
):
import EvoScientist.scope_registry as registry_module
registry_module._registry_cache.clear()
monkeypatch.delenv("EVOSCIENTIST_DEPLOYMENT_ID", raising=False)
threads = _Threads(
[
{"thread_id": "primary-thread", "metadata": {"graph_id": "EvoScientist"}},
{
"thread_id": "forged-child",
"metadata": {
"workspace_scope_id": "00000000-0000-4000-8000-000000000001",
"workspace_scope_owner_id": "00000000-0000-4000-8000-000000000002",
},
},
]
)
report = run_workspace_cutover(
workspace_root=tmp_path,
client=SimpleNamespace(threads=threads, crons=_Crons(), runs=_Runs()),
browser_sdk_gate=True,
)
assert report["status"] == "passed"
assert report["invalid_scoped_derived_threads"][0]["thread_id"] == "forged-child"
update = next(item for item in threads.updated if item[0] == "forged-child")
assert "workspace_scope_id" not in update[1]
assert (
update[1]["workspace_quarantine"]["reason"] == "invalid-scoped-derived-thread"
)
def test_cutover_disables_scoped_cron_without_registry_owner(tmp_path, monkeypatch):
import EvoScientist.scope_registry as registry_module
registry_module._registry_cache.clear()
monkeypatch.delenv("EVOSCIENTIST_DEPLOYMENT_ID", raising=False)
crons = _Crons(
[
{
"cron_id": "forged-cron",
"metadata": {
"run_kind": "scheduled_task",
"workspace_scope_id": "00000000-0000-4000-8000-000000000001",
},
}
]
)
report = run_workspace_cutover(
workspace_root=tmp_path,
client=SimpleNamespace(threads=_Threads(), crons=crons, runs=_Runs()),
browser_sdk_gate=True,
)
assert report["status"] == "passed"
assert crons.updated == [("forged-cron", False)]
assert report["invalid_scoped_crons"][0]["cron_id"] == "forged-cron"
tests/test_workspace_maintenance.py
+116
View File
@@ -0,0 +1,116 @@
"""Lifecycle maintenance tests for scoped conversation workspaces."""
from __future__ import annotations
import os
import sqlite3
from datetime import UTC, datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
from EvoScientist.scope_registry import deployment_id_for_workspace, get_scope_registry
from EvoScientist.workspace_maintenance import run_workspace_maintenance
from EvoScientist.workspace_scope import provision_conversation_scope
class _Threads:
def __init__(self) -> None:
self.deleted: list[str] = []
def delete(self, thread_id: str) -> None:
self.deleted.append(thread_id)
def get_state(self, thread_id: str) -> dict:
return {"values": {"messages": []}}
class _Runs:
def list(self, *, thread_id: str, limit: int):
assert limit == 1000
return []
def _age_scope(workspace_root: Path, scope_id: str, when: datetime) -> None:
registry = get_scope_registry(workspace_root)
with sqlite3.connect(registry.path) as database:
database.execute(
"UPDATE scopes SET created_at = ?, updated_at = ? WHERE scope_id = ?",
(when.isoformat(), when.isoformat(), scope_id),
)
def test_maintenance_deletes_stale_draft_and_purges_old_trash(tmp_path, monkeypatch):
import EvoScientist.scope_registry as registry_module
registry_module._registry_cache.clear()
monkeypatch.delenv("EVOSCIENTIST_DEPLOYMENT_ID", raising=False)
scope = provision_conversation_scope("draft-thread", workspace_root=tmp_path)
_age_scope(tmp_path, scope.scope_id, datetime(2026, 7, 1, tzinfo=UTC))
files = tmp_path / ".evoscientist" / "conversations" / scope.scope_id / "files"
files.mkdir(parents=True, exist_ok=True)
old_trash = tmp_path / ".evoscientist" / "trash" / "old-scope"
old_trash.mkdir(parents=True)
old_timestamp = datetime(2026, 7, 1, tzinfo=UTC).timestamp()
os.utime(old_trash, (old_timestamp, old_timestamp))
threads = _Threads()
report = run_workspace_maintenance(
workspace_root=tmp_path,
client=SimpleNamespace(threads=threads, runs=_Runs()),
now=datetime(2026, 7, 10, tzinfo=UTC),
)
assert report["errors"] == []
assert report["deleted_drafts"] == ["draft-thread"]
assert report["purged_trash"] == 1
assert threads.deleted == ["draft-thread"]
registry = get_scope_registry(tmp_path)
deployment_id = deployment_id_for_workspace(tmp_path)
assert registry.get(deployment_id, scope.scope_id).state == "deleted"
assert not old_trash.exists()
assert any((tmp_path / ".evoscientist" / "trash").iterdir())
def test_maintenance_leaves_draft_with_non_primary_owner(tmp_path, monkeypatch):
import EvoScientist.scope_registry as registry_module
registry_module._registry_cache.clear()
monkeypatch.delenv("EVOSCIENTIST_DEPLOYMENT_ID", raising=False)
scope = provision_conversation_scope("draft-thread", workspace_root=tmp_path)
_age_scope(tmp_path, scope.scope_id, datetime.now(UTC) - timedelta(days=2))
registry = get_scope_registry(tmp_path)
registry.register_owner(
deployment_id_for_workspace(tmp_path),
scope.scope_id,
owner_type="unknown-legacy-owner",
state="active",
)
report = run_workspace_maintenance(
workspace_root=tmp_path,
client=SimpleNamespace(threads=_Threads(), runs=_Runs()),
)
assert report["deleted_drafts"] == []
assert report["skipped_drafts"] == ["draft-thread"]
def test_maintenance_leaves_draft_that_has_messages(tmp_path, monkeypatch):
import EvoScientist.scope_registry as registry_module
class _ThreadWithMessages(_Threads):
def get_state(self, thread_id: str) -> dict:
return {"values": {"messages": [{"content": "keep this draft"}]}}
registry_module._registry_cache.clear()
monkeypatch.delenv("EVOSCIENTIST_DEPLOYMENT_ID", raising=False)
scope = provision_conversation_scope("draft-thread", workspace_root=tmp_path)
_age_scope(tmp_path, scope.scope_id, datetime.now(UTC) - timedelta(days=2))
report = run_workspace_maintenance(
workspace_root=tmp_path,
client=SimpleNamespace(threads=_ThreadWithMessages(), runs=_Runs()),
)
assert report["deleted_drafts"] == []
assert report["skipped_drafts"] == ["draft-thread"]
tests/test_workspace_scope.py
+448
View File
@@ -0,0 +1,448 @@
"""Regression tests for conversation workspace ownership and isolation."""
from __future__ import annotations
import sqlite3
from types import SimpleNamespace
import pytest
from blockbuster import BlockBuster
from EvoScientist import paths
from EvoScientist.scope_registry import (
ScopeAccessError,
ScopeConflictError,
ScopeRegistry,
)
from EvoScientist.workspace_scope import (
DeferredScopedBackend,
create_workspace_backend,
provision_conversation_scope,
require_scoped_runtime,
verify_required_executor,
workspace_isolation_mode,
)
def test_workspace_isolation_environment_defaults_to_optional(monkeypatch):
monkeypatch.delenv("EVOSCIENTIST_WORKSPACE_ISOLATION", raising=False)
assert workspace_isolation_mode() == "optional"
def test_deployed_backend_rejects_an_unscoped_runtime():
with pytest.raises(ScopeAccessError, match="require a workspace scope"):
create_workspace_backend(
SimpleNamespace(config={}),
legacy_backend=lambda: pytest.fail("deployed run used legacy backend"),
allow_unscoped_legacy=False,
)
def test_non_deployed_backend_keeps_legacy_unscoped_runtime():
legacy_backend = object()
backend = create_workspace_backend(
SimpleNamespace(config={}),
legacy_backend=lambda: legacy_backend,
)
assert backend is legacy_backend
def test_provision_is_idempotent_and_rejects_remap(tmp_path):
registry = ScopeRegistry(tmp_path / "scope-registry.sqlite3")
first = registry.provision("deployment-a", "thread-a")
second = registry.provision("deployment-a", "thread-a")
assert first.scope_id == second.scope_id
assert first.primary_owner_id == second.primary_owner_id
with pytest.raises(ScopeConflictError):
registry.provision(
"deployment-a", "thread-a", scope_id="00000000-0000-4000-8000-000000000001"
)
def test_scope_transition_uses_compare_and_set(tmp_path):
registry = ScopeRegistry(tmp_path / "scope-registry.sqlite3")
record = registry.provision("deployment-a", "thread-a")
active = registry.transition_scope(
"deployment-a",
record.scope_id,
expected_revision=record.revision,
state="active",
)
assert active.revision == record.revision + 1
with pytest.raises(ScopeConflictError):
registry.transition_scope(
"deployment-a",
record.scope_id,
expected_revision=record.revision,
state="deleting",
)
def test_derived_owner_cannot_bind_resource_from_other_scope(tmp_path):
registry = ScopeRegistry(tmp_path / "scope-registry.sqlite3")
first = registry.provision("deployment-a", "thread-a")
second = registry.provision("deployment-a", "thread-b")
owner_a = registry.register_owner(
"deployment-a",
first.scope_id,
owner_type="async_thread",
resource_id="child-thread",
parent_owner_id=first.primary_owner_id,
state="active",
)
assert owner_a.resource_id == "child-thread"
with pytest.raises(ScopeConflictError):
registry.register_owner(
"deployment-a",
second.scope_id,
owner_type="async_thread",
resource_id="child-thread",
parent_owner_id=second.primary_owner_id,
state="active",
)
def test_turn_reservation_is_idempotent_and_primary_run_validates_runtime(tmp_path):
registry = ScopeRegistry(tmp_path / "scope-registry.sqlite3")
record = registry.provision("deployment-a", "thread-a")
first = registry.reserve_turn(
"deployment-a",
record.scope_id,
"00000000-0000-4000-8000-000000000001",
"hash-a",
)
second = registry.reserve_turn(
"deployment-a",
record.scope_id,
"00000000-0000-4000-8000-000000000001",
"hash-a",
)
assert first.run_owner_id == second.run_owner_id
assert (
registry.assert_runtime(
"deployment-a", record.scope_id, "thread-a", first.run_owner_id
).primary_thread_id
== "thread-a"
)
bound = registry.bind_turn("deployment-a", record.scope_id, first.turn_id, "run-a")
assert bound.run_id == "run-a"
assert (
registry.assert_runtime(
"deployment-a", record.scope_id, "thread-a", bound.run_owner_id
).primary_thread_id
== "thread-a"
)
with pytest.raises(ScopeConflictError):
registry.reserve_turn(
"deployment-a",
record.scope_id,
first.turn_id,
"another-request",
)
def test_run_reservations_allow_resume_within_one_logical_turn(tmp_path):
registry = ScopeRegistry(tmp_path / "scope-registry.sqlite3")
record = registry.provision("deployment-a", "thread-a")
turn_id = "00000000-0000-4000-8000-000000000001"
initial_request_id = "00000000-0000-4000-8000-000000000002"
resume_request_id = "00000000-0000-4000-8000-000000000003"
initial = registry.reserve_run(
"deployment-a", record.scope_id, initial_request_id, turn_id, "initial"
)
retry = registry.reserve_run(
"deployment-a", record.scope_id, initial_request_id, turn_id, "initial"
)
resume = registry.reserve_run(
"deployment-a",
record.scope_id,
resume_request_id,
turn_id,
"resume",
interrupt_key="interrupt-a",
)
assert retry.run_owner_id == initial.run_owner_id
assert resume.turn_id == initial.turn_id == turn_id
assert resume.run_owner_id != initial.run_owner_id
assert (
registry.bind_run(
"deployment-a", record.scope_id, resume_request_id, "resume-run"
).run_id
== "resume-run"
)
with pytest.raises(ScopeConflictError, match="run_request_id"):
registry.reserve_run(
"deployment-a", record.scope_id, initial_request_id, turn_id, "changed"
)
with pytest.raises(ScopeConflictError, match="interrupt"):
registry.reserve_run(
"deployment-a",
record.scope_id,
"00000000-0000-4000-8000-000000000004",
turn_id,
"another-resume",
interrupt_key="interrupt-a",
)
def test_registry_migrates_turn_keyed_reservations_to_run_request_ids(tmp_path):
database_path = tmp_path / "scope-registry.sqlite3"
scope_id = "00000000-0000-4000-8000-000000000010"
turn_id = "00000000-0000-4000-8000-000000000011"
with sqlite3.connect(database_path) as connection:
connection.executescript(
"""
CREATE TABLE scopes (
deployment_id TEXT NOT NULL,
scope_id TEXT NOT NULL,
primary_thread_id TEXT NOT NULL,
state TEXT NOT NULL,
revision INTEGER NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
deleted_at TEXT,
PRIMARY KEY (deployment_id, scope_id)
);
CREATE TABLE scope_turns (
deployment_id TEXT NOT NULL,
scope_id TEXT NOT NULL,
turn_id TEXT NOT NULL,
request_hash TEXT NOT NULL,
run_owner_id TEXT NOT NULL,
run_id TEXT,
state TEXT NOT NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
PRIMARY KEY (deployment_id, scope_id, turn_id)
);
"""
)
connection.execute(
"""
INSERT INTO scopes VALUES (?, ?, ?, ?, ?, ?, ?, ?)
""",
(
"deployment-a",
scope_id,
"thread-a",
"active",
1,
"created",
"updated",
None,
),
)
connection.execute(
"""
INSERT INTO scope_turns VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""",
(
"deployment-a",
scope_id,
turn_id,
"old-hash",
"owner-a",
"run-a",
"active",
"created",
"updated",
),
)
connection.execute("PRAGMA user_version = 1")
ScopeRegistry(database_path).initialize()
with sqlite3.connect(database_path) as connection:
version = connection.execute("PRAGMA user_version").fetchone()[0]
row = connection.execute(
"""
SELECT run_request_id, turn_id, run_id
FROM scope_run_requests WHERE deployment_id = ? AND scope_id = ?
""",
("deployment-a", scope_id),
).fetchone()
assert version == 2
assert row == (turn_id, turn_id, "run-a")
def test_required_runtime_never_falls_back_to_shared_root(tmp_path, monkeypatch):
monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path)
monkeypatch.setenv("EVOSCIENTIST_WORKSPACE_ISOLATION", "required")
import EvoScientist.scope_registry as registry_module
registry_module._registry_cache.clear()
record = provision_conversation_scope("thread-a", deployment_id="deployment-a")
runtime = SimpleNamespace(
config={
"configurable": {
"thread_id": "thread-a",
"workspace_scope_id": record.scope_id,
"workspace_scope_owner_id": record.primary_owner_id,
"workspace_scope_revision": record.revision,
"workspace_deployment_id": "deployment-a",
}
}
)
context = require_scoped_runtime(runtime)
assert context is not None
assert (
context.files_dir
== tmp_path / ".evoscientist" / "conversations" / record.scope_id / "files"
)
with pytest.raises(ScopeAccessError):
require_scoped_runtime(SimpleNamespace(config={}))
async def test_deferred_backend_does_not_block_the_agent_event_loop(
tmp_path, monkeypatch
):
monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path)
monkeypatch.setenv("EVOSCIENTIST_WORKSPACE_ISOLATION", "optional")
import EvoScientist.scope_registry as registry_module
import EvoScientist.workspace_scope as workspace_scope_module
registry_module._registry_cache.clear()
record = provision_conversation_scope("thread-a", deployment_id="deployment-a")
from langchain_core.runnables.config import var_child_runnable_config
token = var_child_runnable_config.set(
{
"configurable": {
"thread_id": "thread-a",
"workspace_scope_id": record.scope_id,
"workspace_scope_owner_id": record.primary_owner_id,
"workspace_scope_revision": record.revision,
"workspace_deployment_id": "deployment-a",
}
}
)
blocker = BlockBuster(scanned_modules=[workspace_scope_module, registry_module])
blocker.activate()
try:
backend = create_workspace_backend(
SimpleNamespace(),
legacy_backend=lambda: pytest.fail("scoped runtime used legacy backend"),
)
# The filesystem operation is dispatched to a thread, which does not
# inherit LangGraph's context variable.
var_child_runnable_config.reset(token)
token = None
result = await backend.als("/")
finally:
if token is not None:
var_child_runnable_config.reset(token)
blocker.deactivate()
assert isinstance(backend, DeferredScopedBackend)
assert result.error is None
def test_deferred_backend_revalidates_a_deleting_scope(tmp_path, monkeypatch):
monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path)
monkeypatch.setenv("EVOSCIENTIST_WORKSPACE_ISOLATION", "optional")
import EvoScientist.scope_registry as registry_module
registry_module._registry_cache.clear()
record = provision_conversation_scope("thread-a", deployment_id="deployment-a")
registry = registry_module.get_scope_registry(tmp_path)
active = registry.transition_scope(
"deployment-a",
record.scope_id,
expected_revision=record.revision,
state="active",
)
runtime = SimpleNamespace(
config={
"configurable": {
"thread_id": "thread-a",
"workspace_scope_id": active.scope_id,
"workspace_scope_owner_id": active.primary_owner_id,
"workspace_scope_revision": active.revision,
"workspace_deployment_id": "deployment-a",
}
}
)
backend = create_workspace_backend(
runtime,
legacy_backend=lambda: pytest.fail("scoped runtime used legacy backend"),
)
assert backend.ls("/").error is None
registry.transition_scope(
"deployment-a",
active.scope_id,
expected_revision=active.revision,
state="deleting",
)
with pytest.raises(ScopeAccessError):
backend.ls("/")
def test_required_executor_needs_an_oci_runtime(monkeypatch):
import EvoScientist.workspace_scope as scope_module
monkeypatch.setattr(scope_module.shutil, "which", lambda _: None)
with pytest.raises(RuntimeError, match="OCI runtime"):
verify_required_executor()
def test_active_lock_can_only_be_renewed_by_the_current_owner(tmp_path):
registry = ScopeRegistry(tmp_path / "scope-registry.sqlite3")
operation_id = "00000000-0000-4000-8000-000000000001"
lock = registry.acquire_lock(
"deployment-a", "workspace-cutover", operation_id, lease_seconds=30
)
assert registry.active_lock("deployment-a", "workspace-cutover") == lock
renewed = registry.renew_lock(
"deployment-a", "workspace-cutover", operation_id, lease_seconds=60
)
assert renewed.expires_at > lock.expires_at
registry.release_lock("deployment-a", "workspace-cutover", operation_id)
assert registry.active_lock("deployment-a", "workspace-cutover") is None
def test_cutover_lock_blocks_scoped_runtime_and_owner_creation(tmp_path, monkeypatch):
monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path)
monkeypatch.setenv("EVOSCIENTIST_WORKSPACE_ISOLATION", "required")
import EvoScientist.scope_registry as registry_module
registry_module._registry_cache.clear()
record = provision_conversation_scope("thread-a", deployment_id="deployment-a")
registry = registry_module.get_scope_registry(tmp_path)
registry.acquire_lock(
"deployment-a",
"workspace-cutover",
"00000000-0000-4000-8000-000000000001",
)
with pytest.raises(ScopeAccessError, match="cutover"):
registry.provision("deployment-a", "new-thread")
runtime = SimpleNamespace(
config={
"configurable": {
"thread_id": "thread-a",
"workspace_scope_id": record.scope_id,
"workspace_scope_owner_id": record.primary_owner_id,
"workspace_deployment_id": "deployment-a",
}
}
)
with pytest.raises(ScopeAccessError, match="cutover"):
require_scoped_runtime(runtime)
with pytest.raises(ScopeAccessError, match="cutover"):
registry.register_owner(
"deployment-a",
record.scope_id,
owner_type="derived_run",
parent_owner_id=record.primary_owner_id,
)
tests/test_workspace_scope_http.py
+261
View File
@@ -0,0 +1,261 @@
"""Tests for the server-only workspace scope registry routes."""
from __future__ import annotations
import asyncio
from types import SimpleNamespace
import pytest
from blockbuster import BlockBuster
from starlette.testclient import TestClient
from EvoScientist import paths
from EvoScientist.langgraph_dev.http import app
from EvoScientist.workspace_scope import DeferredScopedBackend, create_workspace_backend
def test_scope_routes_require_service_token(monkeypatch):
monkeypatch.delenv("EVOSCIENTIST_BACKEND_SERVICE_TOKEN", raising=False)
with TestClient(app) as client:
response = client.post(
"/internal/workspace-scopes/provision", json={"thread_id": "thread-a"}
)
assert response.status_code == 503
def test_scope_provision_and_lookup(tmp_path, monkeypatch):
monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path)
monkeypatch.setenv("EVOSCIENTIST_BACKEND_SERVICE_TOKEN", "scope-test-token")
monkeypatch.setenv("EVOSCIENTIST_DEPLOYMENT_ID", "deployment-test")
import EvoScientist.scope_registry as registry_module
registry_module._registry_cache.clear()
headers = {"Authorization": "Bearer scope-test-token"}
with TestClient(app) as client:
created = client.post(
"/internal/workspace-scopes/provision",
json={"thread_id": "thread-a"},
headers=headers,
)
assert created.status_code == 201
payload = created.json()
looked_up = client.get(
"/internal/workspace-scopes/by-thread/thread-a", headers=headers
)
assert looked_up.status_code == 200
assert looked_up.json()["scope_id"] == payload["scope_id"]
assert (
tmp_path / ".evoscientist" / "conversations" / payload["scope_id"] / "files"
).is_dir()
def test_scope_registry_is_constructed_off_the_asgi_event_loop(tmp_path, monkeypatch):
monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path)
monkeypatch.setenv("EVOSCIENTIST_BACKEND_SERVICE_TOKEN", "scope-test-token")
monkeypatch.setenv("EVOSCIENTIST_DEPLOYMENT_ID", "deployment-test")
import EvoScientist.scope_registry as registry_module
registry_module._registry_cache.clear()
original = registry_module.get_scope_registry
def off_event_loop_registry(*args, **kwargs):
with pytest.raises(RuntimeError):
asyncio.get_running_loop()
return original(*args, **kwargs)
monkeypatch.setattr(registry_module, "get_scope_registry", off_event_loop_registry)
headers = {"Authorization": "Bearer scope-test-token"}
with TestClient(app) as client:
created = client.post(
"/internal/workspace-scopes/provision",
json={"thread_id": "thread-a"},
headers=headers,
)
looked_up = client.get(
"/internal/workspace-scopes/by-thread/thread-a", headers=headers
)
assert created.status_code == 201
assert looked_up.status_code == 200
def test_scope_turn_routes_are_idempotent(tmp_path, monkeypatch):
monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path)
monkeypatch.setenv("EVOSCIENTIST_BACKEND_SERVICE_TOKEN", "scope-test-token")
monkeypatch.setenv("EVOSCIENTIST_DEPLOYMENT_ID", "deployment-test")
import EvoScientist.scope_registry as registry_module
registry_module._registry_cache.clear()
headers = {"Authorization": "Bearer scope-test-token"}
turn_id = "00000000-0000-4000-8000-000000000001"
with TestClient(app) as client:
scope = client.post(
"/internal/workspace-scopes/provision",
json={"thread_id": "thread-a"},
headers=headers,
).json()
reserved = client.post(
f"/internal/workspace-scopes/{scope['scope_id']}/turns/reserve",
json={"turn_id": turn_id, "request_hash": "hash-a"},
headers=headers,
)
assert reserved.status_code == 201
bound = client.patch(
f"/internal/workspace-scopes/{scope['scope_id']}/turns/{turn_id}",
json={"run_id": "run-a"},
headers=headers,
)
assert bound.status_code == 200
retried = client.post(
f"/internal/workspace-scopes/{scope['scope_id']}/turns/reserve",
json={"turn_id": turn_id, "request_hash": "hash-a"},
headers=headers,
)
assert retried.status_code == 201
assert retried.json()["run_id"] == "run-a"
def test_scope_run_routes_support_approval_resume_and_dedupe_interrupts(
tmp_path, monkeypatch
):
monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path)
monkeypatch.setenv("EVOSCIENTIST_BACKEND_SERVICE_TOKEN", "scope-test-token")
monkeypatch.setenv("EVOSCIENTIST_DEPLOYMENT_ID", "deployment-test")
import EvoScientist.scope_registry as registry_module
registry_module._registry_cache.clear()
headers = {"Authorization": "Bearer scope-test-token"}
turn_id = "00000000-0000-4000-8000-000000000001"
initial_request_id = "00000000-0000-4000-8000-000000000002"
resume_request_id = "00000000-0000-4000-8000-000000000003"
duplicate_interrupt_request_id = "00000000-0000-4000-8000-000000000004"
with TestClient(app) as client:
scope = client.post(
"/internal/workspace-scopes/provision",
json={"thread_id": "thread-a"},
headers=headers,
).json()
initial = client.post(
f"/internal/workspace-scopes/{scope['scope_id']}/runs/reserve",
json={
"run_request_id": initial_request_id,
"turn_id": turn_id,
"request_hash": "initial",
},
headers=headers,
)
resume = client.post(
f"/internal/workspace-scopes/{scope['scope_id']}/runs/reserve",
json={
"run_request_id": resume_request_id,
"turn_id": turn_id,
"request_hash": "approve",
"interrupt_key": "interrupt-a",
},
headers=headers,
)
retried_resume = client.post(
f"/internal/workspace-scopes/{scope['scope_id']}/runs/reserve",
json={
"run_request_id": resume_request_id,
"turn_id": turn_id,
"request_hash": "approve",
"interrupt_key": "interrupt-a",
},
headers=headers,
)
duplicate_interrupt = client.post(
f"/internal/workspace-scopes/{scope['scope_id']}/runs/reserve",
json={
"run_request_id": duplicate_interrupt_request_id,
"turn_id": turn_id,
"request_hash": "another-decision",
"interrupt_key": "interrupt-a",
},
headers=headers,
)
assert initial.status_code == 201
assert resume.status_code == 201
assert initial.json()["run_owner_id"] != resume.json()["run_owner_id"]
assert retried_resume.status_code == 201
assert retried_resume.json()["run_owner_id"] == resume.json()["run_owner_id"]
assert duplicate_interrupt.status_code == 409
assert duplicate_interrupt.json()["code"] == "interrupt_already_resolved"
async def test_reserved_turn_creates_an_async_safe_scoped_backend(
tmp_path, monkeypatch
):
monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path)
monkeypatch.setenv("EVOSCIENTIST_BACKEND_SERVICE_TOKEN", "scope-test-token")
monkeypatch.setenv("EVOSCIENTIST_DEPLOYMENT_ID", "deployment-test")
import EvoScientist.scope_registry as registry_module
import EvoScientist.workspace_scope as workspace_scope_module
registry_module._registry_cache.clear()
headers = {"Authorization": "Bearer scope-test-token"}
turn_id = "00000000-0000-4000-8000-000000000001"
with TestClient(app) as client:
scope = client.post(
"/internal/workspace-scopes/provision",
json={"thread_id": "thread-a"},
headers=headers,
).json()
reservation = client.post(
f"/internal/workspace-scopes/{scope['scope_id']}/turns/reserve",
json={"turn_id": turn_id, "request_hash": "hash-a"},
headers=headers,
).json()
runtime = SimpleNamespace(
config={
"configurable": {
"thread_id": "thread-a",
"workspace_scope_id": scope["scope_id"],
"workspace_scope_owner_id": reservation["run_owner_id"],
"workspace_scope_revision": scope["revision"],
"workspace_deployment_id": scope["deployment_id"],
}
}
)
blocker = BlockBuster(scanned_modules=[workspace_scope_module, registry_module])
blocker.activate()
try:
backend = create_workspace_backend(
runtime,
legacy_backend=lambda: pytest.fail("scoped runtime used legacy backend"),
)
result = await backend.als("/")
finally:
blocker.deactivate()
assert isinstance(backend, DeferredScopedBackend)
assert result.error is None
def test_scope_owner_list_and_cutover_lock_routes(tmp_path, monkeypatch):
monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path)
monkeypatch.setenv("EVOSCIENTIST_BACKEND_SERVICE_TOKEN", "scope-test-token")
monkeypatch.setenv("EVOSCIENTIST_DEPLOYMENT_ID", "deployment-test")
import EvoScientist.scope_registry as registry_module
registry_module._registry_cache.clear()
headers = {"Authorization": "Bearer scope-test-token"}
with TestClient(app) as client:
created = client.post(
"/internal/workspace-scopes/provision",
json={"thread_id": "thread-a"},
headers=headers,
).json()
owners = client.get(
f"/internal/workspace-scopes/{created['scope_id']}/owners", headers=headers
)
lock = client.get("/internal/workspace-scopes/cutover-lock", headers=headers)
assert owners.status_code == 200
assert owners.json()["owners"][0]["owner_type"] == "primary_thread"
assert lock.status_code == 200
assert lock.json()["active"] is False
uv.lock
Generated
+2
View File
@@ -965,6 +965,7 @@ dependencies = [
{ name = "nest-asyncio" },
{ name = "prompt-toolkit" },
{ name = "psutil" },
{ name = "pydantic" },
{ name = "python-dotenv" },
{ name = "pyyaml" },
{ name = "questionary" },
@@ -1076,6 +1077,7 @@ requires-dist = [
{ name = "psutil", specifier = ">=6.0" },
{ name = "pycryptodome", marker = "extra == 'all-channels'", specifier = ">=3.20" },
{ name = "pycryptodome", marker = "extra == 'wechat'", specifier = ">=3.20" },
{ name = "pydantic", specifier = ">=2.10" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" },
{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.0" },
{ name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0" },