From 38668c4ce5128e82204b32e10448aa9522b0bfcc Mon Sep 17 00:00:00 2001 From: m4 Date: Sun, 19 Jul 2026 12:17:18 +0800 Subject: [PATCH] feat: add workspace isolation and provider administration --- .env.example | 38 + .github/workflows/test.yml | 27 +- EvoScientist/EvoScientist.py | 79 +- EvoScientist/config/__init__.py | 23 + EvoScientist/config/provider_admin.py | 70 + EvoScientist/config/provider_profiles.py | 595 +++++++ EvoScientist/config/settings.py | 148 +- EvoScientist/cron/schedule.py | 108 +- EvoScientist/deploy/server.py | 110 +- EvoScientist/deploy/webui.py | 42 +- EvoScientist/langgraph_dev/http.py | 1393 ++++++++++++++++- EvoScientist/langgraph_dev/manager.py | 56 +- EvoScientist/llm/models.py | 375 ++++- EvoScientist/llm/patches.py | 114 +- EvoScientist/llm/provider_operations.py | 519 ++++++ EvoScientist/memory/launch.py | 30 +- EvoScientist/memory/source_context.py | 16 + EvoScientist/middleware/ask_user.py | 119 +- EvoScientist/middleware/configurable_model.py | 7 +- EvoScientist/middleware/scheduler.py | 55 +- EvoScientist/middleware/tool_selector.py | 25 +- EvoScientist/scope_registry.py | 1274 +++++++++++++++ EvoScientist/usage/__init__.py | 6 + EvoScientist/usage/callback.py | 356 +++++ EvoScientist/usage/identity.py | 122 ++ EvoScientist/usage/schema.py | 170 ++ EvoScientist/usage/spool.py | 540 +++++++ EvoScientist/workspace_cutover.py | 417 +++++ EvoScientist/workspace_maintenance.py | 168 ++ EvoScientist/workspace_scope.py | 541 +++++++ README.md | 28 + README.zh-CN.md | 23 + pyproject.toml | 1 + scripts/benchmark_usage_spool.py | 127 ++ scripts/workspace_cutover.py | 49 + scripts/workspace_maintenance.py | 35 + tests/test_ask_user.py | 137 ++ tests/test_cli_deploy.py | 127 +- tests/test_config.py | 26 + tests/test_cron_schedule.py | 8 +- tests/test_langgraph_dev_http.py | 754 ++++++++- tests/test_langgraph_dev_workspace_sidecar.py | 6 +- tests/test_langgraph_manager.py | 5 + tests/test_model_command.py | 4 + tests/test_observation_memory.py | 29 + tests/test_provider_admin.py | 42 + tests/test_provider_operations.py | 280 ++++ tests/test_provider_profiles.py | 453 ++++++ tests/test_scheduler.py | 4 +- tests/test_scheduler_tools.py | 14 + tests/test_usage_tracking.py | 552 +++++++ tests/test_workspace_cutover.py | 223 +++ tests/test_workspace_maintenance.py | 116 ++ tests/test_workspace_scope.py | 448 ++++++ tests/test_workspace_scope_http.py | 261 +++ uv.lock | 2 + 56 files changed, 11090 insertions(+), 207 deletions(-) create mode 100644 EvoScientist/config/provider_admin.py create mode 100644 EvoScientist/config/provider_profiles.py create mode 100644 EvoScientist/llm/provider_operations.py create mode 100644 EvoScientist/scope_registry.py create mode 100644 EvoScientist/usage/__init__.py create mode 100644 EvoScientist/usage/callback.py create mode 100644 EvoScientist/usage/identity.py create mode 100644 EvoScientist/usage/schema.py create mode 100644 EvoScientist/usage/spool.py create mode 100644 EvoScientist/workspace_cutover.py create mode 100644 EvoScientist/workspace_maintenance.py create mode 100644 EvoScientist/workspace_scope.py create mode 100644 scripts/benchmark_usage_spool.py create mode 100644 scripts/workspace_cutover.py create mode 100644 scripts/workspace_maintenance.py create mode 100644 tests/test_provider_admin.py create mode 100644 tests/test_provider_operations.py create mode 100644 tests/test_provider_profiles.py create mode 100644 tests/test_usage_tracking.py create mode 100644 tests/test_workspace_cutover.py create mode 100644 tests/test_workspace_maintenance.py create mode 100644 tests/test_workspace_scope.py create mode 100644 tests/test_workspace_scope_http.py diff --git a/.env.example b/.env.example index 6cf33b5..d4db5c6 100644 --- a/.env.example +++ b/.env.example @@ -30,3 +30,41 @@ OLLAMA_BASE_URL= # http://localhost:11434 (default) # Web search (optional) TAVILY_API_KEY= # app.tavily.com + +# WebUI provider editor. Local processes running as the same OS user share +# ~/.config/evoscientist/provider-admin-token automatically. For different +# hosts, users, or containers, set the same random value on both processes. +# Integrated `EvoSci` WebUI mode also passes a shared token automatically. +EVOSCIENTIST_PROVIDER_ADMIN_TOKEN= + +# WebUI conversation workspace policy. EVOSCIENTIST_WORKSPACE_DIR is the +# deployment root, not a per-conversation directory. In isolated modes each +# conversation is stored under /.evoscientist/conversations//. +# +# EVOSCIENTIST_WORKSPACE_ISOLATION accepts exactly: +# - legacy: all WebUI conversations share the deployment root. Compatibility +# rollback only; files are visible to every conversation using this deployment. +# - optional: default. New WebUI conversations receive isolated scope folders; +# missing Registry/token/scope fails the request instead of silently sharing. +# - required: isolated scopes plus strict runtime validation. It requires a +# completed cutover and a verified OCI executor; no legacy fallback exists. +# +# This is a deployment-startup security setting. Change it only during a +# maintenance window, restart backend and WebUI afterwards, and never use it to +# convert an existing conversation between shared and isolated directories. +EVOSCIENTIST_WORKSPACE_DIR= +EVOSCIENTIST_WORKSPACE_ISOLATION=optional +# Required mode supports only a single-host Registry topology in v1. +EVOSCIENTIST_SCOPE_REGISTRY_TOPOLOGY=single-host +# Required mode: use a pinned image digest, preserve single-host topology, and +# keep the Code Interpreter disabled unless its scoped implementation is enabled. +# Do not put EVOSCIENTIST_BACKEND_SERVICE_TOKEN here for a same-host `EvoSci +# deploy`: it is generated and passed privately at startup. +# EVOSCIENTIST_WORKSPACE_ISOLATION=required +# EVOSCIENTIST_STRICT_EXECUTOR=oci +# EVOSCIENTIST_STRICT_EXECUTOR_IMAGE=registry.example/evoscientist-runtime@sha256:replace-with-verified-digest +# EVOSCIENTIST_STRICT_CODE_INTERPRETER=disabled + +# Conversation workspace isolation retention defaults (used by workspace_maintenance.py). +EVOSCIENTIST_DRAFT_WORKSPACE_TTL_HOURS=24 +EVOSCIENTIST_WORKSPACE_TRASH_RETENTION_DAYS=7 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9bbb96f..cc1dc75 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,11 +16,16 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest] python-version: ["3.11", "3.12"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v5 + - name: Check out shared usage fixtures + uses: actions/checkout@v5 + with: + repository: EvoScientist/EvoScientist-WebUI + path: EvoScientist-WebUI - uses: astral-sh/setup-uv@v6 with: python-version: ${{ matrix.python-version }} @@ -29,3 +34,23 @@ jobs: run: uv sync --dev - name: Run pytest run: uv run pytest -v --timeout=30 + env: + EVOSCIENTIST_USAGE_FIXTURES: ${{ github.workspace }}/EvoScientist-WebUI/docs/schemas/fixtures + + usage-spool-benchmark: + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v5 + - uses: astral-sh/setup-uv@v6 + with: + python-version: "3.11" + cache-dependency-glob: "**/pyproject.toml" + - name: Install dependencies + run: uv sync --dev + - name: Verify durable spool latency + run: uv run python scripts/benchmark_usage_spool.py diff --git a/EvoScientist/EvoScientist.py b/EvoScientist/EvoScientist.py index 326e1e1..eb5a717 100644 --- a/EvoScientist/EvoScientist.py +++ b/EvoScientist/EvoScientist.py @@ -341,9 +341,15 @@ def _inject_subagent_middleware( ToolErrorHandlerMiddleware(), ContextOverflowMapperMiddleware(), ] - if memory_controls.memory_enabled: + if ( + memory_controls.memory_enabled + and cfg.workspace_isolation != "required" + ): middleware.append(memory_middleware) - if memory_controls.worker_needed(MemoryObservationTarget.SUBAGENT_WORKER): + if ( + memory_controls.worker_needed(MemoryObservationTarget.SUBAGENT_WORKER) + and cfg.workspace_isolation != "required" + ): middleware.append( create_memory_lifecycle_middleware( memory_dir, @@ -486,7 +492,9 @@ def _build_base_kwargs( tool_registry = {"think_tool": think_tool} if os.environ.get("TAVILY_API_KEY"): tool_registry["tavily_search"] = tavily_search - base_tools = [think_tool, skill_manager] + base_tools = [think_tool] + if cfg.workspace_isolation != "required": + base_tools.append(skill_manager) subs = load_subagents( SUBAGENTS_CONFIG, @@ -548,7 +556,9 @@ def load_mcp_and_build_kwargs( tool_registry = {"think_tool": think_tool} if os.environ.get("TAVILY_API_KEY"): tool_registry["tavily_search"] = tavily_search - base_tools = [think_tool, skill_manager] + base_tools = [think_tool] + if cfg.workspace_isolation != "required": + base_tools.append(skill_manager) # Fresh tool registry — start from base tools + MCP tools registry = dict(tool_registry) @@ -594,8 +604,8 @@ def load_mcp_and_build_kwargs( # ============================================================================= -def _get_default_backend(): - """Build the default composite backend from current paths.""" +def _get_legacy_backend(): + """Build the deployment-root backend used by CLI and legacy mode only.""" from deepagents.backends import CompositeBackend from .backends import ( @@ -637,6 +647,36 @@ def _get_default_backend(): ) +def _get_default_backend(): + """Return a runtime-scoped backend factory when isolation is enabled.""" + from .workspace_cutover import verify_required_cutover + from .workspace_scope import ( + create_workspace_backend_factory, + verify_required_executor, + ) + + cfg = _ensure_config() + if cfg.workspace_isolation == "legacy": + return _get_legacy_backend() + if cfg.workspace_isolation == "required" and cfg.dangerous_mode: + raise RuntimeError("dangerous_mode is incompatible with required workspace isolation") + if cfg.workspace_isolation == "required": + verify_required_cutover(_paths_mod.WORKSPACE_ROOT) + verify_required_executor() + + return create_workspace_backend_factory( + _get_legacy_backend, + dangerous=cfg.dangerous_mode, + # The CLI and its stripped async-subagent service keep their configured + # shared workspace. The WebUI deployment must receive a scope from the + # trusted WebUI/API boundary instead. + allow_unscoped_legacy=os.environ.get( + "EVOSCIENTIST_DEPLOY_MODE", "" + ).lower() + != "full", + ) + + def _get_default_middleware( *, for_async_subagent: bool = False, @@ -740,17 +780,27 @@ def _get_default_middleware( ), # Interpreter prompt must land before runtime/memory context, so this # middleware sits ahead of runtime_context in the stack. - create_code_interpreter_middleware( - timeout=cfg.code_interpreter_timeout, - max_result_chars=cfg.code_interpreter_max_result_chars, + *( + [] + if cfg.workspace_isolation == "required" + and cfg.strict_code_interpreter == "disabled" + else [ + create_code_interpreter_middleware( + timeout=cfg.code_interpreter_timeout, + max_result_chars=cfg.code_interpreter_max_result_chars, + ) + ] ), ] if cfg.enable_scheduler and not for_async_subagent: mw.append(create_scheduler_middleware()) mw.append(create_runtime_context_middleware()) - if memory_controls.memory_enabled: + if memory_controls.memory_enabled and cfg.workspace_isolation != "required": mw.append(memory_middleware) - if memory_controls.worker_needed(worker_target): + if ( + memory_controls.worker_needed(worker_target) + and cfg.workspace_isolation != "required" + ): mw.append( create_memory_lifecycle_middleware( memory_dir, @@ -770,7 +820,7 @@ def _get_default_middleware( # Background-process tools (run_in_background / check_process / stop_process / # list_processes) — main agent only. Async sub-agents run on langgraph-dev and # must not spawn local OS processes. - if not for_async_subagent: + if not for_async_subagent and cfg.workspace_isolation != "required": from .middleware.background import BackgroundExecutionMiddleware mw.append(BackgroundExecutionMiddleware()) @@ -824,7 +874,10 @@ def _get_default_agent(): ) ) - if os.environ.get("EVOSCIENTIST_DEPLOY_MODE", "").lower() == "stripped": + if ( + os.environ.get("EVOSCIENTIST_DEPLOY_MODE", "").lower() == "stripped" + or cfg.workspace_isolation == "required" + ): kwargs = _build_base_kwargs( be, mw, diff --git a/EvoScientist/config/__init__.py b/EvoScientist/config/__init__.py index fcd1383..ac6af28 100644 --- a/EvoScientist/config/__init__.py +++ b/EvoScientist/config/__init__.py @@ -8,6 +8,16 @@ The onboard module is loaded lazily because it pulls in heavy dependencies (langchain, llm) that are not needed for normal config operations. """ +from .provider_profiles import ( + ProviderModel, + ProviderProfile, + ProviderProfileError, + ProviderProfiles, + get_builtin_provider_profile, + get_provider_profiles_path, + load_provider_profiles, + save_provider_profiles, +) from .settings import ( EvoScientistConfig, MemoryControls, @@ -19,7 +29,9 @@ from .settings import ( get_config_dir, get_config_path, get_config_value, + get_default_workspace_dir, get_effective_config, + is_config_applied_env, list_config, load_config, reset_config, @@ -34,18 +46,29 @@ __all__ = [ "MemoryObservationWriter", "MemorySkillSynthesisCadence", "MemorySkillSynthesisMode", + # provider profiles + "ProviderModel", + "ProviderProfile", + "ProviderProfileError", + "ProviderProfiles", "apply_config_to_env", + "get_builtin_provider_profile", # settings "get_config_dir", "get_config_path", "get_config_value", + "get_default_workspace_dir", "get_effective_config", + "get_provider_profiles_path", + "is_config_applied_env", "list_config", "load_config", + "load_provider_profiles", "reset_config", # onboard (lazy) "run_onboard", "save_config", + "save_provider_profiles", "set_config_value", ] diff --git a/EvoScientist/config/provider_admin.py b/EvoScientist/config/provider_admin.py new file mode 100644 index 0000000..d9efcd5 --- /dev/null +++ b/EvoScientist/config/provider_admin.py @@ -0,0 +1,70 @@ +"""Shared local authentication token for WebUI model administration.""" + +from __future__ import annotations + +import os +import secrets +import threading +from pathlib import Path + +from .settings import get_config_dir + +_TOKEN_FILE_NAME = "provider-admin-token" +_TOKEN_LOCK = threading.RLock() +_cached_file_token: tuple[Path, str] | None = None + + +def get_provider_admin_token_path() -> Path: + """Return the user-scoped provider administration token path.""" + return get_config_dir() / _TOKEN_FILE_NAME + + +def _read_token(path: Path) -> str: + try: + return path.read_text(encoding="utf-8").strip() + except OSError: + return "" + + +def get_provider_admin_token() -> str: + """Return an explicit token or create a shared local token when possible. + + Separate local backend and WebUI processes share the user-scoped token file. + Remote/container deployments can set ``EVOSCIENTIST_PROVIDER_ADMIN_TOKEN`` + explicitly when the processes do not share a filesystem. + """ + explicit = os.environ.get("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "").strip() + if explicit: + return explicit + + path = get_provider_admin_token_path() + global _cached_file_token + with _TOKEN_LOCK: + if _cached_file_token is not None and _cached_file_token[0] == path: + return _cached_file_token[1] + + token = _read_token(path) + if not token: + token = secrets.token_urlsafe(32) + try: + path.parent.mkdir(parents=True, exist_ok=True) + try: + path.parent.chmod(0o700) + except OSError: + pass + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as file: + file.write(f"{token}\n") + except FileExistsError: + token = _read_token(path) + except OSError: + return "" + + if not token: + return "" + try: + path.chmod(0o600) + except OSError: + pass + _cached_file_token = (path, token) + return token diff --git a/EvoScientist/config/provider_profiles.py b/EvoScientist/config/provider_profiles.py new file mode 100644 index 0000000..c8a4eb6 --- /dev/null +++ b/EvoScientist/config/provider_profiles.py @@ -0,0 +1,595 @@ +"""Persistent, runtime-editable LLM provider profiles. + +The registry stores managed built-in connections and custom provider profiles. +``config.yaml`` remains backward compatible with the onboarding flow and acts +as a legacy fallback plus the default provider/model pointer. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import tempfile +import threading +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit, urlunsplit + +import yaml + +from .settings import get_config_dir + +PROVIDER_PROFILES_VERSION = 2 +SUPPORTED_PROVIDER_ADAPTERS = ( + "openai", + "anthropic", + "google-genai", + "grok", + "openrouter", + "nvidia", + "antigravity", + "openai-compatible", + "anthropic-compatible", + "ollama", +) + +_ID_PATTERN = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$") +_BASE_URL_REQUIRED_ADAPTERS = { + "openai-compatible", + "anthropic-compatible", + "antigravity", + "ollama", +} +_DEFAULT_API_PATHS = { + "openai": "/v1", + "openai-compatible": "/v1", + "grok": "/v1", + "openrouter": "/api/v1", + "nvidia": "/v1", + "antigravity": "/v1", + "google-genai": "/v1beta", +} +_RESERVED_PROVIDER_IDS = { + "anthropic", + "openai", + "google-genai", + "minimax", + "nvidia", + "siliconflow", + "openrouter", + "zhipu", + "zhipu-code", + "volcengine", + "dashscope", + "dashscope-code", + "deepseek", + "moonshot", + "kimi-coding", + "custom-openai", + "custom-anthropic", + "ollama", +} +_STORE_LOCK = threading.RLock() + + +class ProviderProfileError(ValueError): + """Raised when a provider profile document or selection is invalid.""" + + +@dataclass(frozen=True) +class ProviderModel: + id: str + name: str + model_id: str + enabled: bool = True + + +@dataclass(frozen=True) +class ProviderProfile: + id: str + name: str + adapter: str + base_url: str + api_key: str + enabled: bool + models: tuple[ProviderModel, ...] + auth_mode: str = "api_key" + + +@dataclass(frozen=True) +class ProviderProfiles: + version: int = PROVIDER_PROFILES_VERSION + builtins: tuple[ProviderProfile, ...] = () + providers: tuple[ProviderProfile, ...] = () + + +def normalize_provider_base_url(adapter: str, base_url: str) -> str: + """Add the adapter's standard API path when only an origin is supplied.""" + normalized = base_url.strip().rstrip("/") + default_path = _DEFAULT_API_PATHS.get(adapter) + if not normalized or not default_path: + return normalized + + parsed = urlsplit(normalized) + if parsed.path not in {"", "/"} or parsed.query or parsed.fragment: + return normalized + return urlunsplit((parsed.scheme, parsed.netloc, default_path, "", "")) + + +def get_provider_profiles_path() -> Path: + """Return the user-scoped provider profile path.""" + return get_config_dir() / "providers.yaml" + + +def _required_string( + raw: dict[str, Any], + key: str, + *, + context: str, + max_length: int, +) -> str: + value = raw.get(key) + if not isinstance(value, str) or not value.strip(): + raise ProviderProfileError(f"{context}.{key} is required.") + value = value.strip() + if len(value) > max_length: + raise ProviderProfileError( + f"{context}.{key} must be at most {max_length} characters." + ) + return value + + +def _validate_id(value: str, *, context: str) -> str: + if not _ID_PATTERN.fullmatch(value): + raise ProviderProfileError( + f"{context} must start with a lowercase letter or digit and contain only " + "lowercase letters, digits, dots, underscores, or hyphens." + ) + return value + + +def _parse_model(raw: Any, *, provider_id: str, index: int) -> ProviderModel: + context = f"providers[{provider_id}].models[{index}]" + if not isinstance(raw, dict): + raise ProviderProfileError(f"{context} must be an object.") + model_id = _validate_id( + _required_string(raw, "id", context=context, max_length=64), + context=f"{context}.id", + ) + return ProviderModel( + id=model_id, + name=_required_string(raw, "name", context=context, max_length=120), + model_id=_required_string(raw, "model_id", context=context, max_length=300), + enabled=bool(raw.get("enabled", True)), + ) + + +def _parse_profile(raw: Any, *, index: int, builtin: bool = False) -> ProviderProfile: + collection = "builtins" if builtin else "providers" + context = f"{collection}[{index}]" + if not isinstance(raw, dict): + raise ProviderProfileError(f"{context} must be an object.") + + provider_id = _validate_id( + _required_string(raw, "id", context=context, max_length=64), + context=f"{context}.id", + ) + if builtin and provider_id not in _RESERVED_PROVIDER_IDS: + raise ProviderProfileError( + f"Built-in provider ID {provider_id!r} is not recognized." + ) + if not builtin and provider_id in _RESERVED_PROVIDER_IDS: + raise ProviderProfileError( + f"Provider ID {provider_id!r} is reserved for a built-in provider. " + f"Use a unique ID such as {f'{provider_id}-custom'!r}." + ) + adapter = _required_string(raw, "adapter", context=context, max_length=64) + if adapter not in SUPPORTED_PROVIDER_ADAPTERS: + raise ProviderProfileError( + f"{context}.adapter must be one of: " + f"{', '.join(SUPPORTED_PROVIDER_ADAPTERS)}." + ) + + base_url_raw = raw.get("base_url", "") + if not isinstance(base_url_raw, str): + raise ProviderProfileError(f"{context}.base_url must be a string.") + base_url = base_url_raw.strip().rstrip("/") + if len(base_url) > 2048: + raise ProviderProfileError( + f"{context}.base_url must be at most 2048 characters." + ) + if adapter in _BASE_URL_REQUIRED_ADAPTERS and not base_url: + raise ProviderProfileError( + f"{context}.base_url is required for adapter {adapter}." + ) + if base_url and not base_url.startswith(("http://", "https://")): + raise ProviderProfileError(f"{context}.base_url must use http:// or https://.") + base_url = normalize_provider_base_url(adapter, base_url) + + api_key_raw = raw.get("api_key", "") + if not isinstance(api_key_raw, str): + raise ProviderProfileError(f"{context}.api_key must be a string.") + if len(api_key_raw) > 8192: + raise ProviderProfileError( + f"{context}.api_key must be at most 8192 characters." + ) + + auth_mode = raw.get("auth_mode", "api_key") + if not isinstance(auth_mode, str) or auth_mode not in {"api_key", "oauth"}: + raise ProviderProfileError(f"{context}.auth_mode must be 'api_key' or 'oauth'.") + + models_raw = raw.get("models", []) + if not isinstance(models_raw, list): + raise ProviderProfileError(f"{context}.models must be a list.") + models = tuple( + _parse_model(model, provider_id=provider_id, index=model_index) + for model_index, model in enumerate(models_raw) + ) + model_ids = [model.id for model in models] + if len(model_ids) != len(set(model_ids)): + raise ProviderProfileError( + f"Provider {provider_id!r} contains duplicate model IDs." + ) + + return ProviderProfile( + id=provider_id, + name=_required_string(raw, "name", context=context, max_length=120), + adapter=adapter, + base_url=base_url, + api_key=api_key_raw, + enabled=bool(raw.get("enabled", True)), + models=models, + auth_mode=auth_mode, + ) + + +def _parse_document(raw: Any) -> ProviderProfiles: + if raw is None: + return ProviderProfiles() + if not isinstance(raw, dict): + raise ProviderProfileError("Provider profile document must be an object.") + version = raw.get("version", 1) + if version not in {1, PROVIDER_PROFILES_VERSION}: + raise ProviderProfileError(f"Unsupported provider profile version {version!r}.") + builtins_raw = raw.get("builtins", []) + if not isinstance(builtins_raw, list): + raise ProviderProfileError("builtins must be a list.") + providers_raw = raw.get("providers", []) + if not isinstance(providers_raw, list): + raise ProviderProfileError("providers must be a list.") + providers = tuple( + _parse_profile(profile, index=index, builtin=False) + for index, profile in enumerate(providers_raw) + ) + builtins = tuple( + _parse_profile(profile, index=index, builtin=True) + for index, profile in enumerate(builtins_raw) + ) + provider_ids = [provider.id for provider in providers] + if len(provider_ids) != len(set(provider_ids)): + raise ProviderProfileError("Provider IDs must be unique.") + builtin_ids = [provider.id for provider in builtins] + if len(builtin_ids) != len(set(builtin_ids)): + raise ProviderProfileError("Built-in provider IDs must be unique.") + return ProviderProfiles( + version=PROVIDER_PROFILES_VERSION, + builtins=builtins, + providers=providers, + ) + + +def _profile_to_dict(profile: ProviderProfile) -> dict[str, Any]: + return { + "id": profile.id, + "name": profile.name, + "adapter": profile.adapter, + "base_url": profile.base_url, + "api_key": profile.api_key, + "auth_mode": profile.auth_mode, + "enabled": profile.enabled, + "models": [ + { + "id": model.id, + "name": model.name, + "model_id": model.model_id, + "enabled": model.enabled, + } + for model in profile.models + ], + } + + +def _document_to_dict(document: ProviderProfiles) -> dict[str, Any]: + return { + "version": PROVIDER_PROFILES_VERSION, + "builtins": [_profile_to_dict(profile) for profile in document.builtins], + "providers": [_profile_to_dict(profile) for profile in document.providers], + } + + +def load_provider_profiles() -> ProviderProfiles: + """Load provider profiles, returning an empty document when absent.""" + path = get_provider_profiles_path() + with _STORE_LOCK: + if not path.exists(): + return ProviderProfiles() + try: + with path.open(encoding="utf-8") as handle: + return _parse_document(yaml.safe_load(handle)) + except ProviderProfileError: + raise + except Exception as exc: + raise ProviderProfileError( + f"Could not load provider profiles: {exc}" + ) from exc + + +def save_provider_profiles(document: ProviderProfiles) -> None: + """Atomically persist provider profiles with private permissions.""" + path = get_provider_profiles_path() + payload = _document_to_dict(document) + with _STORE_LOCK: + path.parent.mkdir(parents=True, exist_ok=True) + try: + path.parent.chmod(0o700) + except OSError: + pass + fd, temporary = tempfile.mkstemp( + prefix="providers-", suffix=".yaml.tmp", dir=path.parent + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + yaml.safe_dump( + payload, + handle, + default_flow_style=False, + sort_keys=False, + allow_unicode=True, + ) + handle.flush() + os.fsync(handle.fileno()) + try: + Path(temporary).chmod(0o600) + except OSError: + pass + os.replace(temporary, path) + try: + path.chmod(0o600) + except OSError: + pass + finally: + try: + Path(temporary).unlink(missing_ok=True) + except OSError: + pass + + +def replace_provider_profiles(raw: Any) -> ProviderProfiles: + """Replace supplied profile collections while preserving the other side. + + Existing API keys are retained when an existing profile sends no ``api_key`` + or sends an empty value. ``clear_api_key: true`` explicitly removes it. + """ + if not isinstance(raw, dict): + raise ProviderProfileError("Request body must be an object.") + if "providers" not in raw and "builtins" not in raw: + raise ProviderProfileError("providers or builtins must be supplied.") + + current = load_provider_profiles() + + def merge_collection( + key: str, + supplied: Any, + existing_profiles: tuple[ProviderProfile, ...], + ) -> list[dict[str, Any]]: + if supplied is None: + return [_profile_to_dict(profile) for profile in existing_profiles] + if not isinstance(supplied, list): + raise ProviderProfileError(f"{key} must be a list.") + existing = {profile.id: profile for profile in existing_profiles} + merged: list[dict[str, Any]] = [] + for index, item in enumerate(supplied): + if not isinstance(item, dict): + raise ProviderProfileError(f"{key}[{index}] must be an object.") + candidate = dict(item) + provider_id = candidate.get("id") + prior = existing.get(provider_id) if isinstance(provider_id, str) else None + if candidate.pop("clear_api_key", False): + candidate["api_key"] = "" + elif not candidate.get("api_key") and prior is not None: + candidate["api_key"] = prior.api_key + else: + candidate.setdefault("api_key", "") + merged.append(candidate) + return merged + + document = _parse_document( + { + "version": PROVIDER_PROFILES_VERSION, + "builtins": merge_collection( + "builtins", raw.get("builtins"), current.builtins + ), + "providers": merge_collection( + "providers", raw.get("providers"), current.providers + ), + } + ) + save_provider_profiles(document) + return document + + +def resolve_provider_profile_draft(raw: Any) -> ProviderProfile: + """Validate a WebUI draft and restore its saved API key when omitted. + + Provider discovery and model tests operate on unsaved form values. The + browser never receives an existing secret, so an empty ``api_key`` keeps + the saved value for the matching provider ID. ``clear_api_key`` explicitly + opts out of that fallback. + """ + if not isinstance(raw, dict): + raise ProviderProfileError("provider must be an object.") + + candidate = dict(raw) + provider_id = candidate.get("id") + is_builtin = isinstance(provider_id, str) and provider_id in _RESERVED_PROVIDER_IDS + prior = ( + get_builtin_provider_profile(provider_id) + if is_builtin + else get_provider_profile(provider_id) + if isinstance(provider_id, str) + else None + ) + if candidate.pop("clear_api_key", False): + candidate["api_key"] = "" + elif not candidate.get("api_key") and prior is not None: + candidate["api_key"] = prior.api_key + else: + candidate.setdefault("api_key", "") + candidate.setdefault("models", []) + return _parse_profile(candidate, index=0, builtin=is_builtin) + + +def _api_key_hint(api_key: str) -> str | None: + if not api_key: + return None + if len(api_key) <= 8: + return "configured" + return f"{api_key[:3]}...{api_key[-4:]}" + + +def _provider_profiles_public_payload(document: ProviderProfiles) -> dict[str, Any]: + def profile_payload(profile: ProviderProfile) -> dict[str, Any]: + return { + "id": profile.id, + "name": profile.name, + "adapter": profile.adapter, + "base_url": profile.base_url, + "auth_mode": profile.auth_mode, + "enabled": profile.enabled, + "api_key_configured": bool(profile.api_key), + "api_key_hint": _api_key_hint(profile.api_key), + "models": [ + { + "id": model.id, + "name": model.name, + "model_id": model.model_id, + "enabled": model.enabled, + } + for model in profile.models + ], + } + + return { + "version": document.version, + "builtins": [profile_payload(profile) for profile in document.builtins], + "providers": [profile_payload(profile) for profile in document.providers], + } + + +def provider_profiles_public( + document: ProviderProfiles | None = None, +) -> dict[str, Any]: + """Return a browser-safe representation with all secrets removed.""" + document = document or load_provider_profiles() + public = _provider_profiles_public_payload(document) + public["reserved_provider_ids"] = sorted(_RESERVED_PROVIDER_IDS) + public["revision"] = provider_profiles_public_revision(document) + return public + + +def provider_profiles_public_revision( + document: ProviderProfiles | None = None, +) -> str: + """Return a browser-safe digest that excludes API key material.""" + document = document or load_provider_profiles() + encoded = json.dumps( + _provider_profiles_public_payload(document), + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest()[:16] + + +def provider_profiles_revision( + document: ProviderProfiles | None = None, +) -> str: + """Return a private revision that changes for connection secrets too.""" + document = document or load_provider_profiles() + encoded = json.dumps( + _document_to_dict(document), sort_keys=True, separators=(",", ":") + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest()[:16] + + +def get_provider_profile(provider_id: str) -> ProviderProfile | None: + for profile in load_provider_profiles().providers: + if profile.id == provider_id: + return profile + return None + + +def get_builtin_provider_profile(provider_id: str) -> ProviderProfile | None: + for profile in load_provider_profiles().builtins: + if profile.id == provider_id: + return profile + return None + + +def get_provider_profile_revision(provider_id: str) -> str | None: + profile = get_builtin_provider_profile(provider_id) or get_provider_profile( + provider_id + ) + if profile is None: + return None + encoded = json.dumps( + _profile_to_dict(profile), sort_keys=True, separators=(",", ":") + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest()[:16] + + +def resolve_provider_model( + provider_id: str, model_name: str +) -> tuple[ProviderProfile, ProviderModel] | None: + """Resolve a configured ``(provider_id, model alias)`` pair.""" + profile = get_provider_profile(provider_id) + if profile is None: + return None + if not profile.enabled: + raise ProviderProfileError(f"Provider {provider_id!r} is disabled.") + for model in profile.models: + if model.id == model_name: + if not model.enabled: + raise ProviderProfileError( + f"Model {model_name!r} in provider {provider_id!r} is disabled." + ) + return profile, model + raise ProviderProfileError( + f"Model {model_name!r} is not configured for provider {provider_id!r}." + ) + + +def list_configured_model_entries() -> list[tuple[str, str, str]]: + """Return enabled dynamic models as ``(alias, upstream id, provider id)``.""" + entries: list[tuple[str, str, str]] = [] + for profile in load_provider_profiles().providers: + if not profile.enabled: + continue + for model in profile.models: + if model.enabled: + entries.append((model.id, model.model_id, profile.id)) + return entries + + +def list_configured_builtin_model_entries() -> list[tuple[str, str, str]]: + """Return enabled managed built-in models.""" + entries: list[tuple[str, str, str]] = [] + for profile in load_provider_profiles().builtins: + if not profile.enabled: + continue + for model in profile.models: + if model.enabled: + entries.append((model.id, model.model_id, profile.id)) + return entries diff --git a/EvoScientist/config/settings.py b/EvoScientist/config/settings.py index 0793a9c..5e8d915 100644 --- a/EvoScientist/config/settings.py +++ b/EvoScientist/config/settings.py @@ -23,6 +23,19 @@ from dotenv import find_dotenv, load_dotenv # (stream/display.py, channels/consumer.py) — keep aligned with the agent's # `interrupt_on` set in EvoScientist.py. HITL_SHELL_TOOLS = ("execute", "run_in_background") +_CONFIG_APPLIED_ENV_VALUES: dict[str, str] = {} + + +def _apply_config_env(name: str, value: str) -> None: + if value and not os.environ.get(name): + os.environ[name] = value + _CONFIG_APPLIED_ENV_VALUES[name] = value + + +def is_config_applied_env(name: str) -> bool: + """Return whether the current env value was injected from config.yaml.""" + applied = _CONFIG_APPLIED_ENV_VALUES.get(name) + return applied is not None and os.environ.get(name) == applied class MemoryObservationTarget(StrEnum): @@ -119,6 +132,11 @@ def get_config_path() -> Path: return get_config_dir() / "config.yaml" +def get_default_workspace_dir() -> Path: + """Return the stable workspace used when no directory is configured.""" + return Path.home() / ".evoscientist" / "workspace" + + # ============================================================================= # Configuration dataclass # ============================================================================= @@ -139,7 +157,8 @@ class EvoScientistConfig: auxiliary_provider: Provider for auxiliary_model (empty = use main provider). auxiliary_model: Model for memory workers + tool selector + scheduler (empty = use main model). default_mode: Default workspace mode ('daemon' or 'run'). - default_workdir: Default workspace directory (empty = use current working directory). + default_workdir: Default workspace directory (empty = use + ~/.evoscientist/workspace). show_thinking: Whether to show thinking panels in CLI. """ @@ -171,6 +190,10 @@ class EvoScientistConfig: # LLM Settings provider: str = "anthropic" model: str = "claude-sonnet-4-6" + # Legacy built-in model allowlist. New WebUI saves store built-in and + # custom model catalogs in providers.yaml; this remains as a read-only + # compatibility fallback until the first Registry save. + model_catalog: list[dict[str, Any]] | None = None model_fallbacks: str = "" # "model:provider,model:provider" fallback chain # Optional auxiliary model for background/helper LLM calls (memory workers + # tool selector). Empty = fall back to the main model/provider. @@ -398,6 +421,17 @@ class EvoScientistConfig: # blocklist (sudo/chmod/dd/...) still applies. Implies auto_approve. dangerous_mode: bool = False + # Conversation workspace isolation. New WebUI conversations use a registry- + # validated scope by default. ``optional`` preserves the legacy execution + # path only when a trusted caller cannot provide a scope. + workspace_isolation: str = "optional" # legacy | optional | required + scope_registry_topology: str = "single-host" + strict_executor: str = "oci" # oci + strict_executor_image: str = "" # immutable OCI digest required in required mode + strict_code_interpreter: str = "disabled" # disabled | scoped + draft_workspace_ttl_hours: int = 24 + workspace_trash_retention_days: int = 7 + # Agent features enable_ask_user: bool = True # Enable ask_user tool for agent-initiated questions @@ -458,6 +492,47 @@ class EvoScientistConfig: if self.dangerous_mode: self.auto_approve = True + if self.workspace_isolation not in {"legacy", "optional", "required"}: + raise ValueError( + "workspace_isolation must be one of legacy, optional, required" + ) + if self.scope_registry_topology != "single-host": + raise ValueError("v1 workspace isolation only supports single-host topology") + if self.strict_executor != "oci": + raise ValueError("strict_executor must be oci") + if self.strict_code_interpreter not in {"disabled", "scoped"}: + raise ValueError("strict_code_interpreter must be disabled or scoped") + if ( + not isinstance(self.draft_workspace_ttl_hours, int) + or isinstance(self.draft_workspace_ttl_hours, bool) + or self.draft_workspace_ttl_hours <= 0 + ): + raise ValueError("draft_workspace_ttl_hours must be a positive integer") + if ( + not isinstance(self.workspace_trash_retention_days, int) + or isinstance(self.workspace_trash_retention_days, bool) + or self.workspace_trash_retention_days <= 0 + ): + raise ValueError("workspace_trash_retention_days must be a positive integer") + if self.workspace_isolation == "required" and self.dangerous_mode: + raise ValueError( + "dangerous_mode is incompatible with required workspace isolation" + ) + if ( + self.workspace_isolation == "required" + and "@sha256:" not in self.strict_executor_image + ): + raise ValueError( + "required workspace isolation needs strict_executor_image pinned by digest" + ) + if ( + self.workspace_isolation == "required" + and self.strict_code_interpreter != "disabled" + ): + raise ValueError( + "strict_code_interpreter=scoped is not available until every PTC tool is scope-verified" + ) + _normalize_str_enum_fields(self) synthesis_time = _normalize_hhmm(self.memory_skill_synthesis_time) @@ -762,6 +837,13 @@ _ENV_MAPPINGS = { "EVOSCIENTIST_OPENROUTER_ANTHROPIC_PROMPT_CACHE" ), "dangerous_mode": "EVOSCIENTIST_DANGEROUS_MODE", + "workspace_isolation": "EVOSCIENTIST_WORKSPACE_ISOLATION", + "scope_registry_topology": "EVOSCIENTIST_SCOPE_REGISTRY_TOPOLOGY", + "strict_executor": "EVOSCIENTIST_STRICT_EXECUTOR", + "strict_executor_image": "EVOSCIENTIST_STRICT_EXECUTOR_IMAGE", + "strict_code_interpreter": "EVOSCIENTIST_STRICT_CODE_INTERPRETER", + "draft_workspace_ttl_hours": "EVOSCIENTIST_DRAFT_WORKSPACE_TTL_HOURS", + "workspace_trash_retention_days": "EVOSCIENTIST_WORKSPACE_TRASH_RETENTION_DAYS", "channel_debug_tracing": "EVOSCIENTIST_CHANNEL_DEBUG_TRACING", "ccproxy_port": "EVOSCIENTIST_CCPROXY_PORT", "use_responses_api": "EVOSCIENTIST_USE_RESPONSES_API", @@ -844,50 +926,26 @@ def apply_config_to_env(config: EvoScientistConfig) -> None: Args: config: Configuration to apply. """ - if config.anthropic_api_key and not os.environ.get("ANTHROPIC_API_KEY"): - os.environ["ANTHROPIC_API_KEY"] = config.anthropic_api_key - if config.anthropic_base_url and not os.environ.get("ANTHROPIC_BASE_URL"): - os.environ["ANTHROPIC_BASE_URL"] = config.anthropic_base_url - if config.openai_api_key and not os.environ.get("OPENAI_API_KEY"): - os.environ["OPENAI_API_KEY"] = config.openai_api_key - if config.nvidia_api_key and not os.environ.get("NVIDIA_API_KEY"): - os.environ["NVIDIA_API_KEY"] = config.nvidia_api_key - if config.google_api_key and not os.environ.get("GOOGLE_API_KEY"): - os.environ["GOOGLE_API_KEY"] = config.google_api_key - if config.minimax_api_key and not os.environ.get("MINIMAX_API_KEY"): - os.environ["MINIMAX_API_KEY"] = config.minimax_api_key - if config.minimax_base_url and not os.environ.get("MINIMAX_BASE_URL"): - os.environ["MINIMAX_BASE_URL"] = config.minimax_base_url - if config.siliconflow_api_key and not os.environ.get("SILICONFLOW_API_KEY"): - os.environ["SILICONFLOW_API_KEY"] = config.siliconflow_api_key - if config.openrouter_api_key and not os.environ.get("OPENROUTER_API_KEY"): - os.environ["OPENROUTER_API_KEY"] = config.openrouter_api_key - if config.deepseek_api_key and not os.environ.get("DEEPSEEK_API_KEY"): - os.environ["DEEPSEEK_API_KEY"] = config.deepseek_api_key - if config.zhipu_api_key and not os.environ.get("ZHIPU_API_KEY"): - os.environ["ZHIPU_API_KEY"] = config.zhipu_api_key - if config.volcengine_api_key and not os.environ.get("VOLCENGINE_API_KEY"): - os.environ["VOLCENGINE_API_KEY"] = config.volcengine_api_key - if config.dashscope_api_key and not os.environ.get("DASHSCOPE_API_KEY"): - os.environ["DASHSCOPE_API_KEY"] = config.dashscope_api_key - if config.moonshot_api_key and not os.environ.get("MOONSHOT_API_KEY"): - os.environ["MOONSHOT_API_KEY"] = config.moonshot_api_key - if config.kimi_api_key and not os.environ.get("KIMI_API_KEY"): - os.environ["KIMI_API_KEY"] = config.kimi_api_key - if config.custom_openai_api_key and not os.environ.get("CUSTOM_OPENAI_API_KEY"): - os.environ["CUSTOM_OPENAI_API_KEY"] = config.custom_openai_api_key - if config.custom_openai_base_url and not os.environ.get("CUSTOM_OPENAI_BASE_URL"): - os.environ["CUSTOM_OPENAI_BASE_URL"] = config.custom_openai_base_url - if config.custom_anthropic_api_key and not os.environ.get( - "CUSTOM_ANTHROPIC_API_KEY" - ): - os.environ["CUSTOM_ANTHROPIC_API_KEY"] = config.custom_anthropic_api_key - if config.custom_anthropic_base_url and not os.environ.get( - "CUSTOM_ANTHROPIC_BASE_URL" - ): - os.environ["CUSTOM_ANTHROPIC_BASE_URL"] = config.custom_anthropic_base_url - if config.ollama_base_url and not os.environ.get("OLLAMA_BASE_URL"): - os.environ["OLLAMA_BASE_URL"] = config.ollama_base_url + _apply_config_env("ANTHROPIC_API_KEY", config.anthropic_api_key) + _apply_config_env("ANTHROPIC_BASE_URL", config.anthropic_base_url) + _apply_config_env("OPENAI_API_KEY", config.openai_api_key) + _apply_config_env("NVIDIA_API_KEY", config.nvidia_api_key) + _apply_config_env("GOOGLE_API_KEY", config.google_api_key) + _apply_config_env("MINIMAX_API_KEY", config.minimax_api_key) + _apply_config_env("MINIMAX_BASE_URL", config.minimax_base_url) + _apply_config_env("SILICONFLOW_API_KEY", config.siliconflow_api_key) + _apply_config_env("OPENROUTER_API_KEY", config.openrouter_api_key) + _apply_config_env("DEEPSEEK_API_KEY", config.deepseek_api_key) + _apply_config_env("ZHIPU_API_KEY", config.zhipu_api_key) + _apply_config_env("VOLCENGINE_API_KEY", config.volcengine_api_key) + _apply_config_env("DASHSCOPE_API_KEY", config.dashscope_api_key) + _apply_config_env("MOONSHOT_API_KEY", config.moonshot_api_key) + _apply_config_env("KIMI_API_KEY", config.kimi_api_key) + _apply_config_env("CUSTOM_OPENAI_API_KEY", config.custom_openai_api_key) + _apply_config_env("CUSTOM_OPENAI_BASE_URL", config.custom_openai_base_url) + _apply_config_env("CUSTOM_ANTHROPIC_API_KEY", config.custom_anthropic_api_key) + _apply_config_env("CUSTOM_ANTHROPIC_BASE_URL", config.custom_anthropic_base_url) + _apply_config_env("OLLAMA_BASE_URL", config.ollama_base_url) if config.tavily_api_key and not os.environ.get("TAVILY_API_KEY"): os.environ["TAVILY_API_KEY"] = config.tavily_api_key if config.reasoning_effort and not os.environ.get("EVOSCIENTIST_REASONING_EFFORT"): diff --git a/EvoScientist/cron/schedule.py b/EvoScientist/cron/schedule.py index 8de7497..2e8a4f7 100644 --- a/EvoScientist/cron/schedule.py +++ b/EvoScientist/cron/schedule.py @@ -12,7 +12,7 @@ multiple clients at one hand-started server they will share the same cron store. from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from langgraph_sdk.schema import Cron, Run @@ -47,22 +47,84 @@ def is_available() -> bool: return bool(is_langgraph_dev_running(base_url=_scheduler_url())) +def _scope_payload( + scope: Any | None, *, owner_id: str | None = None +) -> tuple[dict[str, Any], dict[str, Any]]: + if scope is None: + return {}, {} + metadata = { + "workspace_scope_id": scope.scope_id, + "workspace_scope_owner_id": owner_id or scope.owner_id, + "workspace_deployment_id": scope.deployment_id, + } + return metadata, { + "configurable": { + **metadata, + "workspace_scope_revision": scope.revision, + } + } + + def create_schedule( - *, name: str, schedule: str, prompt: str, timezone: str | None = None + *, + name: str, + schedule: str, + prompt: str, + timezone: str | None = None, + scope: Any | None = None, ) -> Cron: """Create a recurring scheduled task on the scheduler graph.""" # Crons are stored in the langgraph-dev process's .langgraph_api store, not # tagged by workspace. Isolation is process-level (see module docstring). - return _client().crons.create( - assistant_id=SCHEDULER_GRAPH_ID, - schedule=schedule, - input=messages_input(prompt), - metadata={"run_kind": SCHEDULED_RUN_KIND, "name": name, "prompt": prompt}, - timezone=timezone or _default_timezone(), + owner = None + if scope is not None: + from ..scope_registry import get_scope_registry + + owner = get_scope_registry().register_owner( + scope.deployment_id, + scope.scope_id, + owner_type="schedule", + parent_owner_id=scope.owner_id, + state="reserved", + ) + scope_metadata, config = _scope_payload( + scope, owner_id=owner.owner_id if owner else None ) + try: + cron = _client().crons.create( + assistant_id=SCHEDULER_GRAPH_ID, + schedule=schedule, + input=messages_input(prompt), + metadata={ + "run_kind": SCHEDULED_RUN_KIND, + "name": name, + "prompt": prompt, + **scope_metadata, + }, + **({"config": config} if config else {}), + timezone=timezone or _default_timezone(), + ) + except Exception: + if owner is not None: + get_scope_registry().bind_owner( + scope.deployment_id, + scope.scope_id, + owner.owner_id, + f"failed:{owner.owner_id}", + state="terminal", + ) + raise + if owner is not None: + get_scope_registry().bind_owner( + scope.deployment_id, + scope.scope_id, + owner.owner_id, + str(cron["cron_id"]), + ) + return cron -def list_schedules() -> list[Cron]: +def list_schedules(scope: Any | None = None) -> list[Cron]: """Return only EvoScientist scheduled tasks. Filtered server-side by ``run_kind`` metadata (the cron backend matches by @@ -71,10 +133,17 @@ def list_schedules() -> list[Cron]: rather than ``assistant_id`` because the stored ``assistant_id`` is a resolved UUID, not the ``scheduler`` graph name we create with. """ - return _client().crons.search( + rows = _client().crons.search( metadata={"run_kind": SCHEDULED_RUN_KIND}, limit=1000, ) + if scope is None: + return rows + return [ + row + for row in rows + if (row.get("metadata") or {}).get("workspace_scope_id") == scope.scope_id + ] def delete_schedule(cron_id: str) -> None: @@ -87,13 +156,28 @@ def set_enabled(cron_id: str, enabled: bool) -> Cron: return _client().crons.update(cron_id, enabled=enabled) -def run_now(prompt: str) -> Run: +def run_now(prompt: str, scope: Any | None = None) -> Run: """Fire a one-off scheduler run immediately (for ``/schedule run``). Output goes wherever the task's prompt specifies; there is no push notification. """ client = _client() thread = client.threads.create(graph_id=SCHEDULER_GRAPH_ID) + owner = None + if scope is not None: + from ..scope_registry import get_scope_registry + + owner = get_scope_registry().register_owner( + scope.deployment_id, + scope.scope_id, + owner_type="scheduler_run", + resource_id=str(thread["thread_id"]), + parent_owner_id=scope.owner_id, + state="active", + ) + scope_metadata, config = _scope_payload( + scope, owner_id=owner.owner_id if owner else None + ) return client.runs.create( thread_id=str(thread["thread_id"]), assistant_id=SCHEDULER_GRAPH_ID, @@ -102,5 +186,7 @@ def run_now(prompt: str) -> Run: "run_kind": SCHEDULED_RUN_KIND, "name": "manual-run", "prompt": prompt, + **scope_metadata, }, + **({"config": config} if config else {}), ) diff --git a/EvoScientist/deploy/server.py b/EvoScientist/deploy/server.py index 2af9a0b..cf1b78b 100644 --- a/EvoScientist/deploy/server.py +++ b/EvoScientist/deploy/server.py @@ -5,8 +5,8 @@ for consumption by external LangChain-compatible UIs (deep-agents-ui, agent-chat-ui, LangSmith Studio) and SDK clients. Differs from ``EvoSci`` / ``EvoSci serve``: no in-process CLI agent, -no session DB, no channel runtime, no TUI. The terminal only shows -startup progress, the Ready banner, and then blocks until Ctrl+C. +no session DB, no channel runtime, no TUI. The terminal shows startup +progress, the Ready banner, and the live Gateway log until Ctrl+C. Mode dispatch happens via the ``EVOSCIENTIST_DEPLOY_MODE`` env var injected by ``start_langgraph_dev``: ``full`` for the deploy subprocess @@ -20,11 +20,13 @@ loads or skips MCP based on the value. from __future__ import annotations import atexit +import codecs import os import signal +import sys import threading from pathlib import Path -from typing import Any +from typing import Any, TextIO import typer # type: ignore[import-untyped] from rich.panel import Panel @@ -32,6 +34,65 @@ from rich.text import Text from ..cli._app import app from ..stream.console import console +from ..usage import prepare_usage_environment + + +def _follow_gateway_log( + log_path: Path, + start_offset: int, + stop_event: threading.Event, + *, + output: TextIO | None = None, + poll_interval: float = 0.1, +) -> None: + """Mirror appended Gateway log bytes to a text stream until stopped.""" + sink = output if output is not None else sys.stdout + decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") + offset = max(0, start_offset) + + while True: + try: + with log_path.open("rb") as log_file: + log_file.seek(0, os.SEEK_END) + if log_file.tell() < offset: + # Be defensive if another process replaced/truncated the log. + offset = 0 + decoder.reset() + log_file.seek(offset) + chunk = log_file.read() + except OSError: + chunk = b"" + + if chunk: + offset += len(chunk) + text = decoder.decode(chunk) + if text: + sink.write(text) + sink.flush() + + if stop_event.is_set(): + break + stop_event.wait(poll_interval) + + remainder = decoder.decode(b"", final=True) + if remainder: + sink.write(remainder) + sink.flush() + + +def _start_gateway_log_follower( + log_path: Path, + start_offset: int, +) -> tuple[threading.Event, threading.Thread]: + stop_event = threading.Event() + thread = threading.Thread( + target=_follow_gateway_log, + args=(log_path, start_offset, stop_event), + name="evoscientist-gateway-log", + daemon=True, + ) + thread.start() + return stop_event, thread @app.command() @@ -39,7 +100,10 @@ def deploy( workdir: str | None = typer.Option( None, "--workdir", - help="Workspace directory (default: config.default_workdir or cwd)", + help=( + "Workspace directory (default: config.default_workdir or " + "~/.evoscientist/workspace)" + ), ), port: int | None = typer.Option( None, @@ -64,11 +128,16 @@ def deploy( Connect any LangChain-compatible UI or SDK client to the printed endpoint. Press Ctrl+C to stop. """ - from ..config import apply_config_to_env, get_effective_config + from ..config import ( + apply_config_to_env, + get_default_workspace_dir, + get_effective_config, + ) from ..langgraph_dev.manager import ( _DEFAULT_PORT, RUNTIME, _is_port_occupied, + current_log_start_offset, is_langgraph_dev_running, read_tunnel_url, start_langgraph_dev, @@ -88,17 +157,21 @@ def deploy( _configure_logging() apply_config_to_env(config) - # 2. Resolve workspace (CLI > config.default_workdir > cwd) + # 2. Resolve workspace (CLI > config.default_workdir > stable app default) if workdir: ws = os.path.abspath(os.path.expanduser(workdir)) elif config.default_workdir: ws = os.path.abspath(os.path.expanduser(config.default_workdir)) else: - ws = os.getcwd() + ws = str(get_default_workspace_dir()) # Subprocess inherits this path via EVOSCIENTIST_WORKSPACE_DIR (set inside # start_langgraph_dev). Ensure the dir exists; do NOT mutate the parent # process's paths module state — the deploy parent has no in-process agent. os.makedirs(ws, exist_ok=True) + if getattr(config, "workspace_isolation", "optional") != "legacy": + from ..scope_registry import get_scope_service_token + + os.environ["EVOSCIENTIST_BACKEND_SERVICE_TOKEN"] = get_scope_service_token(ws) # 3. Resolve port (explicit None check — don't treat --port 0 as "unset"), # then validate range so misconfigurations fail fast with a clear message @@ -138,6 +211,18 @@ def deploy( ) raise typer.Exit(1) + # Standalone WebUI development runs the backend and Next.js in separate + # terminals. Prepare the same durable identity, sink token, and spool used + # by the integrated launcher so the independently started WebUI can accept + # usage events through the shared data directory. + webui_port = int(getattr(config, "webui_port", 4716)) + try: + usage_env = prepare_usage_environment(ws, webui_port=webui_port) + except (OSError, ValueError) as exc: + console.print(f"[yellow]Token statistics disabled:[/yellow] {exc}") + else: + os.environ.update(usage_env) + # 5. Startup banner _auth_label = _describe_auth(config) console.print( @@ -249,6 +334,15 @@ def deploy( ) ) + # Replay this session's startup output, then mirror new Gateway log lines + # while deploy owns the foreground terminal. The child continues writing + # directly to the file, so terminal backpressure cannot block the server. + console.print("[dim]Gateway logs (also saved to the path above):[/dim]") + log_stop_event, log_thread = _start_gateway_log_follower( + RUNTIME.log_file, + current_log_start_offset(), + ) + # 10. Block on signal — mirror serve's dual-gate (threading.Event + # explicit SIGINT/SIGTERM handlers) so SIGTERM (no default raise) also # triggers clean shutdown. @@ -268,6 +362,8 @@ def deploy( except KeyboardInterrupt: shutdown_event.set() finally: + log_stop_event.set() + log_thread.join(timeout=2.0) signal.signal(signal.SIGINT, _orig_sigint) signal.signal(signal.SIGTERM, _orig_sigterm) # stop_langgraph_dev + stop_ccproxy run via atexit during interpreter diff --git a/EvoScientist/deploy/webui.py b/EvoScientist/deploy/webui.py index 12d750b..f6d1786 100644 --- a/EvoScientist/deploy/webui.py +++ b/EvoScientist/deploy/webui.py @@ -57,12 +57,13 @@ def run_webui(config: Any, workspace_dir: str | None = None) -> None: config: Effective ``EvoScientistConfig`` (already env-applied upstream, but re-applied here so this is safe to call standalone). workspace_dir: Resolved workspace path; falls back to - ``config.default_workdir`` then cwd. + ``config.default_workdir`` then ``~/.evoscientist/workspace``. Blocks until Ctrl+C / SIGTERM, or until the front-end process exits, then tears down both subprocesses. Never returns a value. """ - from ..config import apply_config_to_env + from ..config import apply_config_to_env, get_default_workspace_dir + from ..config.provider_admin import get_provider_admin_token from ..langgraph_dev.manager import ( _DEFAULT_PORT, RUNTIME, @@ -75,7 +76,7 @@ def run_webui(config: Any, workspace_dir: str | None = None) -> None: apply_config_to_env(config) - # 1. Resolve workspace (CLI-resolved value > config.default_workdir > cwd), + # 1. Resolve workspace (CLI > config.default_workdir > stable app default), # mirroring `EvoSci deploy`. The langgraph dev subprocess inherits this via # EVOSCIENTIST_WORKSPACE_DIR (set inside start_langgraph_dev). if workspace_dir: @@ -83,8 +84,14 @@ def run_webui(config: Any, workspace_dir: str | None = None) -> None: elif getattr(config, "default_workdir", ""): ws = os.path.abspath(os.path.expanduser(config.default_workdir)) else: - ws = os.getcwd() + ws = str(get_default_workspace_dir()) os.makedirs(ws, exist_ok=True) + scope_service_token = "" + if getattr(config, "workspace_isolation", "optional") != "legacy": + from ..scope_registry import get_scope_service_token + + scope_service_token = get_scope_service_token(ws) + os.environ["EVOSCIENTIST_BACKEND_SERVICE_TOKEN"] = scope_service_token # 2. Resolve ports: backend = langgraph dev (browser connects here), # webui_port = the local Next.js server the browser actually opens. @@ -130,6 +137,28 @@ def run_webui(config: Any, workspace_dir: str | None = None) -> None: ) raise typer.Exit(1) + # Token tracking is enabled only for this integrated topology. Generate the + # stable deployment/workspace identities and the dedicated transport token + # before starting either child so both processes share the exact values. + from ..usage import prepare_usage_environment + + try: + usage_env = prepare_usage_environment(ws, webui_port=webui_port) + except (OSError, ValueError) as exc: + console.print(f"[yellow]Token statistics disabled:[/yellow] {exc}") + usage_env = {} + os.environ.update(usage_env) + + # The provider editor uses a server-to-server management token. Resolve it + # through the shared user-level token file so a separately started WebUI, + # a fresh backend, and a reused backend all authenticate with the same + # value. The browser never receives it; the Next.js proxy adds it. + provider_admin_token = get_provider_admin_token() + if not provider_admin_token: + console.print("[red]Could not create the provider management token.[/red]") + raise typer.Exit(1) + os.environ["EVOSCIENTIST_PROVIDER_ADMIN_TOKEN"] = provider_admin_token + # 4. Backend (langgraph dev): reuse an EvoSci server already on the port, # else start a fresh deploy-mode one (full MCP + async). Refuse a foreign # occupant — that's a configuration error, not something to silently share. @@ -205,7 +234,12 @@ def run_webui(config: Any, workspace_dir: str | None = None) -> None: # are scrubbed — the browser UI never needs LLM provider API keys. webui_env = _scrubbed_env( { + **usage_env, "EVOSCIENTIST_LANGGRAPH_DEV_PORT": str(backend_port), + "EVOSCIENTIST_BACKEND_URL": f"http://127.0.0.1:{backend_port}", + "EVOSCIENTIST_PROVIDER_ADMIN_TOKEN": provider_admin_token, + "EVOSCIENTIST_BACKEND_SERVICE_TOKEN": scope_service_token, + "EVOSCIENTIST_WORKSPACE_DIR": ws, "PORT": str(webui_port), } ) diff --git a/EvoScientist/langgraph_dev/http.py b/EvoScientist/langgraph_dev/http.py index 619c2ec..2662425 100644 --- a/EvoScientist/langgraph_dev/http.py +++ b/EvoScientist/langgraph_dev/http.py @@ -23,10 +23,13 @@ memory. from __future__ import annotations import asyncio +import hashlib import json import logging import os +import secrets import sqlite3 +import threading from pathlib import Path from typing import Any @@ -39,8 +42,40 @@ from starlette.requests import Request from starlette.responses import JSONResponse from starlette.routing import Route -from EvoScientist.config import get_effective_config -from EvoScientist.llm.models import list_model_picker_entries +from EvoScientist.config import ( + get_effective_config, + is_config_applied_env, + load_config, + save_config, +) +from EvoScientist.config.provider_admin import get_provider_admin_token +from EvoScientist.config.provider_profiles import ( + ProviderModel, + ProviderProfileError, + list_configured_builtin_model_entries, + list_configured_model_entries, + load_provider_profiles, + provider_profiles_public, + provider_profiles_public_revision, + provider_profiles_revision, + replace_provider_profiles, + resolve_provider_profile_draft, +) +from EvoScientist.llm.models import ( + list_builtin_model_catalog_entries, + list_model_picker_entries, + list_models_by_provider, + normalize_builtin_model_catalog, +) +from EvoScientist.llm.provider_operations import ( + ProviderOperationError, + build_builtin_provider_profile, + builtin_provider_adapter, + builtin_provider_ids, + builtin_provider_legacy_api_key, + discover_provider_models, + test_provider_model, +) from EvoScientist.sessions import ( MAIN_THREAD_FILTER_PARAMS, MAIN_THREAD_FILTER_SQL, @@ -50,25 +85,488 @@ from EvoScientist.sessions import ( ) _logger = logging.getLogger(__name__) +_config_file_lock = threading.RLock() + +_LLM_CONFIG_VALUE_FIELDS = ( + "provider", + "model", + "model_fallbacks", + "auxiliary_provider", + "auxiliary_model", + "anthropic_base_url", + "anthropic_auth_mode", + "openai_auth_mode", + "minimax_base_url", + "custom_openai_base_url", + "custom_anthropic_base_url", + "ollama_base_url", +) +_LLM_CONFIG_SECRET_FIELDS = ( + "anthropic_api_key", + "openai_api_key", + "nvidia_api_key", + "google_api_key", + "minimax_api_key", + "siliconflow_api_key", + "openrouter_api_key", + "deepseek_api_key", + "zhipu_api_key", + "volcengine_api_key", + "dashscope_api_key", + "moonshot_api_key", + "kimi_api_key", + "custom_openai_api_key", + "custom_anthropic_api_key", +) +_LLM_CONFIG_ENV_VARS = { + "anthropic_api_key": "ANTHROPIC_API_KEY", + "anthropic_base_url": "ANTHROPIC_BASE_URL", + "anthropic_auth_mode": "EVOSCIENTIST_ANTHROPIC_AUTH_MODE", + "openai_api_key": "OPENAI_API_KEY", + "openai_auth_mode": "EVOSCIENTIST_OPENAI_AUTH_MODE", + "nvidia_api_key": "NVIDIA_API_KEY", + "google_api_key": "GOOGLE_API_KEY", + "minimax_api_key": "MINIMAX_API_KEY", + "minimax_base_url": "MINIMAX_BASE_URL", + "siliconflow_api_key": "SILICONFLOW_API_KEY", + "openrouter_api_key": "OPENROUTER_API_KEY", + "deepseek_api_key": "DEEPSEEK_API_KEY", + "zhipu_api_key": "ZHIPU_API_KEY", + "volcengine_api_key": "VOLCENGINE_API_KEY", + "dashscope_api_key": "DASHSCOPE_API_KEY", + "moonshot_api_key": "MOONSHOT_API_KEY", + "kimi_api_key": "KIMI_API_KEY", + "custom_openai_api_key": "CUSTOM_OPENAI_API_KEY", + "custom_openai_base_url": "CUSTOM_OPENAI_BASE_URL", + "custom_anthropic_api_key": "CUSTOM_ANTHROPIC_API_KEY", + "custom_anthropic_base_url": "CUSTOM_ANTHROPIC_BASE_URL", + "ollama_base_url": "OLLAMA_BASE_URL", + "model_fallbacks": "EVOSCIENTIST_MODEL_FALLBACKS", + "auxiliary_provider": "EVOSCIENTIST_AUXILIARY_PROVIDER", + "auxiliary_model": "EVOSCIENTIST_AUXILIARY_MODEL", +} +_LLM_CONFIG_BASE_URL_FIELDS = { + "anthropic_base_url", + "minimax_base_url", + "custom_openai_base_url", + "custom_anthropic_base_url", + "ollama_base_url", +} +_LLM_CONFIG_AUTH_MODE_FIELDS = {"anthropic_auth_mode", "openai_auth_mode"} +_LLM_CONFIG_IMMEDIATE_FIELDS = { + "provider", + "model", + "model_catalog", + "builtin_providers", +} + + +class _ConfigRevisionConflict(ValueError): + """Raised when config.yaml changed after the WebUI loaded it.""" + + +def _llm_config_revision(config: Any) -> str: + values = { + field: getattr(config, field) + for field in (*_LLM_CONFIG_VALUE_FIELDS, *_LLM_CONFIG_SECRET_FIELDS) + } + values["model_catalog"] = getattr(config, "model_catalog", None) + values["provider_registry_revision"] = provider_profiles_revision() + encoded = json.dumps(values, sort_keys=True, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest()[:16] + + +def _model_catalog_profile_models( + config: Any, provider_id: str +) -> tuple[ProviderModel, ...]: + try: + catalog = normalize_builtin_model_catalog(config.model_catalog) or [] + except ValueError: + return () + return tuple( + ProviderModel( + id=entry["id"], + name=entry["name"], + model_id=entry["model_id"], + enabled=entry["enabled"], + ) + for entry in catalog + if entry["provider"] == provider_id + ) + + +def _builtin_profiles_public(config: Any, document: Any) -> list[dict[str, Any]]: + managed_by_id = {profile.id: profile for profile in document.builtins} + result: list[dict[str, Any]] = [] + for provider_id in builtin_provider_ids(): + managed = managed_by_id.get(provider_id) + effective = build_builtin_provider_profile( + config, + provider_id, + saved_profile=managed, + load_saved=False, + ) + models = ( + managed.models + if managed is not None + else _model_catalog_profile_models(config, provider_id) + ) + result.append( + { + "id": provider_id, + "name": managed.name if managed is not None else provider_id, + "adapter": builtin_provider_adapter(provider_id), + "base_url": effective.base_url, + "auth_mode": effective.auth_mode, + "enabled": managed.enabled if managed is not None else True, + "managed": managed is not None, + "api_key_configured": bool(effective.api_key), + "api_key_hint": ( + f"...{effective.api_key[-4:]}" if effective.api_key else None + ), + "models": [ + { + "id": model.id, + "name": model.name, + "model_id": model.model_id, + "enabled": model.enabled, + } + for model in models + ], + } + ) + return result + + +def _llm_config_public(config: Any) -> dict[str, Any]: + values = {field: getattr(config, field) for field in _LLM_CONFIG_VALUE_FIELDS} + secret_status = {} + for field in _LLM_CONFIG_SECRET_FIELDS: + value = getattr(config, field) + secret_status[field] = { + "configured": bool(value), + "hint": f"...{value[-4:]}" if value else None, + } + env_overrides = { + field: env_name + for field, env_name in _LLM_CONFIG_ENV_VARS.items() + if os.environ.get(env_name) and not is_config_applied_env(env_name) + } + catalog_error = None + provider_document = load_provider_profiles() + managed_builtins = provider_document.builtins + if managed_builtins: + model_catalog = [ + { + "provider": profile.id, + "id": model.id, + "name": model.name, + "model_id": model.model_id, + "enabled": model.enabled, + } + for profile in managed_builtins + for model in profile.models + ] + else: + try: + model_catalog = normalize_builtin_model_catalog(config.model_catalog) + except ValueError as exc: + model_catalog = [] + catalog_error = str(exc) + return { + "revision": _llm_config_revision(config), + "values": values, + "secrets": secret_status, + "env_overrides": env_overrides, + "model_catalog": model_catalog, + "model_catalog_error": catalog_error, + "builtin_providers": _builtin_profiles_public(config, provider_document), + "builtin_model_candidates": [ + { + "id": name, + "name": name, + "model_id": model_id, + "provider": provider, + "enabled": True, + } + for name, model_id, provider in list_models_by_provider() + ], + } + + +def _read_llm_config() -> dict[str, Any]: + with _config_file_lock: + return _llm_config_public(load_config()) + + +def _validated_llm_config_value(field: str, value: Any) -> str: + if not isinstance(value, str): + raise ValueError(f"{field} must be a string.") + value = value.strip() + max_length = 4096 if field == "model_fallbacks" else 2048 + if field in {"provider", "auxiliary_provider"}: + max_length = 64 + elif field in {"model", "auxiliary_model"}: + max_length = 300 + elif field in _LLM_CONFIG_AUTH_MODE_FIELDS: + max_length = 16 + if len(value) > max_length: + raise ValueError(f"{field} must be at most {max_length} characters.") + if field in _LLM_CONFIG_AUTH_MODE_FIELDS and value not in {"api_key", "oauth"}: + raise ValueError(f"{field} must be 'api_key' or 'oauth'.") + if ( + field in _LLM_CONFIG_BASE_URL_FIELDS + and value + and not value.startswith(("http://", "https://")) + ): + raise ValueError(f"{field} must use http:// or https://.") + return value + + +def _patch_llm_config(payload: Any) -> dict[str, Any]: + if not isinstance(payload, dict): + raise ValueError("Request body must be an object.") + revision = payload.get("revision") + if not isinstance(revision, str) or not revision: + raise ValueError("revision is required.") + values = payload.get("values", {}) + secrets_payload = payload.get("secrets", {}) + clear_secrets = payload.get("clear_secrets", []) + builtin_profiles_supplied = "builtin_providers" in payload + builtin_profiles_payload = payload.get("builtin_providers") + if builtin_profiles_supplied and not isinstance(builtin_profiles_payload, list): + raise ValueError("builtin_providers must be a list.") + catalog_supplied = "model_catalog" in payload and not builtin_profiles_supplied + normalized_catalog = ( + normalize_builtin_model_catalog(payload.get("model_catalog")) + if catalog_supplied + else None + ) + if not isinstance(values, dict): + raise ValueError("values must be an object.") + if not isinstance(secrets_payload, dict): + raise ValueError("secrets must be an object.") + if not isinstance(clear_secrets, list) or not all( + isinstance(field, str) for field in clear_secrets + ): + raise ValueError("clear_secrets must be a list of strings.") + + unknown_values = set(values) - set(_LLM_CONFIG_VALUE_FIELDS) + unknown_secrets = set(secrets_payload) - set(_LLM_CONFIG_SECRET_FIELDS) + unknown_clears = set(clear_secrets) - set(_LLM_CONFIG_SECRET_FIELDS) + unknown = sorted(unknown_values | unknown_secrets | unknown_clears) + if unknown: + raise ValueError(f"Unsupported config fields: {', '.join(unknown)}.") + conflicting_secrets = sorted(set(secrets_payload) & set(clear_secrets)) + if conflicting_secrets: + raise ValueError( + "Secrets cannot be replaced and cleared together: " + f"{', '.join(conflicting_secrets)}." + ) + + with _config_file_lock: + config = load_config() + if revision != _llm_config_revision(config): + raise _ConfigRevisionConflict( + "config.yaml changed after it was loaded. Reload and try again." + ) + + changed_fields: list[str] = [] + for field, raw_value in values.items(): + value = _validated_llm_config_value(field, raw_value) + if getattr(config, field) != value: + setattr(config, field, value) + changed_fields.append(field) + + for field, raw_value in secrets_payload.items(): + if not isinstance(raw_value, str): + raise ValueError(f"{field} must be a string.") + value = raw_value.strip() + if len(value) > 8192: + raise ValueError(f"{field} must be at most 8192 characters.") + if value and getattr(config, field) != value: + setattr(config, field, value) + changed_fields.append(field) + + for field in clear_secrets: + if getattr(config, field): + setattr(config, field, "") + changed_fields.append(field) + + if catalog_supplied and config.model_catalog != normalized_catalog: + config.model_catalog = normalized_catalog + changed_fields.append("model_catalog") + + if builtin_profiles_supplied: + current_document = load_provider_profiles() + current_by_id = { + profile.id: profile for profile in current_document.builtins + } + prepared: list[dict[str, Any]] = [] + for index, raw_profile in enumerate(builtin_profiles_payload): + if not isinstance(raw_profile, dict): + raise ValueError(f"builtin_providers[{index}] must be an object.") + if raw_profile.get("managed") is False: + continue + candidate = dict(raw_profile) + provider_id = candidate.get("id") + if not isinstance(provider_id, str) or not provider_id.strip(): + raise ValueError(f"builtin_providers[{index}].id is required.") + provider_id = provider_id.strip() + candidate["id"] = provider_id + candidate.setdefault("name", provider_id) + candidate["adapter"] = builtin_provider_adapter(provider_id) + candidate.pop("managed", None) + candidate.pop("api_key_configured", None) + candidate.pop("api_key_hint", None) + if ( + not candidate.get("api_key") + and not candidate.get("clear_api_key") + and provider_id not in current_by_id + ): + candidate["api_key"] = builtin_provider_legacy_api_key( + config, provider_id + ) + prepared.append(candidate) + + if not current_document.builtins: + prepared_by_id = {candidate["id"]: candidate for candidate in prepared} + try: + legacy_catalog = ( + normalize_builtin_model_catalog(config.model_catalog) or [] + ) + except ValueError: + legacy_catalog = [] + migration_ids = {entry["provider"] for entry in legacy_catalog} + if config.provider in builtin_provider_ids(): + migration_ids.add(config.provider) + for provider_id in migration_ids: + if provider_id in prepared_by_id: + continue + effective = build_builtin_provider_profile( + config, + provider_id, + load_saved=False, + use_environment=False, + ) + models = [ + { + "id": entry["id"], + "name": entry["name"], + "model_id": entry["model_id"], + "enabled": entry["enabled"], + } + for entry in legacy_catalog + if entry["provider"] == provider_id + ] + if not models and provider_id == config.provider: + known = next( + ( + (name, model_id) + for name, model_id, candidate_provider in list_models_by_provider() + if candidate_provider == provider_id + and name == config.model + ), + None, + ) + models = [ + { + "id": config.model, + "name": config.model, + "model_id": known[1] if known else config.model, + "enabled": True, + } + ] + prepared.append( + { + "id": provider_id, + "name": provider_id, + "adapter": builtin_provider_adapter(provider_id), + "base_url": effective.base_url, + "api_key": builtin_provider_legacy_api_key( + config, provider_id + ), + "auth_mode": effective.auth_mode, + "enabled": True, + "models": models, + } + ) + + prepared_entries = { + (model.get("id"), candidate.get("id")) + for candidate in prepared + if candidate.get("enabled", True) + for model in candidate.get("models", []) + if isinstance(model, dict) and model.get("enabled", True) + } + prepared_entries.update( + (model.id, profile.id) + for profile in current_document.providers + if profile.enabled + for model in profile.models + if model.enabled + ) + if (config.model, config.provider) not in prepared_entries: + raise ValueError( + f"Default model {config.model!r} is not enabled for provider " + f"{config.provider!r}." + ) + + before = current_document.builtins + updated_document = replace_provider_profiles({"builtins": prepared}) + if updated_document.builtins != before: + changed_fields.append("builtin_providers") + for profile in updated_document.builtins: + if profile.id == "anthropic" and ( + config.anthropic_auth_mode != profile.auth_mode + ): + config.anthropic_auth_mode = profile.auth_mode + changed_fields.append("anthropic_auth_mode") + if profile.id == "openai" and ( + config.openai_auth_mode != profile.auth_mode + ): + config.openai_auth_mode = profile.auth_mode + changed_fields.append("openai_auth_mode") + + managed_builtin_entries = list_configured_builtin_model_entries() + if managed_builtin_entries or config.model_catalog is not None: + available = set(managed_builtin_entries) + if not managed_builtin_entries: + available.update( + list_builtin_model_catalog_entries(config.model_catalog) or [] + ) + available.update(list_configured_model_entries()) + selected = (config.model, config.provider) + if not any( + name == selected[0] and provider == selected[1] + for name, _, provider in available + ): + raise ValueError( + f"Default model {config.model!r} is not enabled for provider " + f"{config.provider!r}." + ) + + if any(field != "builtin_providers" for field in changed_fields): + save_config(config) + response = _llm_config_public(config) + response["changed_fields"] = sorted(set(changed_fields)) + response["restart_required"] = any( + field not in _LLM_CONFIG_IMMEDIATE_FIELDS for field in changed_fields + ) + return response async def get_models(_request: Request) -> JSONResponse: """Return the model registry as ``{entries, default}``. - ``entries`` preserves the registry order so the WebUI picker can - rank providers per short name the same way the backend would. - Mirrors the TUI ``/model`` picker by appending locally-pulled - Ollama models when ``ollama_base_url`` is configured — same - ``discover_ollama_models()`` call, same 1.5-s timeout, same - fail-soft semantics (the probe returns ``[]`` on any error, never - raises). The TUI's "Custom Ollama model…" sentinel is intentionally - omitted: that's a widget-specific input affordance, not part of - the registry surface. + Managed built-in and custom entries come from ``providers.yaml``. Before + that file exists, the legacy ``config.yaml`` model catalog/static registry + remains available for backward compatibility. - ``default`` reflects the deployment's currently-configured fallback - (``config.yaml``'s ``model`` / ``provider`` — what ``/model reset`` - would land on). Returned even when the configured pair isn't in - the registry, so the picker can still label it. + ``default`` is the configured pair when that pair remains available. If a + WebUI registry has replaced the legacy catalog and the persisted pair is no + longer present, the first enabled registry entry becomes the effective + default so new WebUI runs cannot route through stale provider settings. Uses ``get_effective_config()`` (not ``load_config()``) so env-var overrides like ``OLLAMA_BASE_URL`` from ``_ENV_MAPPINGS`` are @@ -84,16 +582,290 @@ async def get_models(_request: Request) -> JSONResponse: for name, model_id, provider in await list_model_picker_entries( getattr(cfg, "ollama_base_url", None), include_custom_ollama=False, + model_catalog=getattr(cfg, "model_catalog", None), ) ] + default_entry = next( + ( + {"name": entry["name"], "provider": entry["provider"]} + for entry in entries + if entry["name"] == cfg.model and entry["provider"] == cfg.provider + ), + None, + ) + if default_entry is None and entries: + default_entry = { + "name": entries[0]["name"], + "provider": entries[0]["provider"], + } + try: + profiles_revision = await asyncio.to_thread(provider_profiles_public_revision) + except ProviderProfileError: + profiles_revision = "invalid" return JSONResponse( { "entries": entries, - "default": {"name": cfg.model, "provider": cfg.provider}, + "default": default_entry, + "revision": profiles_revision, } ) +def _provider_admin_error(request: Request) -> JSONResponse | None: + expected = get_provider_admin_token() + if not expected: + return JSONResponse( + {"error": "Provider management is not enabled for this deployment."}, + status_code=503, + ) + supplied = request.headers.get("x-evoscientist-admin-token", "") + if not supplied or not secrets.compare_digest(supplied, expected): + return JSONResponse({"error": "Forbidden."}, status_code=403) + return None + + +async def provider_profiles_endpoint(request: Request) -> JSONResponse: + """Read or replace runtime-editable provider profiles. + + API keys are accepted on writes but never returned. Existing keys are kept + when the client sends an empty value, allowing the WebUI to edit metadata + without receiving the secret first. + """ + auth_error = _provider_admin_error(request) + if auth_error is not None: + return auth_error + + try: + if request.method == "GET": + document = await asyncio.to_thread(load_provider_profiles) + else: + payload = await request.json() + document = await asyncio.to_thread(replace_provider_profiles, payload) + return JSONResponse(provider_profiles_public(document)) + except ProviderProfileError as exc: + return JSONResponse({"error": str(exc)}, status_code=400) + except (json.JSONDecodeError, UnicodeDecodeError): + return JSONResponse( + {"error": "Request body must be valid JSON."}, status_code=400 + ) + except Exception: + _logger.exception("Provider profile request failed") + return JSONResponse( + {"error": "Provider profile request failed."}, status_code=500 + ) + + +async def llm_config_endpoint(request: Request) -> JSONResponse: + """Read or patch the LLM-related fields persisted in config.yaml.""" + auth_error = _provider_admin_error(request) + if auth_error is not None: + return auth_error + + try: + if request.method == "GET": + response = await asyncio.to_thread(_read_llm_config) + response["changed_fields"] = [] + response["restart_required"] = False + return JSONResponse(response) + payload = await request.json() + if request.method == "POST": + if not isinstance(payload, dict): + raise ValueError("Request body must be an object.") + action = payload.get("action") + provider_draft = payload.get("provider") + if not isinstance(provider_draft, dict): + raise ValueError("provider must be an object.") + provider_id = provider_draft.get("id") + if not isinstance(provider_id, str) or not provider_id.strip(): + raise ValueError("provider.id is required.") + cfg = await asyncio.to_thread(get_effective_config) + base_url = provider_draft.get("base_url") + api_key = provider_draft.get("api_key") + if base_url is not None and not isinstance(base_url, str): + raise ValueError("provider.base_url must be a string.") + if api_key is not None and not isinstance(api_key, str): + raise ValueError("provider.api_key must be a string.") + profile = build_builtin_provider_profile( + cfg, + provider_id.strip(), + base_url_override=base_url, + api_key_override=api_key, + clear_api_key=provider_draft.get("clear_api_key") is True, + ) + if action == "list_models": + models = await discover_provider_models(profile) + return JSONResponse( + { + "models": [ + {"model_id": model.model_id, "name": model.name} + for model in models + ] + } + ) + if action == "test_model": + model = _provider_model_from_payload(payload.get("model")) + result = await test_provider_model(profile, model) + return JSONResponse( + { + "ok": True, + "latency_ms": result.latency_ms, + "response": result.response, + } + ) + raise ValueError("action must be list_models or test_model.") + return JSONResponse(await asyncio.to_thread(_patch_llm_config, payload)) + except _ConfigRevisionConflict as exc: + return JSONResponse({"error": str(exc)}, status_code=409) + except (json.JSONDecodeError, UnicodeDecodeError): + return JSONResponse( + {"error": "Request body must be valid JSON."}, status_code=400 + ) + except ValueError as exc: + return JSONResponse({"error": str(exc)}, status_code=400) + except ProviderOperationError as exc: + return JSONResponse({"error": str(exc)}, status_code=502) + except Exception: + _logger.exception("LLM config request failed") + return JSONResponse({"error": "LLM config request failed."}, status_code=500) + + +def _persist_default_model(model: str, provider: str) -> None: + """Update the persisted default pair without disturbing other settings.""" + with _config_file_lock: + config = load_config() + config.model = model + config.provider = provider + save_config(config) + + +async def default_model_endpoint(request: Request) -> JSONResponse: + """Validate and persist the deployment's default model selection.""" + auth_error = _provider_admin_error(request) + if auth_error is not None: + return auth_error + + try: + payload = await request.json() + if not isinstance(payload, dict): + raise ValueError("Request body must be an object.") + + model = payload.get("model") + provider = payload.get("provider") + if not isinstance(model, str) or not model.strip(): + raise ValueError("model is required.") + if not isinstance(provider, str) or not provider.strip(): + raise ValueError("provider is required.") + model = model.strip() + provider = provider.strip() + if len(model) > 300: + raise ValueError("model must be at most 300 characters.") + if len(provider) > 64: + raise ValueError("provider must be at most 64 characters.") + + cfg = await asyncio.to_thread(get_effective_config) + entries = await list_model_picker_entries( + getattr(cfg, "ollama_base_url", None), + include_custom_ollama=False, + model_catalog=getattr(cfg, "model_catalog", None), + ) + if not any( + name == model and entry_provider == provider + for name, _, entry_provider in entries + ): + raise ValueError( + f"Model {model!r} is not configured for provider {provider!r}." + ) + + await asyncio.to_thread(_persist_default_model, model, provider) + return JSONResponse({"default": {"name": model, "provider": provider}}) + except (json.JSONDecodeError, UnicodeDecodeError): + return JSONResponse( + {"error": "Request body must be valid JSON."}, status_code=400 + ) + except (ProviderProfileError, ValueError) as exc: + return JSONResponse({"error": str(exc)}, status_code=400) + except Exception: + _logger.exception("Default model request failed") + return JSONResponse({"error": "Default model request failed."}, status_code=500) + + +def _provider_model_from_payload(raw: Any) -> ProviderModel: + if not isinstance(raw, dict): + raise ProviderProfileError("model must be an object.") + model_id = raw.get("id") + name = raw.get("name") + upstream_id = raw.get("model_id") + if not isinstance(model_id, str) or not model_id.strip(): + raise ProviderProfileError("model.id is required.") + if not isinstance(name, str) or not name.strip(): + raise ProviderProfileError("model.name is required.") + if not isinstance(upstream_id, str) or not upstream_id.strip(): + raise ProviderProfileError("model.model_id is required.") + if len(model_id.strip()) > 64: + raise ProviderProfileError("model.id must be at most 64 characters.") + if len(name.strip()) > 120: + raise ProviderProfileError("model.name must be at most 120 characters.") + if len(upstream_id.strip()) > 300: + raise ProviderProfileError("model.model_id must be at most 300 characters.") + return ProviderModel( + id=model_id.strip(), + name=name.strip(), + model_id=upstream_id.strip(), + enabled=True, + ) + + +async def provider_actions_endpoint(request: Request) -> JSONResponse: + """Discover provider models or run a real connectivity test.""" + auth_error = _provider_admin_error(request) + if auth_error is not None: + return auth_error + + try: + payload = await request.json() + if not isinstance(payload, dict): + raise ProviderProfileError("Request body must be an object.") + action = payload.get("action") + profile = await asyncio.to_thread( + resolve_provider_profile_draft, payload.get("provider") + ) + + if action == "list_models": + models = await discover_provider_models(profile) + return JSONResponse( + { + "models": [ + {"model_id": model.model_id, "name": model.name} + for model in models + ] + } + ) + if action == "test_model": + model = _provider_model_from_payload(payload.get("model")) + result = await test_provider_model(profile, model) + return JSONResponse( + { + "ok": True, + "latency_ms": result.latency_ms, + "response": result.response, + } + ) + raise ProviderProfileError("action must be list_models or test_model.") + except (json.JSONDecodeError, UnicodeDecodeError): + return JSONResponse( + {"error": "Request body must be valid JSON."}, status_code=400 + ) + except ProviderProfileError as exc: + return JSONResponse({"error": str(exc)}, status_code=400) + except ProviderOperationError as exc: + return JSONResponse({"error": str(exc)}, status_code=502) + except Exception: + _logger.exception("Provider action request failed") + return JSONResponse( + {"error": "Provider action request failed."}, status_code=500 + ) + + def _message_type(message: Any) -> str | None: if isinstance(message, dict): role = message.get("role") @@ -198,7 +970,9 @@ def _is_tool_selection_text(text: str) -> bool: if _is_tool_selection_payload(stripped): return True objects = _split_json_objects(stripped) - return objects is not None and all(_is_tool_selection_payload(obj) for obj in objects) + return objects is not None and all( + _is_tool_selection_payload(obj) for obj in objects + ) def _extract_final_answer(messages: list[Any]) -> str: @@ -274,7 +1048,9 @@ async def _get_thread_messages_for_http(thread_id: str) -> list: return [] -async def _read_thread_runtime_state(request: Request, thread_id: str) -> dict[str, Any]: +async def _read_thread_runtime_state( + request: Request, thread_id: str +) -> dict[str, Any]: """Read thread status from the co-hosted langgraph-api endpoints.""" base_url = f"{request.url.scheme}://{request.url.netloc}" timeout = httpx.Timeout(2.0, connect=0.5) @@ -351,13 +1127,594 @@ async def get_final_answer(request: Request) -> JSONResponse: ) +def _scope_service_authorized(request: Request) -> JSONResponse | None: + """Require the server-only credential for workspace registry routes.""" + + expected = os.getenv("EVOSCIENTIST_BACKEND_SERVICE_TOKEN", "").strip() + if not expected: + return JSONResponse( + {"error": "workspace registry service token is not configured"}, + status_code=503, + ) + header = request.headers.get("authorization", "") + if not header.startswith("Bearer ") or not secrets.compare_digest( + header[7:], expected + ): + return JSONResponse({"error": "unauthorized"}, status_code=401) + return None + + +def _scope_payload(record: Any) -> dict[str, Any]: + return { + "deployment_id": record.deployment_id, + "scope_id": record.scope_id, + "primary_thread_id": record.primary_thread_id, + "primary_owner_id": record.primary_owner_id, + "state": record.state, + "revision": record.revision, + } + + +def _call_scope_registry(method: str, *args: Any, **kwargs: Any) -> Any: + """Run a deployment-scoped registry operation outside the ASGI loop.""" + + from EvoScientist.scope_registry import get_scope_registry + from EvoScientist.workspace_scope import current_deployment_id + + registry = get_scope_registry() + return getattr(registry, method)(current_deployment_id(), *args, **kwargs) + + +def _provision_scope(thread_id: str, scope_id: str | None) -> Any: + """Create a primary scope outside the ASGI loop.""" + + from EvoScientist.workspace_scope import ( + current_deployment_id, + provision_conversation_scope, + ) + + return provision_conversation_scope( + thread_id, + deployment_id=current_deployment_id(), + scope_id=scope_id, + ) + + +def _reserve_scope_run( + scope_id: str, + run_request_id: str, + turn_id: str, + request_hash: str, + interrupt_key: str | None = None, +) -> Any: + """Reserve and validate a primary-run owner off the ASGI loop.""" + + from EvoScientist.scope_registry import get_scope_registry + from EvoScientist.workspace_scope import current_deployment_id + + deployment_id = current_deployment_id() + registry = get_scope_registry() + run = registry.reserve_run( + deployment_id, + scope_id, + run_request_id, + turn_id, + request_hash, + interrupt_key=interrupt_key, + ) + scope = registry.get(deployment_id, scope_id) + registry.assert_runtime( + deployment_id, + scope_id, + scope.primary_thread_id, + run.run_owner_id, + ) + return run + + +def _reserve_scope_turn(scope_id: str, turn_id: str, request_hash: str) -> Any: + """Compatibility wrapper for the former turn-keyed internal endpoint.""" + + return _reserve_scope_run(scope_id, turn_id, turn_id, request_hash) + + +def _transition_scope(scope_id: str, expected_revision: int, state: str) -> Any: + """Apply a lifecycle transition; operation-time validation observes it.""" + + from EvoScientist.scope_registry import get_scope_registry + from EvoScientist.workspace_scope import current_deployment_id + + deployment_id = current_deployment_id() + record = get_scope_registry().transition_scope( + deployment_id, + scope_id, + expected_revision=expected_revision, + state=state, + ) + return record + + +async def provision_workspace_scope(request: Request) -> JSONResponse: + denied = _scope_service_authorized(request) + if denied is not None: + return denied + try: + payload = await request.json() + except json.JSONDecodeError: + payload = None + if not isinstance(payload, dict) or not isinstance(payload.get("thread_id"), str): + return JSONResponse({"error": "thread_id is required"}, status_code=400) + scope_id = payload.get("scope_id") + if scope_id is not None and not isinstance(scope_id, str): + return JSONResponse({"error": "scope_id must be a string"}, status_code=400) + try: + record = await asyncio.to_thread( + _provision_scope, payload["thread_id"], scope_id + ) + except Exception as exc: + _logger.warning("Could not provision workspace scope: %s", exc) + return JSONResponse( + {"error": "could not provision workspace scope"}, status_code=409 + ) + return JSONResponse(_scope_payload(record), status_code=201) + + +async def get_workspace_scope(request: Request) -> JSONResponse: + denied = _scope_service_authorized(request) + if denied is not None: + return denied + thread_id = str(request.path_params["thread_id"]) + try: + from EvoScientist.scope_registry import ScopeNotFoundError + + record = await asyncio.to_thread( + _call_scope_registry, "get_by_thread", thread_id + ) + except ScopeNotFoundError: + return JSONResponse({"error": "workspace scope not found"}, status_code=404) + except Exception as exc: + _logger.warning("Could not query workspace scope: %s", exc) + return JSONResponse( + {"error": "could not query workspace scope"}, status_code=500 + ) + return JSONResponse(_scope_payload(record)) + + +async def list_workspace_owners(request: Request) -> JSONResponse: + denied = _scope_service_authorized(request) + if denied is not None: + return denied + scope_id = str(request.path_params["scope_id"]) + try: + owners = await asyncio.to_thread(_call_scope_registry, "owners", scope_id) + except Exception as exc: + _logger.warning("Could not list workspace owners: %s", exc) + return JSONResponse( + {"error": "could not list workspace owners"}, status_code=404 + ) + return JSONResponse( + { + "owners": [ + { + "owner_id": owner.owner_id, + "scope_id": owner.scope_id, + "owner_type": owner.owner_type, + "resource_id": owner.resource_id, + "state": owner.state, + } + for owner in owners + ] + } + ) + + +async def get_workspace_cutover_lock(request: Request) -> JSONResponse: + denied = _scope_service_authorized(request) + if denied is not None: + return denied + try: + lock = await asyncio.to_thread( + _call_scope_registry, "active_lock", "workspace-lifecycle" + ) + if lock is None: + lock = await asyncio.to_thread( + _call_scope_registry, "active_lock", "workspace-cutover" + ) + except Exception as exc: + _logger.warning("Could not query workspace cutover lock: %s", exc) + return JSONResponse( + {"error": "could not query workspace cutover lock"}, status_code=500 + ) + return JSONResponse( + { + "active": lock is not None, + "operation_id": lock.operation_id if lock else None, + "expires_at": lock.expires_at if lock else None, + } + ) + + +async def register_workspace_owner(request: Request) -> JSONResponse: + denied = _scope_service_authorized(request) + if denied is not None: + return denied + scope_id = str(request.path_params["scope_id"]) + try: + payload = await request.json() + except json.JSONDecodeError: + payload = None + if not isinstance(payload, dict) or not isinstance(payload.get("owner_type"), str): + return JSONResponse({"error": "owner_type is required"}, status_code=400) + allowed = {"owner_type", "resource_id", "parent_owner_id", "owner_id", "state"} + if set(payload) - allowed: + return JSONResponse({"error": "unsupported owner field"}, status_code=400) + try: + owner = await asyncio.to_thread( + _call_scope_registry, + "register_owner", + scope_id, + owner_type=payload["owner_type"], + resource_id=payload.get("resource_id"), + parent_owner_id=payload.get("parent_owner_id"), + owner_id=payload.get("owner_id"), + state=payload.get("state", "reserved"), + ) + except Exception as exc: + _logger.warning("Could not register workspace owner: %s", exc) + return JSONResponse( + {"error": "could not register workspace owner"}, status_code=409 + ) + return JSONResponse( + { + "owner_id": owner.owner_id, + "scope_id": owner.scope_id, + "owner_type": owner.owner_type, + "resource_id": owner.resource_id, + "state": owner.state, + }, + status_code=201, + ) + + +async def bind_workspace_owner(request: Request) -> JSONResponse: + denied = _scope_service_authorized(request) + if denied is not None: + return denied + scope_id = str(request.path_params["scope_id"]) + owner_id = str(request.path_params["owner_id"]) + try: + payload = await request.json() + except json.JSONDecodeError: + payload = None + if not isinstance(payload, dict) or not isinstance(payload.get("resource_id"), str): + return JSONResponse({"error": "resource_id is required"}, status_code=400) + if set(payload) - {"resource_id", "state"}: + return JSONResponse({"error": "unsupported owner field"}, status_code=400) + state = payload.get("state", "active") + if state not in {"active", "terminal"}: + return JSONResponse({"error": "unsupported owner state"}, status_code=400) + try: + owner = await asyncio.to_thread( + _call_scope_registry, + "bind_owner", + scope_id, + owner_id, + payload["resource_id"], + state=state, + ) + except Exception as exc: + _logger.warning("Could not bind workspace owner: %s", exc) + return JSONResponse( + {"error": "could not bind workspace owner"}, status_code=409 + ) + return JSONResponse( + { + "owner_id": owner.owner_id, + "scope_id": owner.scope_id, + "owner_type": owner.owner_type, + "resource_id": owner.resource_id, + "state": owner.state, + } + ) + + +async def get_workspace_owner_by_resource(request: Request) -> JSONResponse: + denied = _scope_service_authorized(request) + if denied is not None: + return denied + scope_id = str(request.path_params["scope_id"]) + resource_id = str(request.path_params["resource_id"]) + try: + owner = await asyncio.to_thread( + _call_scope_registry, "get_owner_by_resource", resource_id + ) + if owner.scope_id != scope_id: + return JSONResponse({"error": "workspace owner not found"}, status_code=404) + except Exception: + return JSONResponse({"error": "workspace owner not found"}, status_code=404) + return JSONResponse( + { + "owner_id": owner.owner_id, + "scope_id": owner.scope_id, + "owner_type": owner.owner_type, + "resource_id": owner.resource_id, + "state": owner.state, + } + ) + + +async def transition_workspace_scope(request: Request) -> JSONResponse: + denied = _scope_service_authorized(request) + if denied is not None: + return denied + scope_id = str(request.path_params["scope_id"]) + try: + payload = await request.json() + except json.JSONDecodeError: + payload = None + if ( + not isinstance(payload, dict) + or not isinstance(payload.get("expected_revision"), int) + or payload.get("state") not in {"active", "deleting", "deleted"} + ): + return JSONResponse( + {"error": "expected_revision and a valid state are required"}, + status_code=400, + ) + try: + record = await asyncio.to_thread( + _transition_scope, + scope_id, + payload["expected_revision"], + payload["state"], + ) + except Exception as exc: + _logger.warning("Could not transition workspace scope: %s", exc) + return JSONResponse( + {"error": "could not transition workspace scope"}, status_code=409 + ) + return JSONResponse(_scope_payload(record)) + + +def _run_reservation_payload(run: Any) -> dict[str, Any]: + return { + "run_request_id": run.run_request_id, + "turn_id": run.turn_id, + "interrupt_key": run.interrupt_key, + "request_hash": run.request_hash, + "run_owner_id": run.run_owner_id, + "run_id": run.run_id, + "state": run.state, + } + + +def _reservation_error_response(exc: Exception, *, operation: str) -> JSONResponse: + """Return actionable conflict details without leaking registry internals.""" + + error_type = type(exc).__name__ + error_code = { + "ScopeIdempotencyConflictError": "idempotency_key_conflict", + "ScopeInterruptResolvedError": "interrupt_already_resolved", + "ScopeAccessError": "workspace_scope_unavailable", + }.get(error_type, "workspace_reservation_conflict") + if error_code == "workspace_reservation_conflict": + _logger.warning("Could not %s workspace run: %s", operation, exc) + else: + _logger.info("Workspace run %s rejected: %s", operation, exc) + return JSONResponse( + {"error": str(exc) or "workspace run reservation failed", "code": error_code}, + status_code=409, + ) + + +async def reserve_workspace_run(request: Request) -> JSONResponse: + denied = _scope_service_authorized(request) + if denied is not None: + return denied + scope_id = str(request.path_params["scope_id"]) + try: + payload = await request.json() + except json.JSONDecodeError: + payload = None + if ( + not isinstance(payload, dict) + or not isinstance(payload.get("run_request_id"), str) + or not isinstance(payload.get("turn_id"), str) + or not isinstance(payload.get("request_hash"), str) + or ( + "interrupt_key" in payload + and payload["interrupt_key"] is not None + and not isinstance(payload["interrupt_key"], str) + ) + ): + return JSONResponse( + {"error": "run_request_id, turn_id and request_hash are required"}, + status_code=400, + ) + try: + run = await asyncio.to_thread( + _reserve_scope_run, + scope_id, + payload["run_request_id"], + payload["turn_id"], + payload["request_hash"], + payload.get("interrupt_key"), + ) + except Exception as exc: + return _reservation_error_response(exc, operation="reserve") + return JSONResponse(_run_reservation_payload(run), status_code=201) + + +async def bind_workspace_run(request: Request) -> JSONResponse: + denied = _scope_service_authorized(request) + if denied is not None: + return denied + scope_id = str(request.path_params["scope_id"]) + run_request_id = str(request.path_params["run_request_id"]) + try: + payload = await request.json() + except json.JSONDecodeError: + payload = None + if not isinstance(payload, dict) or not isinstance(payload.get("run_id"), str): + return JSONResponse({"error": "run_id is required"}, status_code=400) + try: + run = await asyncio.to_thread( + _call_scope_registry, + "bind_run", + scope_id, + run_request_id, + payload["run_id"], + ) + except Exception as exc: + return _reservation_error_response(exc, operation="bind") + return JSONResponse(_run_reservation_payload(run)) + + +async def reserve_workspace_turn(request: Request) -> JSONResponse: + """Compatibility endpoint for clients that still key a run by turn id.""" + + denied = _scope_service_authorized(request) + if denied is not None: + return denied + scope_id = str(request.path_params["scope_id"]) + try: + payload = await request.json() + except json.JSONDecodeError: + payload = None + if ( + not isinstance(payload, dict) + or not isinstance(payload.get("turn_id"), str) + or not isinstance(payload.get("request_hash"), str) + ): + return JSONResponse( + {"error": "turn_id and request_hash are required"}, status_code=400 + ) + try: + run = await asyncio.to_thread( + _reserve_scope_turn, + scope_id, + payload["turn_id"], + payload["request_hash"], + ) + except Exception as exc: + return _reservation_error_response(exc, operation="reserve") + return JSONResponse(_run_reservation_payload(run), status_code=201) + + +async def bind_workspace_turn(request: Request) -> JSONResponse: + """Compatibility endpoint for clients that still key a run by turn id.""" + + denied = _scope_service_authorized(request) + if denied is not None: + return denied + scope_id = str(request.path_params["scope_id"]) + turn_id = str(request.path_params["turn_id"]) + try: + payload = await request.json() + except json.JSONDecodeError: + payload = None + if not isinstance(payload, dict) or not isinstance(payload.get("run_id"), str): + return JSONResponse({"error": "run_id is required"}, status_code=400) + try: + run = await asyncio.to_thread( + _call_scope_registry, + "bind_turn", + scope_id, + turn_id, + payload["run_id"], + ) + except Exception as exc: + return _reservation_error_response(exc, operation="bind") + return JSONResponse(_run_reservation_payload(run)) + + app = Starlette( routes=[ Route("/api/models", get_models, methods=["GET"]), + Route( + "/api/provider-profiles", + provider_profiles_endpoint, + methods=["GET", "PUT"], + ), + Route( + "/api/config", + llm_config_endpoint, + methods=["GET", "PATCH", "POST"], + ), + Route( + "/api/default-model", + default_model_endpoint, + methods=["PUT"], + ), + Route( + "/api/provider-actions", + provider_actions_endpoint, + methods=["POST"], + ), Route( "/api/threads/{thread_id}/final-answer", get_final_answer, methods=["GET"], ), + Route( + "/internal/workspace-scopes/provision", + provision_workspace_scope, + methods=["POST"], + ), + Route( + "/internal/workspace-scopes/by-thread/{thread_id}", + get_workspace_scope, + methods=["GET"], + ), + Route( + "/internal/workspace-scopes/cutover-lock", + get_workspace_cutover_lock, + methods=["GET"], + ), + Route( + "/internal/workspace-scopes/{scope_id}/owners", + list_workspace_owners, + methods=["GET"], + ), + Route( + "/internal/workspace-scopes/{scope_id}/owners", + register_workspace_owner, + methods=["POST"], + ), + Route( + "/internal/workspace-scopes/{scope_id}/owners/{owner_id}", + bind_workspace_owner, + methods=["PATCH"], + ), + Route( + "/internal/workspace-scopes/{scope_id}/owners/by-resource/{resource_id}", + get_workspace_owner_by_resource, + methods=["GET"], + ), + Route( + "/internal/workspace-scopes/{scope_id}", + transition_workspace_scope, + methods=["PATCH"], + ), + Route( + "/internal/workspace-scopes/{scope_id}/turns/reserve", + reserve_workspace_turn, + methods=["POST"], + ), + Route( + "/internal/workspace-scopes/{scope_id}/turns/{turn_id}", + bind_workspace_turn, + methods=["PATCH"], + ), + Route( + "/internal/workspace-scopes/{scope_id}/runs/reserve", + reserve_workspace_run, + methods=["POST"], + ), + Route( + "/internal/workspace-scopes/{scope_id}/runs/{run_request_id}", + bind_workspace_run, + methods=["PATCH"], + ), ] ) diff --git a/EvoScientist/langgraph_dev/manager.py b/EvoScientist/langgraph_dev/manager.py index 69e5068..f12793a 100644 --- a/EvoScientist/langgraph_dev/manager.py +++ b/EvoScientist/langgraph_dev/manager.py @@ -20,6 +20,7 @@ import shutil import subprocess import threading import time +import uuid from dataclasses import dataclass from pathlib import Path @@ -177,7 +178,9 @@ class WorkspaceMismatchError(RuntimeError): """ -def _write_workspace_sidecar(workspace_dir: Path, pid: int) -> None: +def _write_workspace_sidecar( + workspace_dir: Path, pid: int, *, port: int = _DEFAULT_PORT +) -> None: """Record the workspace + pid of the langgraph dev we just started. Atomic write via temp-file + ``os.replace``: without this, a concurrent @@ -194,8 +197,20 @@ def _write_workspace_sidecar(workspace_dir: Path, pid: int) -> None: try: RUNTIME.pid_dir.mkdir(parents=True, exist_ok=True) tmp = RUNTIME.workspace_sidecar.with_suffix(".json.tmp") + resolved_workspace = workspace_dir.resolve() + deployment_id = os.getenv("EVOSCIENTIST_DEPLOYMENT_ID") or str( + uuid.uuid5(uuid.NAMESPACE_URL, f"evoscientist:{resolved_workspace}") + ) tmp.write_text( - json.dumps({"workspace": str(workspace_dir), "pid": pid}), encoding="utf-8" + json.dumps( + { + "workspace": str(workspace_dir), + "pid": pid, + "deployment_id": deployment_id, + "api_url": _base_url(port), + } + ), + encoding="utf-8", ) os.replace(tmp, RUNTIME.workspace_sidecar) except OSError as exc: @@ -231,11 +246,14 @@ def _read_workspace_sidecar() -> dict | None: return data -def _unlink_workspace_sidecar() -> None: +def _unlink_workspace_sidecar( + runtime: LanggraphRuntimePaths | None = None, +) -> None: """Best-effort sidecar removal — called alongside every ``RUNTIME.pid_file.unlink()`` so the workspace fingerprint never outlives the PID file it pairs with.""" + runtime = runtime or RUNTIME try: - RUNTIME.workspace_sidecar.unlink() + runtime.workspace_sidecar.unlink() except OSError: pass @@ -270,6 +288,16 @@ _LOG_OFFSET_AT_START: int = 0 # langgraph dev log. Mirrors langgraph_api/tunneling/cloudflare.py. _TUNNEL_URL_RE = re.compile(r"https://[A-Za-z0-9.-]+\.trycloudflare\.com") + +def current_log_start_offset() -> int: + """Return the byte offset where the current server session started. + + Consumers that mirror the Gateway log can start here to avoid replaying + output from earlier deploy sessions that share the append-only log file. + """ + return _LOG_OFFSET_AT_START + + # Whether async sub-agents are usable in this process. # # - CLI / serve parent process: starts False; flipped True after @@ -743,7 +771,7 @@ def start_langgraph_dev( except Exception: pass RUNTIME.pid_file.write_text(str(proc.pid), encoding="utf-8") - _write_workspace_sidecar(workspace_dir=workspace_dir, pid=proc.pid) + _write_workspace_sidecar(workspace_dir=workspace_dir, pid=proc.pid, port=port) global _PROCESS_WORKSPACE _PROCESS = proc _PROCESS_WORKSPACE = workspace_dir @@ -820,7 +848,11 @@ def read_tunnel_url(timeout: float = 35.0, poll_interval: float = 0.5) -> str | return None -def stop_langgraph_dev(proc: subprocess.Popen | None = None) -> None: +def stop_langgraph_dev( + proc: subprocess.Popen | None = None, + *, + runtime: LanggraphRuntimePaths | None = None, +) -> None: """Gracefully stop a langgraph dev process. Sends SIGTERM to the process group (langgraph dev spawns worker children), @@ -831,6 +863,7 @@ def stop_langgraph_dev(proc: subprocess.Popen | None = None) -> None: (which also hold ``_LOCK``) don't observe partially-cleared state. """ global _PROCESS, _PROCESS_WORKSPACE + runtime = runtime or RUNTIME with _LOCK: proc = proc if proc is not None else _PROCESS if proc is None: @@ -879,12 +912,12 @@ def stop_langgraph_dev(proc: subprocess.Popen | None = None) -> None: if proc is _PROCESS: _PROCESS = None _PROCESS_WORKSPACE = None - if RUNTIME.pid_file.exists(): + if runtime.pid_file.exists(): try: - RUNTIME.pid_file.unlink() + runtime.pid_file.unlink() except OSError: pass - _unlink_workspace_sidecar() + _unlink_workspace_sidecar(runtime) # Note: ``.langgraph_api/`` is intentionally NOT removed — it holds # langgraph dev's persisted async-task / scheduler / Store state that @@ -1064,5 +1097,8 @@ def _ensure_langgraph_dev_locked( return None _ASYNC_SUBAGENTS_AVAILABLE = True - atexit.register(stop_langgraph_dev, proc) + # Capture the runtime paths alongside the process. Embedded callers and + # tests may replace the module-level RUNTIME before interpreter shutdown; + # consulting that later would unlink an unrelated live deployment's files. + atexit.register(stop_langgraph_dev, proc, runtime=RUNTIME) return proc diff --git a/EvoScientist/llm/models.py b/EvoScientist/llm/models.py index efa61d2..f40c79c 100644 --- a/EvoScientist/llm/models.py +++ b/EvoScientist/llm/models.py @@ -9,12 +9,28 @@ endpoints) and convenient short names for common models. from __future__ import annotations +import asyncio +import hashlib +import json import os +import re import warnings from typing import Any from langchain.chat_models import init_chat_model +from ..config.provider_profiles import ( + ProviderModel, + ProviderProfile, + ProviderProfileError, + get_builtin_provider_profile, + get_provider_profile, + get_provider_profile_revision, + get_provider_profiles_path, + list_configured_builtin_model_entries, + list_configured_model_entries, + resolve_provider_model, +) from .context_window import apply_known_context_window from .patches import ( _is_ccproxy_codex, @@ -241,6 +257,135 @@ MODELS: dict[str, tuple[str, str]] = { } DEFAULT_MODEL = "claude-sonnet-4-6" +_STATIC_PROVIDER_IDS = {provider for _, _, provider in _MODEL_ENTRIES} | {"ollama"} +_MODEL_CATALOG_ID_PATTERN = re.compile(r"^[a-z0-9][a-z0-9._-]{0,63}$") + + +def normalize_builtin_model_catalog(raw: Any) -> list[dict[str, Any]] | None: + """Validate and normalize the optional config.yaml built-in model catalog.""" + if raw is None: + return None + if not isinstance(raw, list): + raise ValueError("model_catalog must be a list or null.") + + normalized: list[dict[str, Any]] = [] + seen: set[tuple[str, str]] = set() + for index, item in enumerate(raw): + context = f"model_catalog[{index}]" + if not isinstance(item, dict): + raise ValueError(f"{context} must be an object.") + + provider = item.get("provider") + alias = item.get("id") + name = item.get("name") + model_id = item.get("model_id") + if not isinstance(provider, str) or not provider.strip(): + raise ValueError(f"{context}.provider is required.") + if not isinstance(alias, str) or not alias.strip(): + raise ValueError(f"{context}.id is required.") + if not isinstance(name, str) or not name.strip(): + raise ValueError(f"{context}.name is required.") + if not isinstance(model_id, str) or not model_id.strip(): + raise ValueError(f"{context}.model_id is required.") + + provider = provider.strip() + alias = alias.strip() + name = name.strip() + model_id = model_id.strip() + if provider not in _STATIC_PROVIDER_IDS: + raise ValueError( + f"{context}.provider must be a built-in provider, got {provider!r}." + ) + if not _MODEL_CATALOG_ID_PATTERN.fullmatch(alias): + raise ValueError( + f"{context}.id must start with a lowercase letter or digit and " + "contain only lowercase letters, digits, dots, underscores, or hyphens." + ) + if len(name) > 120: + raise ValueError(f"{context}.name must be at most 120 characters.") + if len(model_id) > 300: + raise ValueError(f"{context}.model_id must be at most 300 characters.") + + key = (provider, alias) + if key in seen: + raise ValueError( + f"Model alias {alias!r} is duplicated for provider {provider!r}." + ) + seen.add(key) + normalized.append( + { + "provider": provider, + "id": alias, + "name": name, + "model_id": model_id, + "enabled": bool(item.get("enabled", True)), + } + ) + return normalized + + +def list_builtin_model_catalog_entries( + raw: Any, +) -> list[tuple[str, str, str]] | None: + """Return enabled catalog entries, or ``None`` for legacy registry mode.""" + catalog = normalize_builtin_model_catalog(raw) + if catalog is None: + return None + return [ + (entry["id"], entry["model_id"], entry["provider"]) + for entry in catalog + if entry["enabled"] + ] + + +def _resolve_builtin_catalog_model(provider: str, model: str) -> str | None: + from ..config.settings import load_config + + profile = get_builtin_provider_profile(provider) + if profile is not None: + if not profile.enabled: + return None + for candidate in profile.models: + if candidate.id == model: + return candidate.model_id if candidate.enabled else None + return None + try: + catalog = normalize_builtin_model_catalog(load_config().model_catalog) + except ValueError: + return None + if catalog is None: + return None + for entry in catalog: + if entry["provider"] == provider and entry["id"] == model: + return entry["model_id"] if entry["enabled"] else None + return None + + +def get_model_runtime_revision(provider: str | None) -> str | None: + """Return the dynamic profile revision used in model cache keys.""" + if not provider: + return None + if provider in _STATIC_PROVIDER_IDS: + managed_revision = get_provider_profile_revision(provider) + if managed_revision is not None: + return managed_revision + from ..config.settings import load_config + + try: + catalog = normalize_builtin_model_catalog(load_config().model_catalog) + except ValueError: + return "invalid" + if catalog is None: + return None + provider_catalog = [entry for entry in catalog if entry["provider"] == provider] + encoded = json.dumps( + provider_catalog, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest()[:16] + try: + return get_provider_profile_revision(provider) + except ProviderProfileError: + return None def get_models_for_provider(provider: str) -> list[tuple[str, str]]: @@ -252,7 +397,25 @@ def get_models_for_provider(provider: str) -> list[tuple[str, str]]: Returns: List of (short_name, model_id) tuples for the provider. """ - return [(name, model_id) for name, model_id, p in _MODEL_ENTRIES if p == provider] + if provider in _STATIC_PROVIDER_IDS: + managed = get_builtin_provider_profile(provider) + if managed is not None: + if not managed.enabled: + return [] + return [ + (model.id, model.model_id) for model in managed.models if model.enabled + ] + return [ + (name, model_id) + for name, model_id, entry_provider in _MODEL_ENTRIES + if entry_provider == provider + ] + profile = get_provider_profile(provider) + if profile is not None: + if not profile.enabled: + return [] + return [(model.id, model.model_id) for model in profile.models if model.enabled] + return [] def _env_flag_enabled(name: str) -> bool: @@ -365,6 +528,10 @@ def _apply_auto_config( def get_chat_model( model: str | None = None, provider: str | None = None, + *, + profile_override: ProviderProfile | None = None, + model_override: ProviderModel | None = None, + skip_auto_config: bool = False, **kwargs: Any, ) -> Any: """Get a chat model instance. @@ -387,15 +554,42 @@ def get_chat_model( """ model = model or DEFAULT_MODEL - # Look up short name in registry (provider-aware) - model_id = None - if provider: + dynamic_profile = profile_override + dynamic_model = model_override + if dynamic_profile is not None: + provider = dynamic_profile.id + if dynamic_model is None: + dynamic_model = next( + ( + candidate + for candidate in dynamic_profile.models + if candidate.id == model or candidate.model_id == model + ), + None, + ) + if dynamic_model is None: + raise ValueError( + f"Model {model!r} is not configured for provider {dynamic_profile.id!r}." + ) + elif provider and provider not in _STATIC_PROVIDER_IDS: + try: + dynamic_resolution = resolve_provider_model(provider, model) + except ProviderProfileError as exc: + raise ValueError(str(exc)) from exc + if dynamic_resolution is not None: + dynamic_profile, dynamic_model = dynamic_resolution + + # Look up short name in the configured catalog, then the legacy registry. + model_id = dynamic_model.model_id if dynamic_model is not None else None + if model_id is None and provider in _STATIC_PROVIDER_IDS: + model_id = _resolve_builtin_catalog_model(provider, model) + if provider and dynamic_profile is None and model_id is None: # Try exact match with provider first for name, mid, p in _MODEL_ENTRIES: if name == model and p == provider: model_id = mid break - if model_id is None and model in MODELS: + if model_id is None and dynamic_profile is None and model in MODELS: model_id, default_provider = MODELS[model] provider = provider or default_provider @@ -422,19 +616,82 @@ def get_chat_model( ) _is_openai_proxy = False _original_provider: str | None = None - if provider == "anthropic": + managed_builtin_profile = ( + get_builtin_provider_profile(provider) + if provider in _STATIC_PROVIDER_IDS + else None + ) + if managed_builtin_profile is not None: + if not managed_builtin_profile.enabled: + raise ValueError(f"Provider {provider!r} is disabled.") + from ..config.settings import get_effective_config + from .provider_operations import build_builtin_provider_profile + + effective_profile = build_builtin_provider_profile( + get_effective_config(), provider + ) + if effective_profile.api_key and provider != "ollama": + kwargs.setdefault("api_key", effective_profile.api_key) + if effective_profile.base_url: + kwargs.setdefault("base_url", effective_profile.base_url) + if dynamic_profile is not None: + adapter = dynamic_profile.adapter + if dynamic_profile.api_key and adapter != "ollama": + kwargs.setdefault("api_key", dynamic_profile.api_key) + if dynamic_profile.base_url: + kwargs.setdefault("base_url", dynamic_profile.base_url) + elif adapter == "grok": + kwargs.setdefault("base_url", "https://api.x.ai/v1") + + if adapter in {"openai-compatible", "grok", "antigravity"}: + provider = "openai" + _original_provider = dynamic_profile.id + _is_third_party = True + elif adapter == "anthropic-compatible": + provider = "anthropic" + _original_provider = dynamic_profile.id + _is_third_party = True + else: + provider = adapter + _is_third_party = adapter in { + "openrouter", + "openai-compatible", + "anthropic-compatible", + "grok", + "antigravity", + } or (adapter in {"openai", "anthropic"} and bool(dynamic_profile.base_url)) + if _is_third_party: + _original_provider = dynamic_profile.id + + # Some OpenAI-compatible reseller gateways reject the SDK's default + # ``OpenAI/Python`` User-Agent with a generic 403. WebUI profiles with a + # custom Base URL are third-party endpoints, so use the same compatible + # User-Agent as the legacy ``custom-openai`` provider. + if provider == "openai" and dynamic_profile.base_url: + kwargs.setdefault("default_headers", {}).setdefault( + "User-Agent", "codex_cli_rs/0.0.0" + ) + + if provider == "openrouter" and not skip_auto_config: + effort = ( + os.environ.get("EVOSCIENTIST_REASONING_EFFORT", "").strip() or "high" + ) + kwargs.setdefault("reasoning", {"effort": effort, "summary": "auto"}) + _patch_openrouter_strip_responses_reasoning() + + elif provider == "anthropic": base_url = os.environ.get("ANTHROPIC_BASE_URL", "") if base_url: - kwargs["base_url"] = base_url + kwargs.setdefault("base_url", base_url) api_key = os.environ.get("ANTHROPIC_API_KEY", "") if api_key: - kwargs["api_key"] = api_key + kwargs.setdefault("api_key", api_key) # Native OpenAI base_url override (e.g. ccproxy Codex at localhost:8000/codex/v1) elif provider == "openai": base_url = os.environ.get("OPENAI_BASE_URL", "") if base_url: - kwargs["base_url"] = base_url + kwargs.setdefault("base_url", base_url) _is_openai_proxy = _is_ccproxy_codex() if _is_openai_proxy: # Use Responses API for ccproxy: bypasses the format chain @@ -450,14 +707,16 @@ def get_chat_model( kwargs.pop("streaming", None) # remove if set elsewhere api_key = os.environ.get("OPENAI_API_KEY", "") if api_key: - kwargs["api_key"] = api_key + kwargs.setdefault("api_key", api_key) # OpenAI-routed providers → route through OpenAI provider with base_url elif provider in _OPENAI_ROUTED_PROVIDERS: _original_provider = provider base_url_default, api_key_env = _OPENAI_ROUTED_PROVIDERS[provider] if provider == "custom-openai": - base_url = os.environ.get("CUSTOM_OPENAI_BASE_URL", "") + base_url = os.environ.get("CUSTOM_OPENAI_BASE_URL", "") or kwargs.get( + "base_url", "" + ) if not base_url: raise ValueError( "CUSTOM_OPENAI_BASE_URL environment variable is required when using " @@ -468,10 +727,10 @@ def get_chat_model( else: base_url = base_url_default if base_url: - kwargs["base_url"] = base_url + kwargs.setdefault("base_url", base_url) api_key = os.environ.get(api_key_env, "") if api_key: - kwargs["api_key"] = api_key + kwargs.setdefault("api_key", api_key) # SiliconFlow: disable thinking — LangChain drops reasoning_content # from history, causing error 20015 on multi-turn requests. if provider == "siliconflow": @@ -497,7 +756,7 @@ def get_chat_model( _is_third_party = True api_key = os.environ.get("OPENROUTER_API_KEY", "") if api_key: - kwargs["api_key"] = api_key + kwargs.setdefault("api_key", api_key) # Reasoning via `effort` + `summary: "auto"` so a readable reasoning # summary is returned for display. OpenAI-Responses also emits encrypted # reasoning items (`rs_*` id) that can't be replayed on multi-turn @@ -513,7 +772,9 @@ def get_chat_model( _original_provider = provider base_url_default, api_key_env = _ANTHROPIC_ROUTED_PROVIDERS[provider] if provider == "custom-anthropic": - base_url = os.environ.get("CUSTOM_ANTHROPIC_BASE_URL", "") + base_url = os.environ.get("CUSTOM_ANTHROPIC_BASE_URL", "") or kwargs.get( + "base_url", "" + ) if not base_url: raise ValueError( "CUSTOM_ANTHROPIC_BASE_URL environment variable is required when using " @@ -526,10 +787,10 @@ def get_chat_model( else: base_url = base_url_default if base_url: - kwargs["base_url"] = base_url + kwargs.setdefault("base_url", base_url) api_key = os.environ.get(api_key_env, "") if api_key: - kwargs["api_key"] = api_key + kwargs.setdefault("api_key", api_key) # Kimi Coding Plan requires claude-code User-Agent header if provider == "kimi-coding": kwargs.setdefault("default_headers", {})["User-Agent"] = "claude-code/0.1.0" @@ -538,10 +799,13 @@ def get_chat_model( elif provider == "ollama": base_url = os.environ.get("OLLAMA_BASE_URL", "") if base_url: - kwargs["base_url"] = base_url + kwargs.setdefault("base_url", base_url) - _apply_auto_config(provider, model_id, _is_third_party, kwargs, _original_provider) - _apply_openrouter_anthropic_prompt_cache(provider, model_id, kwargs) + if not skip_auto_config: + _apply_auto_config( + provider, model_id, _is_third_party, kwargs, _original_provider + ) + _apply_openrouter_anthropic_prompt_cache(provider, model_id, kwargs) # User-level override for the OpenAI Responses API vs Chat Completions. # When "false", force Chat Completions and drop reasoning (which triggers @@ -556,7 +820,27 @@ def get_chat_model( elif _responses_api_setting == "true": kwargs["use_responses_api"] = True + # Usage tracking is an integrated-WebUI concern and is inert unless the + # launcher supplied the complete sink environment. Preserve the logical + # profile identity before compatible providers are represented by a native + # OpenAI/Anthropic adapter. + from ..usage import UsageModelIdentity, attach_usage_callback + + _usage_profile_id = ( + dynamic_profile.id + if dynamic_profile is not None + else (_original_provider or provider or "unknown") + ) + _usage_identity = UsageModelIdentity( + provider_profile_id=_usage_profile_id, + provider_revision=get_model_runtime_revision(_usage_profile_id), + provider_adapter=provider or "unknown", + model_alias=model, + upstream_model_id=model_id, + ) + chat_model = init_chat_model(model=model_id, model_provider=provider, **kwargs) + chat_model = attach_usage_callback(chat_model, _usage_identity) # Flatten list content to strings for strict OpenAI-compatible providers # (DeepSeek, SiliconFlow, OpenRouter, custom-openai, etc.) and @@ -584,6 +868,24 @@ def get_chat_model( return chat_model +def get_profile_chat_model( + profile: ProviderProfile, + model: ProviderModel, + *, + skip_auto_config: bool = False, + **kwargs: Any, +) -> Any: + """Build a model from an unsaved or persisted provider profile.""" + return get_chat_model( + model=model.id, + provider=profile.id, + profile_override=profile, + model_override=model, + skip_auto_config=skip_auto_config, + **kwargs, + ) + + def list_models() -> list[str]: """List all available model short names. @@ -619,10 +921,37 @@ async def list_model_picker_entries( ollama_base_url: str | None, *, include_custom_ollama: bool, + model_catalog: Any = None, ) -> list[tuple[str, str, str]]: - """Return model picker entries, optionally including local Ollama models.""" - entries = list_models_by_provider() - if ollama_base_url: + """Return models from the WebUI registry, or legacy entries before migration.""" + try: + registry_configured = await asyncio.to_thread( + get_provider_profiles_path().exists + ) + configured_builtin_entries = ( + await asyncio.to_thread(list_configured_builtin_model_entries) + if registry_configured + else list_builtin_model_catalog_entries(model_catalog) + ) + except (ProviderProfileError, ValueError): + configured_builtin_entries = [] + legacy_mode = configured_builtin_entries is None + entries = ( + list_models_by_provider() + if configured_builtin_entries is None + else configured_builtin_entries + ) + seen = {(name, provider) for name, _, provider in entries} + try: + configured_entries = await asyncio.to_thread(list_configured_model_entries) + except ProviderProfileError: + configured_entries = [] + for name, model_id, provider in configured_entries: + key = (name, provider) + if key not in seen: + entries.append((name, model_id, provider)) + seen.add(key) + if legacy_mode and ollama_base_url: from .ollama_discovery import discover_ollama_models for detected_name in await discover_ollama_models( diff --git a/EvoScientist/llm/patches.py b/EvoScientist/llm/patches.py index db786d3..a0839fc 100644 --- a/EvoScientist/llm/patches.py +++ b/EvoScientist/llm/patches.py @@ -926,24 +926,128 @@ def _read_cfg_configurable() -> dict[str, str]: def _merge_runs_config_kwargs(kwargs: dict) -> dict: - """Merge the live model override into ``kwargs`` for ``runs.create``. + """Merge model and usage correlation context into ``runs.create``. Preserves any caller-supplied ``config.configurable`` keys. EvoScientist's keys take precedence on conflict (callers shouldn't be passing model overrides — the CLI is the source of truth). """ overrides = _read_cfg_configurable() - if not overrides: - return kwargs + usage_enabled = os.getenv("EVOSCIENTIST_USAGE_TRACKING", "").strip().lower() in { + "1", + "true", + "yes", + "on", + } + current_metadata: dict = {} + current_configurable: dict = {} + try: + from langgraph.config import get_config + + # Scoped deployments need this context even when usage accounting is + # disabled: DeepAgents creates derived threads through this proxy. + current = get_config() + raw_metadata = current.get("metadata") + raw_configurable = current.get("configurable") + if isinstance(raw_metadata, dict): + current_metadata = raw_metadata + if isinstance(raw_configurable, dict): + current_configurable = raw_configurable + except (LookupError, RuntimeError): + pass + + inherited_scope: dict[str, object] = {} + scope_keys = ( + "workspace_scope_id", + "workspace_scope_owner_id", + "workspace_scope_revision", + "workspace_deployment_id", + ) + if all(current_configurable.get(key) is not None for key in scope_keys): + inherited_scope = {key: current_configurable[key] for key in scope_keys} + target_thread_id = kwargs.get("thread_id") + source_thread_id = current_configurable.get("thread_id") + if ( + isinstance(target_thread_id, str) + and target_thread_id + and target_thread_id != source_thread_id + ): + try: + from EvoScientist.scope_registry import ( + ScopeConflictError, + get_scope_registry, + ) + + registry = get_scope_registry() + try: + owner = registry.register_owner( + str(inherited_scope["workspace_deployment_id"]), + str(inherited_scope["workspace_scope_id"]), + owner_type="derived_run", + resource_id=target_thread_id, + parent_owner_id=str(inherited_scope["workspace_scope_owner_id"]), + state="active", + ) + except ScopeConflictError: + owner = registry.get_owner_by_resource( + str(inherited_scope["workspace_deployment_id"]), + target_thread_id, + ) + if owner.scope_id != inherited_scope["workspace_scope_id"]: + raise + inherited_scope["workspace_scope_owner_id"] = owner.owner_id + inherited_scope["thread_id"] = target_thread_id + except Exception: + # The required-mode backend factory rejects missing/invalid + # ownership at execution time. Optional mode keeps legacy + # integrations working when they cannot register a child. + if os.getenv("EVOSCIENTIST_WORKSPACE_ISOLATION", "optional").lower() == "required": + raise + existing = kwargs.get("config") if not isinstance(existing, dict): existing = {} existing_configurable = existing.get("configurable") if not isinstance(existing_configurable, dict): existing_configurable = {} - merged_configurable = {**existing_configurable, **overrides} + merged_configurable = {**existing_configurable, **overrides, **inherited_scope} kwargs = dict(kwargs) - kwargs["config"] = {**existing, "configurable": merged_configurable} + if overrides or "config" in kwargs: + kwargs["config"] = {**existing, "configurable": merged_configurable} + + if not usage_enabled: + return kwargs + + inherited_metadata = { + key: current_metadata[key] + for key in ( + "usage_context_version", + "turn_id", + "source_session_id", + "source_agent", + "workspace_dir", + ) + if current_metadata.get(key) is not None + } + inherited_metadata.setdefault( + "source_session_id", + current_metadata.get("thread_id") + or current_metadata.get("langgraph_thread_id") + or current_configurable.get("thread_id"), + ) + if inherited_metadata.get("source_session_id") is None: + inherited_metadata.pop("source_session_id", None) + existing_metadata = kwargs.get("metadata") + if not isinstance(existing_metadata, dict): + existing_metadata = {} + merged_metadata = {**inherited_metadata, **existing_metadata} + run_kind = merged_metadata.get("run_kind") + if "usage_scope" not in merged_metadata: + if isinstance(run_kind, str) and run_kind.startswith("evomemory_"): + merged_metadata["usage_scope"] = "memory" + else: + merged_metadata["usage_scope"] = "async_subagent" + kwargs["metadata"] = merged_metadata return kwargs diff --git a/EvoScientist/llm/provider_operations.py b/EvoScientist/llm/provider_operations.py new file mode 100644 index 0000000..02b9aef --- /dev/null +++ b/EvoScientist/llm/provider_operations.py @@ -0,0 +1,519 @@ +"""Provider model discovery and lightweight connectivity tests.""" + +from __future__ import annotations + +import asyncio +import os +import time +from dataclasses import dataclass +from typing import Any + +import httpx + +from ..config.provider_profiles import ( + ProviderModel, + ProviderProfile, + get_builtin_provider_profile, + normalize_provider_base_url, +) +from ..config.settings import is_config_applied_env + +_DEFAULT_BASE_URLS = { + "openai": "https://api.openai.com/v1", + "anthropic": "https://api.anthropic.com", + "google-genai": "https://generativelanguage.googleapis.com/v1beta", + "grok": "https://api.x.ai/v1", + "openrouter": "https://openrouter.ai/api/v1", + "nvidia": "https://integrate.api.nvidia.com/v1", + "ollama": "http://127.0.0.1:11434", +} +_API_KEY_REQUIRED_ADAPTERS = { + "openai", + "anthropic", + "google-genai", + "grok", + "openrouter", + "nvidia", +} +_MAX_DISCOVERED_MODELS = 2_000 +_MAX_PAGES = 100 +_MAX_REQUEST_ATTEMPTS = 2 + + +@dataclass(frozen=True) +class BuiltinProviderConnection: + adapter: str + api_key_field: str | None + base_url_field: str | None = None + default_base_url: str = "" + + +_BUILTIN_PROVIDER_CONNECTIONS = { + "anthropic": BuiltinProviderConnection( + "anthropic", "anthropic_api_key", "anthropic_base_url" + ), + "openai": BuiltinProviderConnection("openai", "openai_api_key"), + "google-genai": BuiltinProviderConnection("google-genai", "google_api_key"), + "minimax": BuiltinProviderConnection( + "anthropic-compatible", + "minimax_api_key", + "minimax_base_url", + "https://api.minimaxi.com/anthropic", + ), + "nvidia": BuiltinProviderConnection("nvidia", "nvidia_api_key"), + "siliconflow": BuiltinProviderConnection( + "openai-compatible", + "siliconflow_api_key", + default_base_url="https://api.siliconflow.cn/v1", + ), + "openrouter": BuiltinProviderConnection("openrouter", "openrouter_api_key"), + "zhipu": BuiltinProviderConnection( + "openai-compatible", + "zhipu_api_key", + default_base_url="https://open.bigmodel.cn/api/paas/v4", + ), + "zhipu-code": BuiltinProviderConnection( + "openai-compatible", + "zhipu_api_key", + default_base_url="https://open.bigmodel.cn/api/coding/paas/v4", + ), + "volcengine": BuiltinProviderConnection( + "openai-compatible", + "volcengine_api_key", + default_base_url="https://ark.cn-beijing.volces.com/api/v3", + ), + "dashscope": BuiltinProviderConnection( + "openai-compatible", + "dashscope_api_key", + default_base_url="https://dashscope.aliyuncs.com/compatible-mode/v1", + ), + "dashscope-code": BuiltinProviderConnection( + "openai-compatible", + "dashscope_api_key", + default_base_url="https://coding.dashscope.aliyuncs.com/v1", + ), + "deepseek": BuiltinProviderConnection( + "openai-compatible", + "deepseek_api_key", + default_base_url="https://api.deepseek.com", + ), + "moonshot": BuiltinProviderConnection( + "openai-compatible", + "moonshot_api_key", + default_base_url="https://api.moonshot.cn/v1", + ), + "kimi-coding": BuiltinProviderConnection( + "anthropic-compatible", + "kimi_api_key", + default_base_url="https://api.kimi.com/coding/", + ), + "custom-openai": BuiltinProviderConnection( + "openai-compatible", + "custom_openai_api_key", + "custom_openai_base_url", + ), + "custom-anthropic": BuiltinProviderConnection( + "anthropic-compatible", + "custom_anthropic_api_key", + "custom_anthropic_base_url", + ), + "ollama": BuiltinProviderConnection("ollama", None, "ollama_base_url"), +} +_BUILTIN_API_KEY_ENVS = { + "anthropic": "ANTHROPIC_API_KEY", + "openai": "OPENAI_API_KEY", + "google-genai": "GOOGLE_API_KEY", + "minimax": "MINIMAX_API_KEY", + "nvidia": "NVIDIA_API_KEY", + "siliconflow": "SILICONFLOW_API_KEY", + "openrouter": "OPENROUTER_API_KEY", + "zhipu": "ZHIPU_API_KEY", + "zhipu-code": "ZHIPU_API_KEY", + "volcengine": "VOLCENGINE_API_KEY", + "dashscope": "DASHSCOPE_API_KEY", + "dashscope-code": "DASHSCOPE_API_KEY", + "deepseek": "DEEPSEEK_API_KEY", + "moonshot": "MOONSHOT_API_KEY", + "kimi-coding": "KIMI_API_KEY", + "custom-openai": "CUSTOM_OPENAI_API_KEY", + "custom-anthropic": "CUSTOM_ANTHROPIC_API_KEY", +} +_BUILTIN_BASE_URL_ENVS = { + "anthropic": "ANTHROPIC_BASE_URL", + "openai": "OPENAI_BASE_URL", + "minimax": "MINIMAX_BASE_URL", + "custom-openai": "CUSTOM_OPENAI_BASE_URL", + "custom-anthropic": "CUSTOM_ANTHROPIC_BASE_URL", + "ollama": "OLLAMA_BASE_URL", +} + + +class ProviderOperationError(RuntimeError): + """Raised when provider discovery or testing cannot complete.""" + + +def build_builtin_provider_profile( + config: Any, + provider_id: str, + *, + saved_profile: ProviderProfile | None = None, + load_saved: bool = True, + use_environment: bool = True, + base_url_override: str | None = None, + api_key_override: str | None = None, + clear_api_key: bool = False, +) -> ProviderProfile: + """Resolve a built-in profile with env > registry > legacy config priority.""" + spec = _BUILTIN_PROVIDER_CONNECTIONS.get(provider_id) + if spec is None: + raise ProviderOperationError(f"Unknown built-in provider {provider_id!r}.") + + saved = get_builtin_provider_profile(provider_id) if load_saved else saved_profile + base_url = saved.base_url if saved is not None else spec.default_base_url + if saved is None and spec.base_url_field: + configured_base_url = getattr(config, spec.base_url_field, "") + if isinstance(configured_base_url, str) and configured_base_url.strip(): + base_url = configured_base_url.strip() + base_url_env = _BUILTIN_BASE_URL_ENVS.get(provider_id) + if ( + base_url_env + and use_environment + and os.environ.get(base_url_env, "").strip() + and (saved is None or not is_config_applied_env(base_url_env)) + ): + base_url = os.environ[base_url_env].strip() + if base_url_override is not None: + base_url = base_url_override.strip() or spec.default_base_url + + api_key = saved.api_key if saved is not None else "" + if saved is None and spec.api_key_field: + configured_api_key = getattr(config, spec.api_key_field, "") + if isinstance(configured_api_key, str): + api_key = configured_api_key.strip() + api_key_env = _BUILTIN_API_KEY_ENVS.get(provider_id) + if ( + api_key_env + and use_environment + and os.environ.get(api_key_env, "").strip() + and (saved is None or not is_config_applied_env(api_key_env)) + ): + api_key = os.environ[api_key_env].strip() + if clear_api_key: + api_key = "" + elif api_key_override is not None and api_key_override.strip(): + api_key = api_key_override.strip() + + return ProviderProfile( + id=provider_id, + name=provider_id, + adapter=spec.adapter, + base_url=base_url, + api_key=api_key, + enabled=saved.enabled if saved is not None else True, + models=saved.models if saved is not None else (), + auth_mode=( + saved.auth_mode + if saved is not None + else getattr(config, f"{provider_id}_auth_mode", "api_key") + if provider_id in {"anthropic", "openai"} + else "api_key" + ), + ) + + +def builtin_provider_ids() -> tuple[str, ...]: + return tuple(_BUILTIN_PROVIDER_CONNECTIONS) + + +def builtin_provider_adapter(provider_id: str) -> str: + spec = _BUILTIN_PROVIDER_CONNECTIONS.get(provider_id) + if spec is None: + raise ProviderOperationError(f"Unknown built-in provider {provider_id!r}.") + return spec.adapter + + +def builtin_provider_legacy_api_key(config: Any, provider_id: str) -> str: + """Return only the config.yaml key, excluding environment overrides.""" + spec = _BUILTIN_PROVIDER_CONNECTIONS.get(provider_id) + if spec is None or spec.api_key_field is None: + return "" + value = getattr(config, spec.api_key_field, "") + return value.strip() if isinstance(value, str) else "" + + +@dataclass(frozen=True) +class DiscoveredProviderModel: + model_id: str + name: str + + +@dataclass(frozen=True) +class ProviderModelTestResult: + latency_ms: int + response: str + + +def _base_url(profile: ProviderProfile) -> str: + base_url = profile.base_url or _DEFAULT_BASE_URLS.get(profile.adapter, "") + if not base_url: + raise ProviderOperationError( + f"Base URL is required for adapter {profile.adapter}." + ) + return normalize_provider_base_url(profile.adapter, base_url) + + +def _models_url(profile: ProviderProfile) -> str: + base_url = _base_url(profile) + if profile.adapter in {"anthropic", "anthropic-compatible"}: + return ( + f"{base_url}/models" + if base_url.endswith(("/v1", "/v1beta")) + else f"{base_url}/v1/models" + ) + if profile.adapter == "ollama": + return f"{base_url}/api/tags" + return f"{base_url}/models" + + +def _headers(profile: ProviderProfile) -> dict[str, str]: + if profile.adapter in _API_KEY_REQUIRED_ADAPTERS and not profile.api_key: + raise ProviderOperationError( + f"API key is required for adapter {profile.adapter}." + ) + if profile.adapter in {"anthropic", "anthropic-compatible"}: + headers = {"anthropic-version": "2023-06-01"} + if profile.api_key: + headers["x-api-key"] = profile.api_key + return headers + if profile.adapter == "google-genai": + return {"x-goog-api-key": profile.api_key} + if profile.adapter == "ollama" or not profile.api_key: + return {} + return {"Authorization": f"Bearer {profile.api_key}"} + + +def _provider_error_message(response: httpx.Response) -> str: + message = "" + try: + payload = response.json() + if isinstance(payload, dict): + error = payload.get("error") + if isinstance(error, dict): + raw_message = error.get("message") + if isinstance(raw_message, str): + message = raw_message.strip() + elif isinstance(error, str): + message = error.strip() + if not message: + raw_message = payload.get("message") + if isinstance(raw_message, str): + message = raw_message.strip() + except (ValueError, TypeError): + pass + suffix = f": {message[:500]}" if message else "" + return f"Provider returned HTTP {response.status_code}{suffix}" + + +def _model_name(raw: dict[str, Any], model_id: str) -> str: + for key in ("display_name", "displayName", "name"): + value = raw.get(key) + if isinstance(value, str) and value.strip(): + return value.strip()[:120] + return model_id[:120] + + +def _parse_models( + profile: ProviderProfile, payload: Any +) -> list[DiscoveredProviderModel]: + if not isinstance(payload, dict): + raise ProviderOperationError("Provider returned an invalid model list.") + + records = ( + payload.get("models") if profile.adapter == "ollama" else payload.get("data") + ) + if profile.adapter == "google-genai": + records = payload.get("models") + elif not isinstance(records, list): + records = payload.get("models") + if not isinstance(records, list): + raise ProviderOperationError("Provider response does not contain a model list.") + + result: list[DiscoveredProviderModel] = [] + for raw in records: + if not isinstance(raw, dict): + continue + model_id_raw = ( + (raw.get("model") or raw.get("name")) + if profile.adapter == "ollama" + else raw.get("id") + ) + if profile.adapter == "google-genai": + model_id_raw = raw.get("name") + methods = raw.get("supportedGenerationMethods") + if isinstance(methods, list) and "generateContent" not in methods: + continue + if not isinstance(model_id_raw, str) or not model_id_raw.strip(): + continue + model_id = model_id_raw.strip() + if profile.adapter == "google-genai": + model_id = model_id.removeprefix("models/") + result.append( + DiscoveredProviderModel( + model_id=model_id[:300], + name=_model_name(raw, model_id), + ) + ) + return result + + +async def discover_provider_models( + profile: ProviderProfile, + *, + transport: httpx.AsyncBaseTransport | None = None, +) -> list[DiscoveredProviderModel]: + """Fetch every model exposed by a provider's model-list endpoint.""" + url = _models_url(profile) + headers = _headers(profile) + params: dict[str, str | int] = {} + discovered: list[DiscoveredProviderModel] = [] + seen: set[str] = set() + + timeout = httpx.Timeout(30.0, connect=10.0) + async with httpx.AsyncClient(timeout=timeout, transport=transport) as client: + for _ in range(_MAX_PAGES): + response: httpx.Response | None = None + last_request_error: httpx.HTTPError | None = None + for attempt in range(_MAX_REQUEST_ATTEMPTS): + try: + response = await client.get(url, headers=headers, params=params) + break + except (httpx.TimeoutException, httpx.NetworkError) as exc: + last_request_error = exc + if attempt + 1 < _MAX_REQUEST_ATTEMPTS: + await asyncio.sleep(0.25 * (attempt + 1)) + if response is None: + if isinstance(last_request_error, httpx.TimeoutException): + raise ProviderOperationError( + f"Provider model discovery timed out after " + f"{_MAX_REQUEST_ATTEMPTS} attempts." + ) from last_request_error + error_name = ( + last_request_error.__class__.__name__ + if last_request_error is not None + else "NetworkError" + ) + raise ProviderOperationError( + f"Could not connect to provider after " + f"{_MAX_REQUEST_ATTEMPTS} attempts: {error_name}." + ) from last_request_error + if response.status_code >= 400: + raise ProviderOperationError(_provider_error_message(response)) + try: + payload = response.json() + except ValueError as exc: + content_type = ( + response.headers.get("content-type", "unknown") + .split(";", 1)[0] + .strip() + or "unknown" + ) + hint = ( + " OpenAI-compatible Base URLs normally end in /v1." + if profile.adapter + in { + "openai", + "openai-compatible", + "grok", + "openrouter", + "nvidia", + "antigravity", + } + else "" + ) + raise ProviderOperationError( + f"Provider returned a non-JSON model list from {url} " + f"(Content-Type: {content_type}).{hint}" + ) from exc + + for model in _parse_models(profile, payload): + if model.model_id in seen: + continue + seen.add(model.model_id) + discovered.append(model) + if len(discovered) >= _MAX_DISCOVERED_MODELS: + return sorted(discovered, key=lambda item: item.model_id.lower()) + + if profile.adapter in {"anthropic", "anthropic-compatible"}: + if not isinstance(payload, dict) or payload.get("has_more") is not True: + break + last_id = payload.get("last_id") + if not isinstance(last_id, str) or not last_id: + break + params = {"after_id": last_id, "limit": 100} + continue + if profile.adapter == "google-genai": + next_page = ( + payload.get("nextPageToken") if isinstance(payload, dict) else None + ) + if not isinstance(next_page, str) or not next_page: + break + params = {"pageToken": next_page, "pageSize": 1000} + continue + break + + return sorted(discovered, key=lambda item: item.model_id.lower()) + + +def _response_text(response: Any) -> str: + content = getattr(response, "content", response) + if isinstance(content, str): + return content.strip() + if isinstance(content, list): + parts: list[str] = [] + for block in content: + if isinstance(block, str): + parts.append(block) + elif isinstance(block, dict): + text = block.get("text") or block.get("content") + if isinstance(text, str): + parts.append(text) + return "\n".join(parts).strip() + return str(content).strip() + + +def _safe_test_error(exc: Exception, profile: ProviderProfile) -> str: + message = str(exc).strip() or exc.__class__.__name__ + if profile.api_key: + message = message.replace(profile.api_key, "[redacted]") + return message[:800] + + +async def test_provider_model( + profile: ProviderProfile, + model: ProviderModel, +) -> ProviderModelTestResult: + """Build the configured model and run a minimal real inference request.""" + from .models import get_profile_chat_model + + started = time.perf_counter() + try: + chat_model = await asyncio.to_thread( + get_profile_chat_model, + profile, + model, + max_tokens=64, + skip_auto_config=True, + ) + response = await asyncio.wait_for( + chat_model.ainvoke("Reply with exactly: OK"), timeout=45.0 + ) + except TimeoutError as exc: + raise ProviderOperationError("Model test timed out after 45 seconds.") from exc + except Exception as exc: + raise ProviderOperationError(_safe_test_error(exc, profile)) from exc + + latency_ms = max(1, round((time.perf_counter() - started) * 1000)) + return ProviderModelTestResult( + latency_ms=latency_ms, + response=_response_text(response)[:500], + ) diff --git a/EvoScientist/memory/launch.py b/EvoScientist/memory/launch.py index d15ddda..f94759e 100644 --- a/EvoScientist/memory/launch.py +++ b/EvoScientist/memory/launch.py @@ -90,7 +90,7 @@ def _worker_workspace_dir(workspace_dir: str | Path) -> str: def _memory_worker_metadata(context: MemorySourceContext) -> dict[str, str]: - return { + metadata = { "run_kind": f"evomemory_{context.source_type.value}_worker", "source_session_id": context.session_id, "source_agent": context.source_agent, @@ -98,6 +98,13 @@ def _memory_worker_metadata(context: MemorySourceContext) -> dict[str, str]: "trajectory_digest": context.trajectory_digest, "workspace_dir": _worker_workspace_dir(context.workspace_dir), } + from ..usage.callback import usage_tracking_requested + + if usage_tracking_requested(): + metadata["usage_scope"] = "memory" + if usage_tracking_requested() and context.turn_id: + metadata["turn_id"] = context.turn_id + return metadata def _memory_worker_run_payload( @@ -107,19 +114,22 @@ def _memory_worker_run_payload( ) -> BackgroundRunPayload: """Build the LangGraph SDK run payload for a memory worker.""" metadata = _memory_worker_metadata(context) + configurable = { + "thread_id": thread_id, + "evomemory_source_session_id": context.session_id, + "evomemory_source_agent": context.source_agent, + "evomemory_project_id": context.project_id, + "evomemory_trajectory_digest": context.trajectory_digest, + } + from ..usage.callback import usage_tracking_requested + + if usage_tracking_requested() and context.turn_id: + configurable["evomemory_source_turn_id"] = context.turn_id payload: BackgroundRunPayload = { "assistant_id": _memory_worker_graph_id(context.source_type), "input": messages_input(_memory_worker_user_prompt(context)), "metadata": metadata, - "config": { - "configurable": { - "thread_id": thread_id, - "evomemory_source_session_id": context.session_id, - "evomemory_source_agent": context.source_agent, - "evomemory_project_id": context.project_id, - "evomemory_trajectory_digest": context.trajectory_digest, - } - }, + "config": {"configurable": configurable}, } return _runs_create_kwargs(payload) diff --git a/EvoScientist/memory/source_context.py b/EvoScientist/memory/source_context.py index 4d87050..1ac79a3 100644 --- a/EvoScientist/memory/source_context.py +++ b/EvoScientist/memory/source_context.py @@ -40,6 +40,7 @@ class MemorySourceContext: session_id: str trajectory: list[CompactMessage] trajectory_digest: str + turn_id: str | None = None def _task_tool_call_ids(messages: list[BaseMessage]) -> set[str]: @@ -203,6 +204,20 @@ def _runtime_thread_id(runtime: Runtime | None) -> str | None: return None +def _active_turn_id() -> str | None: + """Read the WebUI turn correlation value from the active run config.""" + try: + from langgraph.config import get_config + + metadata = get_config().get("metadata", {}) + except (LookupError, RuntimeError): + return None + if not isinstance(metadata, dict): + return None + value = metadata.get("turn_id") + return value if isinstance(value, str) and value else None + + def _short_hash(text: str) -> str: """Return the short hash fragment used in generated ids.""" return hashlib.sha256(text.encode("utf-8")).hexdigest()[:16] @@ -240,6 +255,7 @@ def build_memory_source_context( project_id=project_id, source_agent=source_agent, session_id=session_id, + turn_id=_active_turn_id(), trajectory=trajectory, trajectory_digest=_trajectory_digest(trajectory), ) diff --git a/EvoScientist/middleware/ask_user.py b/EvoScientist/middleware/ask_user.py index 31d9616..723fb44 100644 --- a/EvoScientist/middleware/ask_user.py +++ b/EvoScientist/middleware/ask_user.py @@ -29,13 +29,46 @@ from langchain.agents.middleware.types import ( ) from langchain.tools import InjectedToolCallId from langchain_core.messages import AIMessage, SystemMessage, ToolMessage -from langchain_core.tools import tool +from langchain_core.tools import BaseTool, tool from langgraph.types import Command, interrupt from pydantic import BeforeValidator, Field from typing_extensions import TypedDict logger = logging.getLogger(__name__) +_VALID_REVIEW_MODES = frozenset({"manual", "auto", "full"}) + + +def _review_mode() -> str: + """Return the per-run WebUI review mode, failing closed to Manual.""" + try: + from langgraph.config import get_config + + config = get_config() + except Exception: + return "manual" + if not isinstance(config, dict): + return "manual" + configurable = config.get("configurable") or {} + if not isinstance(configurable, dict): + return "manual" + mode = configurable.get("review_mode") + return mode if mode in _VALID_REVIEW_MODES else "manual" + + +def _tool_name(tool_value: BaseTool | dict[str, Any]) -> str | None: + if isinstance(tool_value, BaseTool): + return tool_value.name or None + name = tool_value.get("name") + if isinstance(name, str) and name: + return name + function = tool_value.get("function") + if isinstance(function, dict): + nested_name = function.get("name") + if isinstance(nested_name, str) and nested_name: + return nested_name + return None + # --------------------------------------------------------------------------- # Data types @@ -202,6 +235,16 @@ or available tools. - Never ask more than once per decision point — respect the user's time - After receiving answers, summarize what you understood before proceeding""" +FULL_APPROVE_SYSTEM_PROMPT = """\ +You are running in Full approve mode. Do not wait for user clarification. When +information is missing, make reasonable, conservative assumptions, continue the +task, and report material assumptions in the final response.""" + +FULL_APPROVE_TOOL_MESSAGE = ( + "Full approve mode: continue with reasonable assumptions and do not ask " + "the user again." +) + # --------------------------------------------------------------------------- # Validation & parsing @@ -344,6 +387,17 @@ def _parse_answers( # --------------------------------------------------------------------------- +def _with_system_prompt(request: ModelRequest[ContextT], prompt: str) -> SystemMessage: + if request.system_message is not None: + content = [ + *request.system_message.content_blocks, + {"type": "text", "text": f"\n\n{prompt}"}, + ] + else: + content = [{"type": "text", "text": prompt}] + return SystemMessage(content=cast("list[str | dict[str, str]]", content)) + + class AskUserMiddleware(AgentMiddleware[Any, ContextT, ResponseT]): """Middleware that provides an ``ask_user`` tool for interactive questioning. @@ -369,6 +423,17 @@ class AskUserMiddleware(AgentMiddleware[Any, ContextT, ResponseT]): tool_call_id: Annotated[str, InjectedToolCallId], ) -> Command[Any]: """Ask the user one or more questions.""" + if _review_mode() == "full": + return Command( + update={ + "messages": [ + ToolMessage( + FULL_APPROVE_TOOL_MESSAGE, + tool_call_id=tool_call_id, + ) + ] + } + ) _validate_questions(questions) ask_request = AskUserRequest( type="ask_user", @@ -386,18 +451,22 @@ class AskUserMiddleware(AgentMiddleware[Any, ContextT, ResponseT]): request: ModelRequest[ContextT], handler: Callable[[ModelRequest[ContextT]], ModelResponse[ResponseT]], ) -> ModelResponse[ResponseT] | AIMessage: - """Inject the ask_user system prompt.""" - if request.system_message is not None: - new_system_content = [ - *request.system_message.content_blocks, - {"type": "text", "text": f"\n\n{self.system_prompt}"}, - ] - else: - new_system_content = [{"type": "text", "text": self.system_prompt}] - new_system_message = SystemMessage( - content=cast("list[str | dict[str, str]]", new_system_content) + """Apply the interactive or unattended prompt and tool policy.""" + if _review_mode() == "full": + tools = [tool for tool in request.tools if _tool_name(tool) != "ask_user"] + return handler( + request.override( + system_message=_with_system_prompt( + request, FULL_APPROVE_SYSTEM_PROMPT + ), + tools=tools, + ) + ) + return handler( + request.override( + system_message=_with_system_prompt(request, self.system_prompt) + ) ) - return handler(request.override(system_message=new_system_message)) async def awrap_model_call( self, @@ -406,15 +475,19 @@ class AskUserMiddleware(AgentMiddleware[Any, ContextT, ResponseT]): [ModelRequest[ContextT]], Awaitable[ModelResponse[ResponseT]] ], ) -> ModelResponse[ResponseT] | AIMessage: - """Inject the ask_user system prompt (async).""" - if request.system_message is not None: - new_system_content = [ - *request.system_message.content_blocks, - {"type": "text", "text": f"\n\n{self.system_prompt}"}, - ] - else: - new_system_content = [{"type": "text", "text": self.system_prompt}] - new_system_message = SystemMessage( - content=cast("list[str | dict[str, str]]", new_system_content) + """Apply the interactive or unattended prompt and tool policy (async).""" + if _review_mode() == "full": + tools = [tool for tool in request.tools if _tool_name(tool) != "ask_user"] + return await handler( + request.override( + system_message=_with_system_prompt( + request, FULL_APPROVE_SYSTEM_PROMPT + ), + tools=tools, + ) + ) + return await handler( + request.override( + system_message=_with_system_prompt(request, self.system_prompt) + ) ) - return await handler(request.override(system_message=new_system_message)) diff --git a/EvoScientist/middleware/configurable_model.py b/EvoScientist/middleware/configurable_model.py index 4c65f4e..656f1bb 100644 --- a/EvoScientist/middleware/configurable_model.py +++ b/EvoScientist/middleware/configurable_model.py @@ -101,7 +101,7 @@ class ConfigurableModelMiddleware(AgentMiddleware): def __init__(self) -> None: super().__init__() - self._cache: dict[tuple[str, str | None], Any] = {} + self._cache: dict[tuple[str, str | None, str | None], Any] = {} self._lock = threading.Lock() # Track the last (model, provider) pair we INFO-logged so we only # surface a banner on transition. Without this, every LLM call in a @@ -130,7 +130,10 @@ class ConfigurableModelMiddleware(AgentMiddleware): def _resolve(self, model: str, provider: str | None) -> Any: """Return a cached or freshly-built chat model for ``(model, provider)``.""" - key = (model, provider) + from ..llm.models import get_model_runtime_revision + + revision = get_model_runtime_revision(provider) + key = (model, provider, revision) with self._lock: cached = self._cache.get(key) if cached is not None: diff --git a/EvoScientist/middleware/scheduler.py b/EvoScientist/middleware/scheduler.py index 4d35b1f..5aa3c59 100644 --- a/EvoScientist/middleware/scheduler.py +++ b/EvoScientist/middleware/scheduler.py @@ -13,12 +13,14 @@ from __future__ import annotations import asyncio import time from collections.abc import Awaitable, Callable +from types import SimpleNamespace from langchain.agents.middleware.types import ( AgentMiddleware, ModelRequest, ModelResponse, ) +from langchain.tools import ToolRuntime from langchain_core.tools import tool from .utils import append_to_system_message @@ -47,8 +49,20 @@ manage them. # --------------------------------------------------------------------------- +def _scope_for_runtime(runtime: ToolRuntime | None): + from ..workspace_scope import require_scoped_runtime + + return require_scoped_runtime(runtime, kind="scheduler") + + @tool -def schedule_task(name: str, cron: str, prompt: str, timezone: str = "") -> str: +def schedule_task( + name: str, + cron: str, + prompt: str, + timezone: str = "", + runtime: ToolRuntime = None, +) -> str: """Create a recurring scheduled task that runs unattended in the background. Translate the user's natural-language timing into a standard 5-field cron @@ -67,7 +81,11 @@ def schedule_task(name: str, cron: str, prompt: str, timezone: str = "") -> str: return "Scheduler unavailable: the langgraph dev backend is not running." try: rec = crons.create_schedule( - name=name, schedule=cron, prompt=prompt, timezone=timezone or None + name=name, + schedule=cron, + prompt=prompt, + timezone=timezone or None, + scope=_scope_for_runtime(runtime), ) except Exception as e: return f"Error: {e}" @@ -78,14 +96,14 @@ def schedule_task(name: str, cron: str, prompt: str, timezone: str = "") -> str: @tool -def list_scheduled_tasks() -> str: +def list_scheduled_tasks(runtime: ToolRuntime = None) -> str: """List the user's recurring scheduled tasks (id, name, schedule, enabled).""" from ..cron import schedule as crons if not crons.is_available(): return "Scheduler unavailable: the langgraph dev backend is not running." try: - rows = crons.list_schedules() + rows = crons.list_schedules(_scope_for_runtime(runtime)) except Exception as e: return f"Error: {e}" if not rows: @@ -101,7 +119,7 @@ def list_scheduled_tasks() -> str: @tool -def cancel_scheduled_task(cron_id: str) -> str: +def cancel_scheduled_task(cron_id: str, runtime: ToolRuntime = None) -> str: """Cancel (delete) a scheduled task. Pass the id (or its prefix) shown by list_scheduled_tasks.""" from ..cron import schedule as crons @@ -111,7 +129,7 @@ def cancel_scheduled_task(cron_id: str) -> str: # Empty prefix would match (and delete) the only cron — refuse it. return "Provide the id (or a prefix) of the task to cancel." try: - rows = crons.list_schedules() + rows = crons.list_schedules(_scope_for_runtime(runtime)) # B2: collect ALL prefix matches before acting to detect ambiguity. matches = [ r for r in rows if str(r.get("cron_id", "")).startswith(requested_id) @@ -142,11 +160,15 @@ class SchedulerMiddleware(AgentMiddleware): def __init__(self) -> None: super().__init__() - self._cache: str | None = None - self._cache_at: float = 0.0 + self._cache: dict[str, tuple[float, str]] = {} self.tools = [schedule_task, list_scheduled_tasks, cancel_scheduled_task] - def _schedules_block(self) -> str: + def _runtime_scope(self): + from langgraph.config import get_config + + return _scope_for_runtime(SimpleNamespace(config=get_config())) + + def _schedules_block(self, scope=None) -> str: """Build the dynamic ```` block (empty if none / down).""" from ..cron import schedule as crons @@ -158,7 +180,7 @@ class SchedulerMiddleware(AgentMiddleware): try: if not crons.is_available(): return "" - rows = crons.list_schedules() + rows = crons.list_schedules(scope) except Exception: return "" if not rows: @@ -182,10 +204,15 @@ class SchedulerMiddleware(AgentMiddleware): def _cached_schedules_block(self) -> str: now = time.monotonic() - if self._cache is None or (now - self._cache_at) > _CACHE_TTL_SECONDS: - self._cache = self._schedules_block() - self._cache_at = now - return self._cache + try: + scope = self._runtime_scope() + except Exception: + return "" + key = f"{scope.scope_id}:{scope.revision}" if scope is not None else "legacy" + cached = self._cache.get(key) + if cached is None or (now - cached[0]) > _CACHE_TTL_SECONDS: + self._cache[key] = (now, self._schedules_block(scope)) + return self._cache[key][1] def _injection(self, schedules_block: str) -> str: """Static instructions, then the dynamic list (static→dynamic, like memory).""" diff --git a/EvoScientist/middleware/tool_selector.py b/EvoScientist/middleware/tool_selector.py index 3e28432..427ad29 100644 --- a/EvoScientist/middleware/tool_selector.py +++ b/EvoScientist/middleware/tool_selector.py @@ -258,6 +258,29 @@ def create_tool_selector_middleware( model = _ensure_chat_model() safe_model = disable_thinking(model) + selector_model = safe_model + from EvoScientist.usage.callback import usage_tracking_enabled + + if usage_tracking_enabled(): + try: + selector_model = safe_model.model_copy( + update={ + "metadata": { + **(safe_model.metadata or {}), + "usage_scope": "tool_selector", + } + } + ) + if not isinstance(selector_model, BaseChatModel): + raise TypeError("selector model copy is not a BaseChatModel") + except Exception: + # The model factory preflights this path before enabling its callback. + # Keep the selector operational even for an unusual third-party model. + from EvoScientist.usage.spool import mark_tracking_degraded + + mark_tracking_degraded("selector_model_copy_unsupported") + selector_model = safe_model + logger.exception("Could not attach Tool Selector usage scope") system_prompt = ( "You are selecting tools for a scientific research agent. " @@ -270,7 +293,7 @@ def create_tool_selector_middleware( def selector_factory(always_include: list[str]) -> AgentMiddleware: return LLMToolSelectorMiddleware( - model=safe_model, + model=selector_model, system_prompt=system_prompt, always_include=always_include, ) diff --git a/EvoScientist/scope_registry.py b/EvoScientist/scope_registry.py new file mode 100644 index 0000000..69ec337 --- /dev/null +++ b/EvoScientist/scope_registry.py @@ -0,0 +1,1274 @@ +"""Persistent ownership registry for conversation workspaces. + +The LangGraph checkpoint store is not an authority for filesystem ownership: +threads, runs and cron records can be created independently and the filesystem +cannot participate in their transactions. This module keeps the small, +deployment-local registry that binds all of them to one conversation scope. + +The v1 implementation intentionally uses SQLite. It is safe for the supported +single-host deployment topology and keeps the registry outside every agent +workspace. Callers must use the public methods below rather than addressing the +database directly so a future PostgreSQL adapter has one replacement point. +""" + +from __future__ import annotations + +import os +import secrets +import sqlite3 +import threading +import uuid +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Literal + +from . import paths + +ScopeState = Literal["provisioning", "draft", "active", "deleting", "deleted"] +OwnerState = Literal[ + "reserved", "active", "draining", "terminal", "failed", "quarantined" +] + + +class ScopeRegistryError(RuntimeError): + """Base class for registry failures.""" + + +class ScopeNotFoundError(ScopeRegistryError): + """Raised when no matching deployment scope exists.""" + + +class ScopeConflictError(ScopeRegistryError): + """Raised for stale revisions or incompatible ownership mappings.""" + + +class ScopeIdempotencyConflictError(ScopeConflictError): + """Raised when one run request id is reused for another payload.""" + + +class ScopeInterruptResolvedError(ScopeConflictError): + """Raised when an interrupt already has a persisted resolution.""" + + +class ScopeAccessError(ScopeRegistryError): + """Raised when a runtime does not own the requested scope.""" + + +@dataclass(frozen=True, slots=True) +class ScopeRecord: + deployment_id: str + scope_id: str + primary_thread_id: str + state: ScopeState + revision: int + primary_owner_id: str + created_at: str + updated_at: str + + +@dataclass(frozen=True, slots=True) +class OwnerRecord: + deployment_id: str + owner_id: str + scope_id: str + owner_type: str + resource_id: str | None + parent_owner_id: str | None + state: OwnerState + created_at: str + updated_at: str + + +@dataclass(frozen=True, slots=True) +class DeploymentLock: + deployment_id: str + lock_name: str + operation_id: str + expires_at: str + + +@dataclass(frozen=True, slots=True) +class ScopeOperation: + deployment_id: str + operation_id: str + scope_id: str | None + kind: str + state: str + result_sha256: str | None + last_error_code: str | None + + +@dataclass(frozen=True, slots=True) +class RunReservation: + deployment_id: str + scope_id: str + run_request_id: str + turn_id: str + interrupt_key: str | None + request_hash: str + run_owner_id: str + run_id: str | None + state: str + + +# Kept as a source-compatible type name for callers that have not yet switched +# to the run-request terminology. A turn is a logical conversation unit; a +# reservation belongs to one concrete run request. +TurnReservation = RunReservation + + +_TERMINAL_SCOPE_STATES = frozenset({"deleted"}) +_TERMINAL_OWNER_STATES = frozenset({"terminal", "quarantined"}) +_SCOPE_TRANSITIONS: dict[str, frozenset[str]] = { + "provisioning": frozenset({"draft", "deleting", "deleted"}), + "draft": frozenset({"active", "deleting", "deleted"}), + "active": frozenset({"deleting"}), + "deleting": frozenset({"deleted"}), + "deleted": frozenset(), +} + + +def _utc_now() -> str: + return datetime.now(UTC).isoformat() + + +def _ensure_uuid(value: str, field: str) -> str: + try: + return str(uuid.UUID(value)) + except (TypeError, ValueError) as exc: + raise ScopeRegistryError(f"{field} must be a UUID") from exc + + +def deployment_id_for_workspace(workspace_root: Path | str | None = None) -> str: + """Return the stable deployment identifier for a workspace root.""" + + configured = os.getenv("EVOSCIENTIST_DEPLOYMENT_ID", "").strip() + if configured: + return configured + # Deploy resolves the workspace before starting LangGraph. Avoid resolve() + # here because this function also runs on the agent's async execution path. + root = Path(workspace_root or paths.WORKSPACE_ROOT).expanduser() + return str(uuid.uuid5(uuid.NAMESPACE_URL, f"evoscientist:{root}")) + + +def default_registry_path(workspace_root: Path | str | None = None) -> Path: + root = Path(workspace_root or paths.WORKSPACE_ROOT).expanduser() + return root / ".evoscientist" / "control" / "scope-registry.sqlite3" + + +def scope_service_token_path(workspace_root: Path | str | None = None) -> Path: + """Location of the local backend-to-BFF registry credential.""" + + return default_registry_path(workspace_root).with_name("scope-service-token") + + +def get_scope_service_token(workspace_root: Path | str | None = None) -> str: + """Return the durable host-local token shared by deploy and the WebUI. + + It lives beside the control-plane database, never in a conversation scope + or browser-delivered configuration. Creation is atomic so a concurrent + launcher cannot replace a running backend's credential. + """ + + path = scope_service_token_path(workspace_root) + path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + try: + path.parent.chmod(0o700) + except OSError: + pass + try: + token = path.read_text(encoding="utf-8").strip() + except FileNotFoundError: + token = "" + if token: + return token + generated = secrets.token_urlsafe(32) + try: + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError: + token = path.read_text(encoding="utf-8").strip() + if token: + return token + raise ScopeRegistryError("workspace scope service token is empty") from None + with os.fdopen(fd, "w", encoding="utf-8") as token_file: + token_file.write(generated) + token_file.write("\n") + try: + path.chmod(0o600) + except OSError: + pass + return generated + + +class ScopeRegistry: + """SQLite-backed registry with revision-checked state transitions.""" + + def __init__(self, database_path: Path | str): + self.path = Path(database_path).expanduser() + self._init_lock = threading.Lock() + self._initialized = False + + def initialize(self) -> None: + with self._init_lock: + if self._initialized: + return + self.path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + try: + self.path.parent.chmod(0o700) + except OSError: + pass + with self._connect() as conn: + self._migrate(conn) + try: + self.path.chmod(0o600) + except OSError: + pass + self._initialized = True + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.path, timeout=30, isolation_level=None) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + conn.execute("PRAGMA journal_mode = WAL") + conn.execute("PRAGMA busy_timeout = 30000") + return conn + + @staticmethod + def _migrate(conn: sqlite3.Connection) -> None: + version = int(conn.execute("PRAGMA user_version").fetchone()[0]) + if version > 2: + raise ScopeRegistryError("scope registry schema is newer than this binary") + if version == 0: + conn.executescript( + """ + CREATE TABLE scopes ( + deployment_id TEXT NOT NULL, + scope_id TEXT NOT NULL, + primary_thread_id TEXT NOT NULL, + state TEXT NOT NULL, + revision INTEGER NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + deleted_at TEXT, + PRIMARY KEY (deployment_id, scope_id), + UNIQUE (deployment_id, primary_thread_id) + ); + + CREATE TABLE scope_owners ( + deployment_id TEXT NOT NULL, + owner_id TEXT NOT NULL, + scope_id TEXT NOT NULL, + owner_type TEXT NOT NULL, + resource_id TEXT, + parent_owner_id TEXT, + state TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + terminal_at TEXT, + PRIMARY KEY (deployment_id, owner_id), + FOREIGN KEY (deployment_id, scope_id) + REFERENCES scopes(deployment_id, scope_id) + ); + CREATE UNIQUE INDEX scope_owner_resource_unique + ON scope_owners(deployment_id, owner_type, resource_id) + WHERE resource_id IS NOT NULL; + CREATE INDEX scope_owners_scope_state + ON scope_owners(deployment_id, scope_id, state); + + CREATE TABLE scope_run_requests ( + deployment_id TEXT NOT NULL, + scope_id TEXT NOT NULL, + run_request_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + interrupt_key TEXT, + request_hash TEXT NOT NULL, + run_owner_id TEXT NOT NULL, + run_id TEXT, + state TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (deployment_id, scope_id, run_request_id), + FOREIGN KEY (deployment_id, scope_id) + REFERENCES scopes(deployment_id, scope_id) + ); + CREATE INDEX scope_run_requests_turn + ON scope_run_requests(deployment_id, scope_id, turn_id); + CREATE UNIQUE INDEX scope_run_requests_interrupt_unique + ON scope_run_requests(deployment_id, scope_id, interrupt_key) + WHERE interrupt_key IS NOT NULL; + + CREATE TABLE scope_operations ( + deployment_id TEXT NOT NULL, + operation_id TEXT NOT NULL, + scope_id TEXT, + kind TEXT NOT NULL, + expected_revision INTEGER, + state TEXT NOT NULL, + external_resource_id TEXT, + result_sha256 TEXT, + last_error_code TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (deployment_id, operation_id) + ); + CREATE INDEX scope_operations_scope_state + ON scope_operations(deployment_id, scope_id, state); + + CREATE TABLE deployment_locks ( + deployment_id TEXT NOT NULL, + lock_name TEXT NOT NULL, + operation_id TEXT NOT NULL, + expires_at TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY (deployment_id, lock_name) + ); + """ + ) + conn.execute("PRAGMA user_version = 2") + return + if version == 1: + # SQLite cannot change a composite primary key in place. Historical + # reservations used turn_id as the idempotency key, so copy each row + # with run_request_id = turn_id. New resume runs may then retain the + # logical turn while receiving distinct request ids. + conn.executescript( + """ + CREATE TABLE scope_run_requests ( + deployment_id TEXT NOT NULL, + scope_id TEXT NOT NULL, + run_request_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + interrupt_key TEXT, + request_hash TEXT NOT NULL, + run_owner_id TEXT NOT NULL, + run_id TEXT, + state TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (deployment_id, scope_id, run_request_id), + FOREIGN KEY (deployment_id, scope_id) + REFERENCES scopes(deployment_id, scope_id) + ); + INSERT INTO scope_run_requests( + deployment_id, scope_id, run_request_id, turn_id, + interrupt_key, request_hash, run_owner_id, run_id, state, + created_at, updated_at + ) + SELECT deployment_id, scope_id, turn_id, turn_id, + NULL, request_hash, run_owner_id, run_id, state, + created_at, updated_at + FROM scope_turns; + CREATE INDEX scope_run_requests_turn + ON scope_run_requests(deployment_id, scope_id, turn_id); + CREATE UNIQUE INDEX scope_run_requests_interrupt_unique + ON scope_run_requests(deployment_id, scope_id, interrupt_key) + WHERE interrupt_key IS NOT NULL; + DROP TABLE scope_turns; + """ + ) + conn.execute("PRAGMA user_version = 2") + + @contextmanager + def _transaction(self) -> Iterator[sqlite3.Connection]: + self.initialize() + with self._connect() as conn: + conn.execute("BEGIN IMMEDIATE") + try: + yield conn + except Exception: + conn.rollback() + raise + else: + conn.commit() + + @staticmethod + def _scope_from_row(row: sqlite3.Row, owner_id: str) -> ScopeRecord: + return ScopeRecord( + deployment_id=str(row["deployment_id"]), + scope_id=str(row["scope_id"]), + primary_thread_id=str(row["primary_thread_id"]), + state=str(row["state"]), # type: ignore[arg-type] + revision=int(row["revision"]), + primary_owner_id=owner_id, + created_at=str(row["created_at"]), + updated_at=str(row["updated_at"]), + ) + + @staticmethod + def _owner_from_row(row: sqlite3.Row) -> OwnerRecord: + return OwnerRecord( + deployment_id=str(row["deployment_id"]), + owner_id=str(row["owner_id"]), + scope_id=str(row["scope_id"]), + owner_type=str(row["owner_type"]), + resource_id=(str(row["resource_id"]) if row["resource_id"] else None), + parent_owner_id=( + str(row["parent_owner_id"]) if row["parent_owner_id"] else None + ), + state=str(row["state"]), # type: ignore[arg-type] + created_at=str(row["created_at"]), + updated_at=str(row["updated_at"]), + ) + + @staticmethod + def _primary_owner( + conn: sqlite3.Connection, deployment_id: str, scope_id: str + ) -> str: + row = conn.execute( + """ + SELECT owner_id FROM scope_owners + WHERE deployment_id = ? AND scope_id = ? AND owner_type = 'primary_thread' + LIMIT 1 + """, + (deployment_id, scope_id), + ).fetchone() + if row is None: + raise ScopeRegistryError("scope has no primary owner") + return str(row["owner_id"]) + + @staticmethod + def _workspace_mutation_lock_active( + conn: sqlite3.Connection, + deployment_id: str, + operation_id: str | None = None, + ) -> bool: + row = conn.execute( + """ + SELECT 1 FROM deployment_locks + WHERE deployment_id = ? + AND lock_name IN ('workspace-cutover', 'workspace-lifecycle') + AND expires_at > ? + AND (? IS NULL OR operation_id != ?) + """, + (deployment_id, _utc_now(), operation_id, operation_id), + ).fetchone() + return row is not None + + def provision( + self, + deployment_id: str, + primary_thread_id: str, + *, + scope_id: str | None = None, + state: ScopeState = "draft", + operation_id: str | None = None, + lock_operation_id: str | None = None, + ) -> ScopeRecord: + """Reserve one immutable scope for a primary thread. + + Retrying the same primary thread returns its existing mapping. Passing a + different explicit scope for an existing thread is a conflict rather than + an opportunity to silently remap its files. + """ + + if state not in {"provisioning", "draft"}: + raise ScopeRegistryError("new scopes must start provisioning or draft") + requested_scope = ( + _ensure_uuid(scope_id, "scope_id") if scope_id else str(uuid.uuid4()) + ) + now = _utc_now() + operation_id = ( + _ensure_uuid(operation_id, "operation_id") + if operation_id + else str(uuid.uuid4()) + ) + if lock_operation_id is not None: + lock_operation_id = _ensure_uuid(lock_operation_id, "lock_operation_id") + with self._transaction() as conn: + existing = conn.execute( + """ + SELECT * FROM scopes WHERE deployment_id = ? AND primary_thread_id = ? + """, + (deployment_id, primary_thread_id), + ).fetchone() + if existing is not None: + if scope_id and str(existing["scope_id"]) != requested_scope: + raise ScopeConflictError("thread already belongs to another scope") + primary_owner = self._primary_owner( + conn, deployment_id, str(existing["scope_id"]) + ) + return self._scope_from_row(existing, primary_owner) + if self._workspace_mutation_lock_active( + conn, deployment_id, lock_operation_id + ): + raise ScopeAccessError( + "workspace cutover or maintenance is in progress" + ) + conn.execute( + """ + INSERT INTO scopes( + deployment_id, scope_id, primary_thread_id, state, revision, + created_at, updated_at + ) VALUES (?, ?, ?, ?, 1, ?, ?) + """, + (deployment_id, requested_scope, primary_thread_id, state, now, now), + ) + primary_owner = str(uuid.uuid4()) + conn.execute( + """ + INSERT INTO scope_owners( + deployment_id, owner_id, scope_id, owner_type, resource_id, + parent_owner_id, state, created_at, updated_at + ) VALUES (?, ?, ?, 'primary_thread', ?, NULL, 'active', ?, ?) + """, + ( + deployment_id, + primary_owner, + requested_scope, + primary_thread_id, + now, + now, + ), + ) + conn.execute( + """ + INSERT OR REPLACE INTO scope_operations( + deployment_id, operation_id, scope_id, kind, state, created_at, updated_at + ) VALUES (?, ?, ?, 'provision', 'completed', ?, ?) + """, + (deployment_id, operation_id, requested_scope, now, now), + ) + return ScopeRecord( + deployment_id=deployment_id, + scope_id=requested_scope, + primary_thread_id=primary_thread_id, + state=state, + revision=1, + primary_owner_id=primary_owner, + created_at=now, + updated_at=now, + ) + + def get_by_thread(self, deployment_id: str, thread_id: str) -> ScopeRecord: + self.initialize() + with self._connect() as conn: + row = conn.execute( + "SELECT * FROM scopes WHERE deployment_id = ? AND primary_thread_id = ?", + (deployment_id, thread_id), + ).fetchone() + if row is None: + raise ScopeNotFoundError("no workspace scope for thread") + return self._scope_from_row( + row, self._primary_owner(conn, deployment_id, str(row["scope_id"])) + ) + + def get(self, deployment_id: str, scope_id: str) -> ScopeRecord: + scope_id = _ensure_uuid(scope_id, "scope_id") + self.initialize() + with self._connect() as conn: + row = conn.execute( + "SELECT * FROM scopes WHERE deployment_id = ? AND scope_id = ?", + (deployment_id, scope_id), + ).fetchone() + if row is None: + raise ScopeNotFoundError("workspace scope not found") + return self._scope_from_row( + row, self._primary_owner(conn, deployment_id, scope_id) + ) + + def list_scopes( + self, deployment_id: str, *, state: ScopeState | None = None + ) -> list[ScopeRecord]: + """List deployment scopes for administrative maintenance only.""" + + if state is not None and state not in _SCOPE_TRANSITIONS: + raise ScopeRegistryError("invalid workspace scope state") + self.initialize() + with self._connect() as conn: + if state is None: + rows = conn.execute( + """ + SELECT * FROM scopes WHERE deployment_id = ? ORDER BY created_at ASC + """, + (deployment_id,), + ).fetchall() + else: + rows = conn.execute( + """ + SELECT * FROM scopes WHERE deployment_id = ? AND state = ? + ORDER BY created_at ASC + """, + (deployment_id, state), + ).fetchall() + return [ + self._scope_from_row( + row, + self._primary_owner(conn, deployment_id, str(row["scope_id"])), + ) + for row in rows + ] + + def transition_scope( + self, + deployment_id: str, + scope_id: str, + *, + expected_revision: int, + state: ScopeState, + ) -> ScopeRecord: + scope_id = _ensure_uuid(scope_id, "scope_id") + now = _utc_now() + with self._transaction() as conn: + current = conn.execute( + "SELECT * FROM scopes WHERE deployment_id = ? AND scope_id = ?", + (deployment_id, scope_id), + ).fetchone() + if current is None: + raise ScopeNotFoundError("workspace scope not found") + current_state = str(current["state"]) + if state not in _SCOPE_TRANSITIONS.get(current_state, frozenset()): + raise ScopeConflictError( + f"cannot transition scope from {current_state} to {state}" + ) + deleted_at = now if state == "deleted" else None + updated = conn.execute( + """ + UPDATE scopes + SET state = ?, revision = revision + 1, updated_at = ?, + deleted_at = COALESCE(?, deleted_at) + WHERE deployment_id = ? AND scope_id = ? AND revision = ? + """, + (state, now, deleted_at, deployment_id, scope_id, expected_revision), + ) + if updated.rowcount != 1: + raise ScopeConflictError("scope revision changed") + row = conn.execute( + "SELECT * FROM scopes WHERE deployment_id = ? AND scope_id = ?", + (deployment_id, scope_id), + ).fetchone() + assert row is not None + return self._scope_from_row( + row, self._primary_owner(conn, deployment_id, scope_id) + ) + + def register_owner( + self, + deployment_id: str, + scope_id: str, + *, + owner_type: str, + resource_id: str | None = None, + parent_owner_id: str | None = None, + owner_id: str | None = None, + state: OwnerState = "reserved", + ) -> OwnerRecord: + scope_id = _ensure_uuid(scope_id, "scope_id") + owner_id = _ensure_uuid(owner_id, "owner_id") if owner_id else str(uuid.uuid4()) + now = _utc_now() + with self._transaction() as conn: + scope = conn.execute( + "SELECT state FROM scopes WHERE deployment_id = ? AND scope_id = ?", + (deployment_id, scope_id), + ).fetchone() + if scope is None: + raise ScopeNotFoundError("workspace scope not found") + if self._workspace_mutation_lock_active(conn, deployment_id): + raise ScopeAccessError( + "workspace cutover or maintenance is in progress" + ) + if ( + str(scope["state"]) in _TERMINAL_SCOPE_STATES + or str(scope["state"]) == "deleting" + ): + raise ScopeAccessError("scope does not accept new owners") + if parent_owner_id: + parent = conn.execute( + """ + SELECT state FROM scope_owners + WHERE deployment_id = ? AND owner_id = ? AND scope_id = ? + """, + (deployment_id, parent_owner_id, scope_id), + ).fetchone() + if parent is None or str(parent["state"]) != "active": + raise ScopeAccessError("parent owner is not active") + try: + conn.execute( + """ + INSERT INTO scope_owners( + deployment_id, owner_id, scope_id, owner_type, resource_id, + parent_owner_id, state, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + deployment_id, + owner_id, + scope_id, + owner_type, + resource_id, + parent_owner_id, + state, + now, + now, + ), + ) + except sqlite3.IntegrityError as exc: + raise ScopeConflictError( + "owner or external resource already exists" + ) from exc + return OwnerRecord( + deployment_id=deployment_id, + owner_id=owner_id, + scope_id=scope_id, + owner_type=owner_type, + resource_id=resource_id, + parent_owner_id=parent_owner_id, + state=state, + created_at=now, + updated_at=now, + ) + + def bind_owner( + self, + deployment_id: str, + scope_id: str, + owner_id: str, + resource_id: str, + *, + state: OwnerState = "active", + ) -> OwnerRecord: + scope_id = _ensure_uuid(scope_id, "scope_id") + owner_id = _ensure_uuid(owner_id, "owner_id") + now = _utc_now() + with self._transaction() as conn: + try: + result = conn.execute( + """ + UPDATE scope_owners + SET resource_id = ?, state = ?, updated_at = ? + WHERE deployment_id = ? AND scope_id = ? AND owner_id = ? + AND state NOT IN ('terminal', 'quarantined') + """, + (resource_id, state, now, deployment_id, scope_id, owner_id), + ) + except sqlite3.IntegrityError as exc: + raise ScopeConflictError( + "external resource belongs to another scope" + ) from exc + if result.rowcount != 1: + raise ScopeAccessError("owner cannot be bound") + row = conn.execute( + """ + SELECT * FROM scope_owners + WHERE deployment_id = ? AND scope_id = ? AND owner_id = ? + """, + (deployment_id, scope_id, owner_id), + ).fetchone() + assert row is not None + return self._owner_from_row(row) + + def assert_runtime( + self, + deployment_id: str, + scope_id: str, + thread_id: str, + owner_id: str, + ) -> ScopeRecord: + """Verify a graph/tool runtime is an active owner of this scope.""" + + scope_id = _ensure_uuid(scope_id, "workspace_scope_id") + owner_id = _ensure_uuid(owner_id, "workspace_scope_owner_id") + self.initialize() + with self._connect() as conn: + scope = conn.execute( + "SELECT * FROM scopes WHERE deployment_id = ? AND scope_id = ?", + (deployment_id, scope_id), + ).fetchone() + if scope is None: + raise ScopeNotFoundError("workspace scope not found") + if str(scope["state"]) not in {"draft", "active"}: + raise ScopeAccessError("workspace scope is not active") + if self._workspace_mutation_lock_active(conn, deployment_id): + raise ScopeAccessError( + "workspace cutover or maintenance is in progress" + ) + owner = conn.execute( + """ + SELECT * FROM scope_owners + WHERE deployment_id = ? AND scope_id = ? AND owner_id = ? + """, + (deployment_id, scope_id, owner_id), + ).fetchone() + if owner is None or ( + str(owner["state"]) != "active" + and not ( + str(owner["owner_type"]) == "primary_run" + and str(owner["state"]) == "reserved" + ) + ): + raise ScopeAccessError("workspace owner is not active") + if str(owner["owner_type"]) == "primary_thread": + if ( + str(scope["primary_thread_id"]) != thread_id + or str(owner["resource_id"]) != thread_id + ): + raise ScopeAccessError("primary thread does not own this scope") + elif str(owner["owner_type"]) == "primary_run": + if str(scope["primary_thread_id"]) != thread_id or str( + owner["parent_owner_id"] + ) != self._primary_owner(conn, deployment_id, scope_id): + raise ScopeAccessError("primary run does not own this scope") + elif str(owner["owner_type"]) != "schedule" and str( + owner["resource_id"] + ) not in {None, thread_id}: + raise ScopeAccessError("derived thread does not own this scope") + return self._scope_from_row( + scope, self._primary_owner(conn, deployment_id, scope_id) + ) + + def owners(self, deployment_id: str, scope_id: str) -> list[OwnerRecord]: + scope_id = _ensure_uuid(scope_id, "scope_id") + self.initialize() + with self._connect() as conn: + rows = conn.execute( + """ + SELECT * FROM scope_owners + WHERE deployment_id = ? AND scope_id = ? ORDER BY created_at ASC + """, + (deployment_id, scope_id), + ).fetchall() + return [self._owner_from_row(row) for row in rows] + + def get_owner_by_resource( + self, deployment_id: str, resource_id: str + ) -> OwnerRecord: + """Return the durable owner for one external child resource.""" + + self.initialize() + with self._connect() as conn: + row = conn.execute( + """ + SELECT * FROM scope_owners + WHERE deployment_id = ? AND resource_id = ? + """, + (deployment_id, resource_id), + ).fetchone() + if row is None: + raise ScopeNotFoundError("workspace owner not found") + return self._owner_from_row(row) + + def reserve_run( + self, + deployment_id: str, + scope_id: str, + run_request_id: str, + turn_id: str, + request_hash: str, + *, + interrupt_key: str | None = None, + ) -> RunReservation: + """Reserve one idempotent run request before creating it remotely. + + ``turn_id`` groups a user message and all of its approval resumes. + ``run_request_id`` identifies one external ``runs.create`` call, so a + lost response can be retried without turning a valid resume into a + conflict with the initial run. + """ + + scope_id = _ensure_uuid(scope_id, "scope_id") + run_request_id = _ensure_uuid(run_request_id, "run_request_id") + turn_id = _ensure_uuid(turn_id, "turn_id") + if interrupt_key is not None: + if ( + not isinstance(interrupt_key, str) + or not interrupt_key + or len(interrupt_key) > 256 + ): + raise ScopeRegistryError( + "interrupt_key must be a non-empty string up to 256 characters" + ) + now = _utc_now() + with self._transaction() as conn: + existing = conn.execute( + """ + SELECT * FROM scope_run_requests + WHERE deployment_id = ? AND scope_id = ? AND run_request_id = ? + """, + (deployment_id, scope_id, run_request_id), + ).fetchone() + if existing is not None: + existing_interrupt_key = ( + str(existing["interrupt_key"]) + if existing["interrupt_key"] + else None + ) + if ( + str(existing["request_hash"]) != request_hash + or str(existing["turn_id"]) != turn_id + or existing_interrupt_key != interrupt_key + ): + raise ScopeIdempotencyConflictError( + "run_request_id was reused with another request" + ) + return RunReservation( + deployment_id=deployment_id, + scope_id=scope_id, + run_request_id=str(existing["run_request_id"]), + turn_id=turn_id, + interrupt_key=existing_interrupt_key, + request_hash=request_hash, + run_owner_id=str(existing["run_owner_id"]), + run_id=str(existing["run_id"]) if existing["run_id"] else None, + state=str(existing["state"]), + ) + if interrupt_key is not None: + resolved_interrupt = conn.execute( + """ + SELECT run_request_id FROM scope_run_requests + WHERE deployment_id = ? AND scope_id = ? AND interrupt_key = ? + """, + (deployment_id, scope_id, interrupt_key), + ).fetchone() + if resolved_interrupt is not None: + raise ScopeInterruptResolvedError( + "interrupt already has a resume request" + ) + scope = conn.execute( + "SELECT state FROM scopes WHERE deployment_id = ? AND scope_id = ?", + (deployment_id, scope_id), + ).fetchone() + if scope is None or str(scope["state"]) not in {"draft", "active"}: + raise ScopeAccessError("workspace scope does not accept a run") + if self._workspace_mutation_lock_active(conn, deployment_id): + raise ScopeAccessError( + "workspace cutover or maintenance is in progress" + ) + parent_owner = self._primary_owner(conn, deployment_id, scope_id) + run_owner_id = str(uuid.uuid4()) + conn.execute( + """ + INSERT INTO scope_owners( + deployment_id, owner_id, scope_id, owner_type, resource_id, + parent_owner_id, state, created_at, updated_at + ) VALUES (?, ?, ?, 'primary_run', NULL, ?, 'reserved', ?, ?) + """, + (deployment_id, run_owner_id, scope_id, parent_owner, now, now), + ) + conn.execute( + """ + INSERT INTO scope_run_requests( + deployment_id, scope_id, run_request_id, turn_id, interrupt_key, + request_hash, run_owner_id, run_id, state, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, NULL, 'reserved', ?, ?) + """, + ( + deployment_id, + scope_id, + run_request_id, + turn_id, + interrupt_key, + request_hash, + run_owner_id, + now, + now, + ), + ) + return RunReservation( + deployment_id=deployment_id, + scope_id=scope_id, + run_request_id=run_request_id, + turn_id=turn_id, + interrupt_key=interrupt_key, + request_hash=request_hash, + run_owner_id=run_owner_id, + run_id=None, + state="reserved", + ) + + def bind_run( + self, + deployment_id: str, + scope_id: str, + run_request_id: str, + run_id: str, + ) -> RunReservation: + """Attach the remote run id to a prior reservation exactly once.""" + + scope_id = _ensure_uuid(scope_id, "scope_id") + run_request_id = _ensure_uuid(run_request_id, "run_request_id") + now = _utc_now() + with self._transaction() as conn: + row = conn.execute( + """ + SELECT * FROM scope_run_requests + WHERE deployment_id = ? AND scope_id = ? AND run_request_id = ? + """, + (deployment_id, scope_id, run_request_id), + ).fetchone() + if row is None: + raise ScopeNotFoundError("workspace run reservation not found") + if row["run_id"] and str(row["run_id"]) != run_id: + raise ScopeConflictError("run request is already bound to another run") + owner_id = str(row["run_owner_id"]) + conn.execute( + """ + UPDATE scope_owners + SET resource_id = ?, state = 'active', updated_at = ? + WHERE deployment_id = ? AND scope_id = ? AND owner_id = ? + AND state IN ('reserved', 'active') + """, + (run_id, now, deployment_id, scope_id, owner_id), + ) + conn.execute( + """ + UPDATE scope_run_requests SET run_id = ?, state = 'active', updated_at = ? + WHERE deployment_id = ? AND scope_id = ? AND run_request_id = ? + """, + (run_id, now, deployment_id, scope_id, run_request_id), + ) + return RunReservation( + deployment_id=deployment_id, + scope_id=scope_id, + run_request_id=run_request_id, + turn_id=str(row["turn_id"]), + interrupt_key=( + str(row["interrupt_key"]) if row["interrupt_key"] else None + ), + request_hash=str(row["request_hash"]), + run_owner_id=owner_id, + run_id=run_id, + state="active", + ) + + def reserve_turn( + self, + deployment_id: str, + scope_id: str, + turn_id: str, + request_hash: str, + ) -> RunReservation: + """Backward-compatible reservation for pre-run-request callers.""" + + return self.reserve_run( + deployment_id, + scope_id, + turn_id, + turn_id, + request_hash, + ) + + def bind_turn( + self, + deployment_id: str, + scope_id: str, + turn_id: str, + run_id: str, + ) -> RunReservation: + """Backward-compatible binding for pre-run-request callers.""" + + return self.bind_run(deployment_id, scope_id, turn_id, run_id) + + def acquire_lock( + self, + deployment_id: str, + lock_name: str, + operation_id: str, + *, + lease_seconds: int = 60, + ) -> DeploymentLock: + operation_id = _ensure_uuid(operation_id, "operation_id") + now = datetime.now(UTC) + expires = now + timedelta(seconds=max(1, lease_seconds)) + now_text, expires_text = now.isoformat(), expires.isoformat() + with self._transaction() as conn: + row = conn.execute( + """ + SELECT operation_id, expires_at FROM deployment_locks + WHERE deployment_id = ? AND lock_name = ? + """, + (deployment_id, lock_name), + ).fetchone() + if ( + row is not None + and str(row["operation_id"]) != operation_id + and str(row["expires_at"]) > now_text + ): + raise ScopeConflictError("deployment lock is held") + conn.execute( + """ + INSERT INTO deployment_locks( + deployment_id, lock_name, operation_id, expires_at, created_at + ) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(deployment_id, lock_name) DO UPDATE SET + operation_id = excluded.operation_id, + expires_at = excluded.expires_at, + created_at = excluded.created_at + """, + (deployment_id, lock_name, operation_id, expires_text, now_text), + ) + return DeploymentLock(deployment_id, lock_name, operation_id, expires_text) + + def release_lock( + self, deployment_id: str, lock_name: str, operation_id: str + ) -> None: + with self._transaction() as conn: + result = conn.execute( + """ + DELETE FROM deployment_locks + WHERE deployment_id = ? AND lock_name = ? AND operation_id = ? + """, + (deployment_id, lock_name, operation_id), + ) + if result.rowcount != 1: + raise ScopeAccessError("deployment lock is not held by this operation") + + def renew_lock( + self, + deployment_id: str, + lock_name: str, + operation_id: str, + *, + lease_seconds: int = 60, + ) -> DeploymentLock: + """Renew a lease only while this operation still owns an active lock.""" + + operation_id = _ensure_uuid(operation_id, "operation_id") + now = datetime.now(UTC) + now_text = now.isoformat() + expires_text = (now + timedelta(seconds=max(1, lease_seconds))).isoformat() + with self._transaction() as conn: + result = conn.execute( + """ + UPDATE deployment_locks SET expires_at = ? + WHERE deployment_id = ? AND lock_name = ? AND operation_id = ? + AND expires_at > ? + """, + (expires_text, deployment_id, lock_name, operation_id, now_text), + ) + if result.rowcount != 1: + raise ScopeAccessError( + "deployment lock expired or belongs to another operation" + ) + return DeploymentLock(deployment_id, lock_name, operation_id, expires_text) + + def active_lock(self, deployment_id: str, lock_name: str) -> DeploymentLock | None: + """Return an unexpired deployment lock without mutating its lease.""" + + self.initialize() + now = _utc_now() + with self._connect() as conn: + row = conn.execute( + """ + SELECT operation_id, expires_at FROM deployment_locks + WHERE deployment_id = ? AND lock_name = ? AND expires_at > ? + """, + (deployment_id, lock_name, now), + ).fetchone() + if row is None: + return None + return DeploymentLock( + deployment_id=deployment_id, + lock_name=lock_name, + operation_id=str(row["operation_id"]), + expires_at=str(row["expires_at"]), + ) + + def begin_operation( + self, + deployment_id: str, + operation_id: str, + *, + kind: str, + scope_id: str | None = None, + ) -> None: + operation_id = _ensure_uuid(operation_id, "operation_id") + if scope_id is not None: + scope_id = _ensure_uuid(scope_id, "scope_id") + now = _utc_now() + with self._transaction() as conn: + existing = conn.execute( + """ + SELECT kind, state FROM scope_operations + WHERE deployment_id = ? AND operation_id = ? + """, + (deployment_id, operation_id), + ).fetchone() + if existing is not None: + if str(existing["kind"]) != kind: + raise ScopeConflictError("operation id belongs to another kind") + return + conn.execute( + """ + INSERT INTO scope_operations( + deployment_id, operation_id, scope_id, kind, state, created_at, updated_at + ) VALUES (?, ?, ?, ?, 'running', ?, ?) + """, + (deployment_id, operation_id, scope_id, kind, now, now), + ) + + def finish_operation( + self, + deployment_id: str, + operation_id: str, + *, + state: str, + result_sha256: str | None = None, + last_error_code: str | None = None, + ) -> None: + if state not in {"completed", "failed"}: + raise ScopeRegistryError( + "operation terminal state must be completed or failed" + ) + operation_id = _ensure_uuid(operation_id, "operation_id") + with self._transaction() as conn: + result = conn.execute( + """ + UPDATE scope_operations + SET state = ?, result_sha256 = ?, last_error_code = ?, updated_at = ? + WHERE deployment_id = ? AND operation_id = ? AND state = 'running' + """, + ( + state, + result_sha256, + last_error_code, + _utc_now(), + deployment_id, + operation_id, + ), + ) + if result.rowcount != 1: + raise ScopeAccessError("operation is not running") + + def get_operation(self, deployment_id: str, operation_id: str) -> ScopeOperation: + operation_id = _ensure_uuid(operation_id, "operation_id") + self.initialize() + with self._connect() as conn: + row = conn.execute( + """ + SELECT scope_id, kind, state, result_sha256, last_error_code + FROM scope_operations WHERE deployment_id = ? AND operation_id = ? + """, + (deployment_id, operation_id), + ).fetchone() + if row is None: + raise ScopeNotFoundError("scope operation not found") + return ScopeOperation( + deployment_id=deployment_id, + operation_id=operation_id, + scope_id=str(row["scope_id"]) if row["scope_id"] else None, + kind=str(row["kind"]), + state=str(row["state"]), + result_sha256=( + str(row["result_sha256"]) if row["result_sha256"] else None + ), + last_error_code=( + str(row["last_error_code"]) if row["last_error_code"] else None + ), + ) + + +_registry_cache: dict[Path, ScopeRegistry] = {} +_registry_cache_lock = threading.Lock() + + +def get_scope_registry(workspace_root: Path | str | None = None) -> ScopeRegistry: + path = default_registry_path(workspace_root) + with _registry_cache_lock: + registry = _registry_cache.get(path) + if registry is None: + registry = ScopeRegistry(path) + _registry_cache[path] = registry + registry.initialize() + return registry diff --git a/EvoScientist/usage/__init__.py b/EvoScientist/usage/__init__.py new file mode 100644 index 0000000..2fc7918 --- /dev/null +++ b/EvoScientist/usage/__init__.py @@ -0,0 +1,6 @@ +"""Fail-open model usage capture for the integrated EvoScientist WebUI.""" + +from .callback import UsageModelIdentity, attach_usage_callback +from .identity import prepare_usage_environment + +__all__ = ["UsageModelIdentity", "attach_usage_callback", "prepare_usage_environment"] diff --git a/EvoScientist/usage/callback.py b/EvoScientist/usage/callback.py new file mode 100644 index 0000000..26620fb --- /dev/null +++ b/EvoScientist/usage/callback.py @@ -0,0 +1,356 @@ +"""LangChain terminal callback that emits one UsageEvent per real model call.""" + +from __future__ import annotations + +import copy +import logging +import os +import threading +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import Any +from uuid import UUID + +from langchain_core.callbacks import BaseCallbackHandler + +from .schema import MAX_SAFE_TOKEN_INTEGER, UsageEventV1, UsageScope +from .spool import get_usage_spool, mark_tracking_degraded + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class UsageModelIdentity: + provider_profile_id: str + provider_revision: str | None + provider_adapter: str + model_alias: str + upstream_model_id: str + + +@dataclass +class _CallState: + started_at: datetime + parent_run_id: str | None + metadata: dict[str, Any] = field(default_factory=dict) + last_usage: dict[str, Any] | None = None + provider_request_id: str | None = None + + +def usage_tracking_requested() -> bool: + """Return whether this process was launched in usage-tracking mode.""" + enabled = os.getenv("EVOSCIENTIST_USAGE_TRACKING", "").strip().lower() + return enabled in {"1", "true", "yes", "on"} + + +def usage_tracking_enabled() -> bool: + """Return whether the integrated launcher supplied a complete usage sink.""" + return usage_tracking_requested() and all( + os.getenv(name, "").strip() + for name in ( + "EVOSCIENTIST_USAGE_SINK_URL", + "EVOSCIENTIST_USAGE_SINK_TOKEN", + "EVOSCIENTIST_DEPLOYMENT_ID", + "EVOSCIENTIST_WORKSPACE_ID", + "EVOSCIENTIST_USAGE_SPOOL_DIR", + ) + ) + + +def _text(value: Any, *, limit: int = 256) -> str | None: + if value is None: + return None + rendered = "".join( + character + for character in str(value) + if ord(character) > 31 and ord(character) != 127 + ) + return rendered[:limit] if rendered else None + + +def _metadata_value(metadata: Mapping[str, Any], *keys: str) -> Any: + for key in keys: + value = metadata.get(key) + if value is not None: + return value + return None + + +def _scope(metadata: Mapping[str, Any]) -> UsageScope: + explicit = metadata.get("usage_scope") + allowed = { + "main", "sync_subagent", "async_subagent", "tool_selector", "summarizer", + "memory", "scheduler", "autoskills", "diagnostic", "skill_eval", "unattributed", + } + if explicit in allowed: + return explicit # type: ignore[return-value] + if metadata.get("lc_source") == "summarization": + return "summarizer" + run_kind = metadata.get("run_kind") + if run_kind == "scheduled_task": + return "scheduler" + if run_kind == "evomemory_autoskills": + return "autoskills" + if isinstance(run_kind, str) and run_kind.startswith("evomemory_"): + return "memory" + # deepagents attaches this stable metadata field to every compiled agent + # graph. Remote async runs carry an explicit usage_scope and have already + # returned above; remaining non-main names are synchronous subagents. + agent_name = metadata.get("lc_agent_name") + if agent_name == "EvoScientist": + return "main" + if isinstance(agent_name, str) and agent_name: + return "sync_subagent" + if metadata.get("async_task_id") or metadata.get("source_session_id"): + return "async_subagent" + if metadata.get("thread_id") or metadata.get("langgraph_thread_id"): + return "main" + return "unattributed" + + +def _clean_details(value: Any) -> dict[str, int]: + if not isinstance(value, Mapping): + return {} + result: dict[str, int] = {} + for key, count in value.items(): + if ( + isinstance(key, str) + and 0 < len(key) <= 128 + and isinstance(count, int) + and not isinstance(count, bool) + and 0 <= count <= MAX_SAFE_TOKEN_INTEGER + ): + result[key] = count + return result + + +def _normalize_usage(value: Any) -> dict[str, Any] | None: + if not isinstance(value, Mapping): + return None + input_tokens = value.get("input_tokens", value.get("prompt_tokens")) + output_tokens = value.get("output_tokens", value.get("completion_tokens")) + if not all( + isinstance(token, int) + and not isinstance(token, bool) + and 0 <= token <= MAX_SAFE_TOKEN_INTEGER + for token in (input_tokens, output_tokens) + ): + return None + if input_tokens + output_tokens > MAX_SAFE_TOKEN_INTEGER: + return None + provider_total = value.get("total_tokens") + if not ( + isinstance(provider_total, int) + and not isinstance(provider_total, bool) + and 0 <= provider_total <= MAX_SAFE_TOKEN_INTEGER + ): + provider_total = None + return { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "provider_total_tokens": provider_total, + "input_token_details": _clean_details(value.get("input_token_details")), + "output_token_details": _clean_details(value.get("output_token_details")), + } + + +def _message_from_chunk(chunk: Any) -> Any: + return getattr(chunk, "message", chunk) + + +def _message_usage(message: Any) -> dict[str, Any] | None: + return _normalize_usage(getattr(message, "usage_metadata", None)) + + +def _request_id(message: Any) -> str | None: + metadata = getattr(message, "response_metadata", None) + if not isinstance(metadata, Mapping): + return None + return _text(_metadata_value(metadata, "request_id", "id", "x_request_id")) + + +class UsageCaptureCallback(BaseCallbackHandler): + """Capture final provider usage without influencing the model invocation.""" + + def __init__(self, identity: UsageModelIdentity) -> None: + self.identity = identity + self._lock = threading.Lock() + self._calls: dict[str, _CallState] = {} + + def on_chat_model_start( + self, + serialized: dict[str, Any], + messages: list[list[Any]], + *, + run_id: UUID, + parent_run_id: UUID | None = None, + metadata: dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + try: + with self._lock: + self._calls[str(run_id)] = _CallState( + started_at=datetime.now(UTC), + parent_run_id=str(parent_run_id) if parent_run_id else None, + metadata=dict(metadata or {}), + ) + except Exception: + logger.exception("Usage callback could not initialize run state") + + def on_llm_new_token( + self, token: Any, *, run_id: UUID, chunk: Any = None, **kwargs: Any + ) -> None: + try: + message = _message_from_chunk(chunk) + usage = _message_usage(message) + if usage is None: + return + with self._lock: + state = self._calls.get(str(run_id)) + if state: + state.last_usage = usage + state.provider_request_id = _request_id(message) or state.provider_request_id + except Exception: + logger.exception("Usage callback could not inspect a stream chunk") + + def on_llm_end(self, response: Any, *, run_id: UUID, **kwargs: Any) -> None: + usage: dict[str, Any] | None = None + request_id: str | None = None + try: + for group in getattr(response, "generations", []) or []: + for generation in group or []: + message = getattr(generation, "message", generation) + usage = _message_usage(message) or usage + request_id = _request_id(message) or request_id + if usage is None: + output = getattr(response, "llm_output", None) + if isinstance(output, Mapping): + usage = _normalize_usage(output.get("token_usage") or output.get("usage")) + except Exception: + logger.exception("Usage callback could not inspect terminal usage") + try: + self._finish(str(run_id), usage=usage, request_id=request_id) + except Exception: + logger.exception("Usage callback could not emit terminal usage") + + def on_llm_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None: + try: + self._finish(str(run_id), usage=None, request_id=None) + except Exception: + logger.exception("Usage callback could not emit unknown terminal usage") + + def _finish( + self, run_id: str, *, usage: dict[str, Any] | None, request_id: str | None + ) -> None: + with self._lock: + state = self._calls.pop(run_id, None) + if state is None: + return + usage = usage or state.last_usage + completed = datetime.now(UTC) + metadata = state.metadata + confirmed = usage is not None + event = UsageEventV1( + schema_version=1, + event_id=( + f"{os.environ['EVOSCIENTIST_DEPLOYMENT_ID']}:{run_id}:callback_final:1" + ), + event_type="usage_observed", + source="callback_final", + authority_class="observed_final", + revision=1, + deployment_id=os.environ["EVOSCIENTIST_DEPLOYMENT_ID"], + workspace_id=os.environ["EVOSCIENTIST_WORKSPACE_ID"], + model_call_id=run_id, + parent_run_id=state.parent_run_id, + provider_request_id=request_id or state.provider_request_id, + thread_id=_text( + _metadata_value(metadata, "thread_id", "langgraph_thread_id") + ), + source_session_id=_text( + _metadata_value( + metadata, "source_session_id", "evomemory_source_session_id" + ) + ), + turn_id=_text( + _metadata_value(metadata, "turn_id", "evomemory_source_turn_id") + ), + workspace_dir=_text( + _metadata_value(metadata, "workspace_dir"), limit=4096 + ) + or _text(os.getenv("EVOSCIENTIST_WORKSPACE_DIR"), limit=4096), + scope=_scope(metadata), + source_agent=_text( + _metadata_value(metadata, "source_agent", "evomemory_source_agent") + ), + provider_profile_id=_text( + self.identity.provider_profile_id, limit=512 + ) + or "unknown", + provider_revision=_text(self.identity.provider_revision), + provider_adapter=_text(self.identity.provider_adapter, limit=512) + or "unknown", + model_alias=_text(self.identity.model_alias, limit=512) or "unknown", + upstream_model_id=_text( + self.identity.upstream_model_id, limit=512 + ) + or "unknown", + usage_status="confirmed" if confirmed else "unknown", + input_tokens=usage["input_tokens"] if usage else None, + output_tokens=usage["output_tokens"] if usage else None, + provider_total_tokens=usage["provider_total_tokens"] if usage else None, + input_token_details=usage["input_token_details"] if usage else {}, + output_token_details=usage["output_token_details"] if usage else {}, + started_at=state.started_at, + observed_at=completed, + completed_at=completed, + ) + get_usage_spool().enqueue(event) + + +def _merged_callbacks(existing: Any, callback: UsageCaptureCallback) -> Any: + if existing is None: + return [callback] + if isinstance(existing, (list, tuple)): + return [*existing, callback] + if hasattr(existing, "add_handler"): + manager = copy.copy(existing) + manager.add_handler(callback, inherit=True) + return manager + raise TypeError("unsupported callbacks value") + + +def attach_usage_callback(model: Any, identity: UsageModelIdentity) -> Any: + """Return a callback-enabled model copy, or the original model on degradation.""" + + if not usage_tracking_enabled(): + return model + try: + from langchain_core.language_models import BaseChatModel + + if not isinstance(model, BaseChatModel): + raise TypeError("model is not a BaseChatModel") + callback = UsageCaptureCallback(identity) + updated = model.model_copy( + update={"callbacks": _merged_callbacks(model.callbacks, callback)} + ) + if not isinstance(updated, BaseChatModel): + raise TypeError("callback model_copy did not return a BaseChatModel") + # Tool Selector depends on a second metadata-only model copy. Verify the + # contract before enabling tracking for this model so a selector call + # can never be silently counted as a main-agent call. + probe = updated.model_copy( + update={"metadata": {**(updated.metadata or {}), "usage_scope": "tool_selector"}} + ) + if not isinstance(probe, BaseChatModel): + raise TypeError("selector metadata model_copy did not return a BaseChatModel") + # Start capability negotiation and heartbeat at backend startup rather + # than waiting for the first completed model call. This lets the UI + # distinguish a healthy empty database from unsupported tracking. + get_usage_spool() + return updated + except Exception: + mark_tracking_degraded("selector_model_copy_unsupported") + logger.exception("Usage callback injection failed; model behavior is unchanged") + return model diff --git a/EvoScientist/usage/identity.py b/EvoScientist/usage/identity.py new file mode 100644 index 0000000..55dee84 --- /dev/null +++ b/EvoScientist/usage/identity.py @@ -0,0 +1,122 @@ +"""Stable deployment/workspace identity and launcher environment setup.""" + +from __future__ import annotations + +import hashlib +import ntpath +import os +import secrets +import time +import unicodedata +import uuid +from collections.abc import Callable +from pathlib import Path + + +def _fsync_directory(path: Path) -> None: + try: + descriptor = os.open(path, os.O_RDONLY) + except OSError: + return + try: + os.fsync(descriptor) + except OSError: + pass + finally: + os.close(descriptor) + + +def _load_or_create(path: Path, factory: Callable[[], str]) -> str: + path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + value = factory() + try: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError: + # Another launcher may have won O_EXCL but not completed its fsync yet. + for _ in range(100): + existing = path.read_text(encoding="utf-8").strip() + if existing: + return existing + time.sleep(0.01) + raise ValueError(f"Identity file is empty: {path}") from None + try: + os.write(descriptor, f"{value}\n".encode()) + os.fsync(descriptor) + finally: + os.close(descriptor) + _fsync_directory(path.parent) + return value + + +def resolve_data_dir() -> Path: + configured = os.getenv("EVOSCIENTIST_DATA_DIR", "").strip() + if configured: + path = Path(configured) + if not path.is_absolute(): + raise ValueError("EVOSCIENTIST_DATA_DIR must be absolute") + return path + return Path.home() / ".evoscientist" + + +def normalize_workspace_path_v1( + resolved_path: str, *, windows: bool | None = None +) -> str: + """Normalize an already-real path using the frozen ws1 cross-platform rules.""" + + use_windows = os.name == "nt" if windows is None else windows + normalized = unicodedata.normalize("NFC", resolved_path) + if use_windows: + windows_path = ntpath.normcase(normalized) + _drive, tail = ntpath.splitdrive(windows_path) + normalized = windows_path.replace("\\", "/") + if tail in {"\\", "/"}: + return normalized.rstrip("/") + "/" + return normalized.rstrip("/") + if normalized == "/": + return normalized + return normalized.rstrip("/") + + +def workspace_identity_from_normalized(deployment_id: str, normalized_path: str) -> str: + """Hash a normalized path; split out for cross-language contract fixtures.""" + + digest = hashlib.sha256(f"{deployment_id}\0{normalized_path}".encode()).hexdigest() + return f"ws1_{digest}" + + +def workspace_identity(deployment_id: str, workspace_dir: str | Path) -> str: + resolved = os.path.realpath(os.path.expanduser(str(workspace_dir)), strict=True) + normalized = normalize_workspace_path_v1(resolved) + return workspace_identity_from_normalized(deployment_id, normalized) + + +def prepare_usage_environment( + workspace_dir: str | Path, *, webui_port: int +) -> dict[str, str]: + """Create stable secrets/identities and return the shared child environment.""" + + if not 1 <= webui_port <= 65535: + raise ValueError("webui_port must be an integer in [1, 65535]") + + data_dir = resolve_data_dir().expanduser().resolve() + data_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + deployment_id = _load_or_create( + data_dir / "deployment-id", lambda: str(uuid.uuid4()) + ) + # Reject corrupt identity files early instead of silently creating a second identity. + deployment_id = str(uuid.UUID(deployment_id)) + sink_token = _load_or_create( + data_dir / "usage-sink-token", lambda: secrets.token_urlsafe(32) + ) + workspace_id = workspace_identity(deployment_id, workspace_dir) + return { + "EVOSCIENTIST_DATA_DIR": str(data_dir), + "EVOSCIENTIST_USAGE_TRACKING": "true", + "EVOSCIENTIST_USAGE_SINK_URL": ( + f"http://127.0.0.1:{webui_port}/api/usage/events" + ), + "EVOSCIENTIST_USAGE_SINK_TOKEN": sink_token, + "EVOSCIENTIST_DEPLOYMENT_ID": deployment_id, + "EVOSCIENTIST_WORKSPACE_ID": workspace_id, + "EVOSCIENTIST_USAGE_SPOOL_DIR": str(data_dir / "usage-spool"), + } diff --git a/EvoScientist/usage/schema.py b/EvoScientist/usage/schema.py new file mode 100644 index 0000000..5684e3c --- /dev/null +++ b/EvoScientist/usage/schema.py @@ -0,0 +1,170 @@ +"""UsageEvent v1 model shared by the callback and durable spool.""" + +from __future__ import annotations + +import re +from datetime import datetime +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +MAX_SAFE_TOKEN_INTEGER = 9_007_199_254_740_991 +_UTC_TIMESTAMP = re.compile(r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$") +UsageScope = Literal[ + "main", + "sync_subagent", + "async_subagent", + "tool_selector", + "summarizer", + "memory", + "scheduler", + "autoskills", + "diagnostic", + "skill_eval", + "unattributed", +] + + +class UsageEventV1(BaseModel): + """Validated terminal observation for exactly one LangChain model run.""" + + model_config = ConfigDict(extra="forbid") + + schema_version: Literal[1] + event_id: str = Field(min_length=1, max_length=1024) + event_type: Literal["usage_observed"] + source: Literal["callback_final"] + authority_class: Literal["observed_final"] + revision: Literal[1] + + deployment_id: str = Field(min_length=1, max_length=256) + workspace_id: str = Field(min_length=1, max_length=256) + model_call_id: str = Field(min_length=1, max_length=256) + parent_run_id: str | None = Field(min_length=1, max_length=256) + provider_request_id: str | None = Field(min_length=1, max_length=256) + thread_id: str | None = Field(min_length=1, max_length=256) + source_session_id: str | None = Field(min_length=1, max_length=256) + turn_id: str | None = Field(min_length=1, max_length=256) + workspace_dir: str | None = Field(min_length=1, max_length=4096) + scope: UsageScope + source_agent: str | None = Field(min_length=1, max_length=256) + + provider_profile_id: str = Field(min_length=1, max_length=512) + provider_revision: str | None = Field(min_length=1, max_length=256) + provider_adapter: str = Field(min_length=1, max_length=512) + model_alias: str = Field(min_length=1, max_length=512) + upstream_model_id: str = Field(min_length=1, max_length=512) + + usage_status: Literal["confirmed", "unknown"] + input_tokens: int | None = Field(ge=0, le=MAX_SAFE_TOKEN_INTEGER) + output_tokens: int | None = Field(ge=0, le=MAX_SAFE_TOKEN_INTEGER) + provider_total_tokens: int | None = Field(ge=0, le=MAX_SAFE_TOKEN_INTEGER) + input_token_details: dict[str, int] + output_token_details: dict[str, int] + + started_at: datetime | None + observed_at: datetime + completed_at: datetime + + @model_validator(mode="before") + @classmethod + def validate_json_types(cls, value: object) -> object: + if not isinstance(value, dict): + return value + integer_fields = ( + "schema_version", + "revision", + "input_tokens", + "output_tokens", + "provider_total_tokens", + ) + for name in integer_fields: + item = value.get(name) + if item is not None and ( + not isinstance(item, int) or isinstance(item, bool) + ): + raise ValueError(f"{name} must be an integer or null") + for name in ("input_token_details", "output_token_details"): + details = value.get(name) + if isinstance(details, dict) and any( + not isinstance(item, int) or isinstance(item, bool) + for item in details.values() + ): + raise ValueError(f"{name} values must be integers") + for name in ("started_at", "observed_at", "completed_at"): + timestamp = value.get(name) + if timestamp is not None and not isinstance(timestamp, (str, datetime)): + raise ValueError(f"{name} must be an RFC 3339 UTC timestamp") + if isinstance(timestamp, str) and not _UTC_TIMESTAMP.fullmatch(timestamp): + raise ValueError(f"{name} must be an RFC 3339 UTC timestamp") + return value + + @model_validator(mode="after") + def validate_contract(self) -> UsageEventV1: + expected = f"{self.deployment_id}:{self.model_call_id}:callback_final:1" + if self.event_id != expected: + raise ValueError("event_id does not match the v1 identity") + if self.usage_status == "unknown": + if any( + value is not None + for value in ( + self.input_tokens, + self.output_tokens, + self.provider_total_tokens, + ) + ): + raise ValueError("unknown usage must have null token fields") + else: + if self.input_tokens is None or self.output_tokens is None: + raise ValueError("confirmed usage requires input and output tokens") + if self.input_tokens + self.output_tokens > MAX_SAFE_TOKEN_INTEGER: + raise ValueError("input plus output tokens exceeds safe integer") + for details in (self.input_token_details, self.output_token_details): + if any( + not key + or len(key) > 128 + or any( + ord(character) <= 31 or ord(character) == 127 for character in key + ) + or value < 0 + or value > MAX_SAFE_TOKEN_INTEGER + for key, value in details.items() + ): + raise ValueError("invalid token details") + for name in ( + "event_id", + "deployment_id", + "workspace_id", + "model_call_id", + "parent_run_id", + "provider_request_id", + "thread_id", + "source_session_id", + "turn_id", + "workspace_dir", + "source_agent", + "provider_profile_id", + "provider_revision", + "provider_adapter", + "model_alias", + "upstream_model_id", + ): + text = getattr(self, name) + if text is not None and any( + ord(character) <= 31 or ord(character) == 127 for character in text + ): + raise ValueError(f"{name} contains a control character") + for timestamp in (self.started_at, self.observed_at, self.completed_at): + if timestamp is not None: + offset = timestamp.utcoffset() + if ( + timestamp.tzinfo is None + or offset is None + or offset.total_seconds() != 0 + ): + raise ValueError("usage timestamps must be UTC") + if self.started_at and ( + self.started_at > self.observed_at or self.started_at > self.completed_at + ): + raise ValueError("started_at cannot follow terminal timestamps") + return self diff --git a/EvoScientist/usage/spool.py b/EvoScientist/usage/spool.py new file mode 100644 index 0000000..8c0d5e5 --- /dev/null +++ b/EvoScientist/usage/spool.py @@ -0,0 +1,540 @@ +"""Local durable outbox and background sender for UsageEvent v1.""" + +from __future__ import annotations + +import atexit +import hashlib +import json +import logging +import os +import random +import threading +import time +import uuid +from datetime import UTC, datetime +from pathlib import Path +from urllib.parse import urlsplit, urlunsplit + +import httpx + +from .schema import UsageEventV1 + +logger = logging.getLogger(__name__) + + +def _now() -> str: + return datetime.now(UTC).isoformat().replace("+00:00", "Z") + + +def _int_env(name: str, default: int, minimum: int = 1) -> int: + try: + return max(minimum, int(os.getenv(name, str(default)))) + except ValueError: + return default + + +def _float_env(name: str, default: float) -> float: + try: + return max(0.05, float(os.getenv(name, str(default)))) + except ValueError: + return default + + +def _fsync_dir(path: Path) -> None: + try: + descriptor = os.open(path, os.O_RDONLY) + except OSError: + return + try: + os.fsync(descriptor) + except OSError: + pass + finally: + os.close(descriptor) + + +def _endpoint(sink_url: str, suffix: str) -> str: + parts = urlsplit(sink_url) + base = parts.path.removesuffix("/api/usage/events") + return urlunsplit((parts.scheme, parts.netloc, f"{base}{suffix}", "", "")) + + +class UsageSpool: + """Synchronous durable enqueue with an asynchronous at-least-once sender.""" + + def __init__(self) -> None: + self.sink_url = os.environ["EVOSCIENTIST_USAGE_SINK_URL"] + self.token = os.environ["EVOSCIENTIST_USAGE_SINK_TOKEN"] + self.deployment_id = os.environ["EVOSCIENTIST_DEPLOYMENT_ID"] + self.workspace_id = os.environ["EVOSCIENTIST_WORKSPACE_ID"] + self.root = Path(os.environ["EVOSCIENTIST_USAGE_SPOOL_DIR"]) + if not self.root.is_absolute(): + raise ValueError("EVOSCIENTIST_USAGE_SPOOL_DIR must be absolute") + self.tmp = self.root / "tmp" + self.pending = self.root / "pending" + self.inflight = self.root / "inflight" + self.quarantine = self.root / "quarantine" + for directory in (self.tmp, self.pending, self.inflight, self.quarantine): + directory.mkdir(parents=True, exist_ok=True, mode=0o700) + self.status_path = self.root / "status.json" + self.max_files = _int_env("EVOSCIENTIST_USAGE_SPOOL_MAX_FILES", 100_000) + self.max_bytes = _int_env("EVOSCIENTIST_USAGE_SPOOL_MAX_BYTES", 1_073_741_824) + self.max_event_bytes = _int_env("EVOSCIENTIST_USAGE_MAX_EVENT_BYTES", 262_144) + self.lease_seconds = _int_env("EVOSCIENTIST_USAGE_INFLIGHT_LEASE_SECONDS", 120) + self.heartbeat_interval = _float_env( + "EVOSCIENTIST_USAGE_HEARTBEAT_INTERVAL_SECONDS", 15 + ) + self.retry_initial = _float_env("EVOSCIENTIST_USAGE_RETRY_INITIAL_SECONDS", 1) + self.retry_max = _float_env("EVOSCIENTIST_USAGE_RETRY_MAX_SECONDS", 60) + self.connect_timeout = _float_env( + "EVOSCIENTIST_USAGE_HTTP_CONNECT_TIMEOUT_SECONDS", 1 + ) + self.request_timeout = _float_env("EVOSCIENTIST_USAGE_HTTP_TIMEOUT_SECONDS", 3) + self.unsupported_reprobe = _float_env( + "EVOSCIENTIST_USAGE_UNSUPPORTED_REPROBE_SECONDS", 300 + ) + self.schema_reprobe = _float_env( + "EVOSCIENTIST_USAGE_SCHEMA_REPROBE_SECONDS", 60 + ) + self.quarantine_retention_days = _int_env( + "EVOSCIENTIST_USAGE_QUARANTINE_RETENTION_DAYS", 90 + ) + self._state_lock = threading.Lock() + self._size_lock = threading.Lock() + self._cached_files = 0 + self._cached_bytes = 0 + self._size_cache_at = 0.0 + self.first_loss_at: str | None = None + self.degraded_reason: str | None = None + self.last_error_code: str | None = None + self._load_status() + self._clean_expired_quarantine() + self._clean_stale_local_artifacts() + self._last_cleanup_at = time.monotonic() + self._stop = threading.Event() + self._wake = threading.Event() + self._thread = threading.Thread( + target=self._run, name="evoscientist-usage-sender", daemon=True + ) + self._thread.start() + + def _load_status(self) -> None: + try: + data = json.loads(self.status_path.read_text(encoding="utf-8")) + self.first_loss_at = data.get("first_loss_at") + self.degraded_reason = data.get("tracking_degraded_reason") + except (OSError, ValueError, TypeError): + pass + + def _persist_status(self) -> None: + temporary = self.root / f"status.{os.getpid()}.{uuid.uuid4().hex}.tmp" + payload = json.dumps( + { + "first_loss_at": self.first_loss_at, + "tracking_degraded_reason": self.degraded_reason, + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + try: + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + os.write(descriptor, payload) + os.fsync(descriptor) + finally: + os.close(descriptor) + os.replace(temporary, self.status_path) + _fsync_dir(self.root) + except OSError: + logger.exception("Could not persist usage tracking degraded status") + try: + temporary.unlink(missing_ok=True) + except OSError: + pass + + def mark_degraded(self, reason: str) -> None: + with self._state_lock: + if self.first_loss_at is None: + self.first_loss_at = _now() + if self.degraded_reason is None: + self.degraded_reason = reason[:512] + self._persist_status() + self._wake.set() + + def _spool_size(self) -> tuple[int, int]: + with self._size_lock: + if time.monotonic() - self._size_cache_at < 1: + return self._cached_files, self._cached_bytes + count = 0 + size = 0 + for directory in (self.pending, self.inflight, self.quarantine): + try: + for item in os.scandir(directory): + if item.is_file() and item.name.endswith(".json"): + count += 1 + try: + size += item.stat().st_size + except OSError: + pass + except OSError: + continue + with self._size_lock: + self._cached_files = count + self._cached_bytes = size + self._size_cache_at = time.monotonic() + return count, size + + def _adjust_spool_size(self, count: int, size: int) -> None: + with self._size_lock: + self._cached_files = max(0, self._cached_files + count) + self._cached_bytes = max(0, self._cached_bytes + size) + + def enqueue(self, event: UsageEventV1) -> None: + try: + payload = json.dumps( + event.model_dump(mode="json"), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + if len(payload) > self.max_event_bytes: + self.mark_degraded("event_too_large") + logger.error("Usage event %s exceeds spool event limit", event.event_id) + return + count, size = self._spool_size() + if count >= self.max_files or size + len(payload) > self.max_bytes: + self.mark_degraded("spool_soft_limit_reached") + logger.error("Usage spool soft limit reached; event was not persisted") + return + key = hashlib.sha256(event.event_id.encode()).hexdigest() + temporary = self.tmp / f"{key}.{os.getpid()}.{uuid.uuid4().hex}.tmp" + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + os.write(descriptor, payload) + os.fsync(descriptor) + finally: + os.close(descriptor) + lock_path = self.root / f"{key}.lock" + deadline = time.monotonic() + 0.05 + while True: + try: + lock_fd = os.open( + lock_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600 + ) + break + except FileExistsError: + if time.monotonic() < deadline: + time.sleep(0.001) + continue + os.replace( + temporary, + self.quarantine + / f"{key}.lock-contention.{uuid.uuid4().hex}.json", + ) + self._adjust_spool_size(1, len(payload)) + _fsync_dir(self.quarantine) + logger.error( + "Usage event %s quarantined after spool lock contention", + event.event_id, + ) + self.mark_degraded("spool_lock_contention") + self._wake.set() + return + try: + target = self.pending / f"{key}.json" + existing = target if target.exists() else self.inflight / target.name + if existing.exists(): + if existing.read_bytes() != payload: + os.replace( + temporary, + self.quarantine / f"{key}.conflict.{uuid.uuid4().hex}.json", + ) + self._adjust_spool_size(1, len(payload)) + _fsync_dir(self.quarantine) + self.mark_degraded("local_event_payload_conflict") + else: + temporary.unlink(missing_ok=True) + else: + os.replace(temporary, target) + self._adjust_spool_size(1, len(payload)) + _fsync_dir(self.pending) + finally: + os.close(lock_fd) + lock_path.unlink(missing_ok=True) + except Exception: + self.mark_degraded("spool_write_failed") + logger.exception("Usage capture could not persist a terminal event") + finally: + self._wake.set() + + def _recover_stale_inflight(self) -> None: + cutoff = time.time() - self.lease_seconds + try: + items = list(self.inflight.glob("*.json")) + except OSError: + return + for item in items: + try: + if item.stat().st_mtime < cutoff: + os.replace(item, self.pending / item.name) + _fsync_dir(self.inflight) + _fsync_dir(self.pending) + except (FileNotFoundError, OSError): + continue + + def _clean_expired_quarantine(self) -> None: + cutoff = time.time() - self.quarantine_retention_days * 86_400 + removed = 0 + for item in self.quarantine.glob("*.json"): + try: + if item.stat().st_mtime < cutoff: + item.unlink() + removed += 1 + except (FileNotFoundError, OSError): + continue + if removed: + logger.warning("Removed %d expired usage quarantine events", removed) + _fsync_dir(self.quarantine) + + def _clean_stale_local_artifacts(self) -> None: + cutoff = time.time() - self.lease_seconds + candidates = [*self.tmp.glob("*.tmp"), *self.root.glob("*.lock")] + for item in candidates: + try: + if item.stat().st_mtime < cutoff: + item.unlink() + except (FileNotFoundError, OSError): + continue + + def _counts(self) -> tuple[int, int, int, int]: + def files(directory: Path) -> list[Path]: + try: + return list(directory.glob("*.json")) + except OSError: + return [] + + pending = files(self.pending) + inflight = files(self.inflight) + quarantine = files(self.quarantine) + size = 0 + for item in pending + inflight + quarantine: + try: + size += item.stat().st_size + except OSError: + pass + return len(pending), len(inflight), len(quarantine), size + + def _heartbeat(self, client: httpx.Client) -> None: + pending, inflight, quarantine, size = self._counts() + with self._state_lock: + body = { + "deployment_id": self.deployment_id, + "workspace_id": self.workspace_id, + "emitter_version": "2.0", + "schema_version": 1, + "sender_status": ( + "degraded" + if self.first_loss_at or self.degraded_reason + else "healthy" + ), + "spool_pending": pending, + "spool_inflight": inflight, + "spool_quarantined": quarantine, + "spool_bytes": size, + "first_loss_at": self.first_loss_at, + "tracking_degraded_reason": self.degraded_reason, + "last_error_code": self.last_error_code, + "sent_at": _now(), + } + try: + response = client.post( + _endpoint(self.sink_url, "/api/usage/sources/heartbeat"), json=body + ) + if response.status_code >= 400: + self.last_error_code = f"heartbeat_http_{response.status_code}" + except httpx.HTTPError: + self.last_error_code = "heartbeat_unreachable" + + def _capable(self, client: httpx.Client) -> bool: + try: + response = client.get(_endpoint(self.sink_url, "/api/usage/capabilities")) + if response.status_code != 200: + self.last_error_code = f"capabilities_http_{response.status_code}" + return False + versions = response.json().get("supported_schema_versions", []) + if 1 not in versions: + self.last_error_code = "schema_incompatible" + return False + self.last_error_code = None + return True + except (httpx.HTTPError, ValueError, TypeError): + self.last_error_code = "collector_unreachable" + return False + + def _send_one(self, client: httpx.Client) -> bool: + pending: Path | None = None + lock_fd: int | None = None + lock_path: Path | None = None + try: + candidates = self.pending.glob("*.json") + for candidate in candidates: + candidate_lock = self.root / f"{candidate.stem}.lock" + try: + descriptor = os.open( + candidate_lock, + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0o600, + ) + except FileExistsError: + continue + pending = candidate + lock_fd = descriptor + lock_path = candidate_lock + break + except OSError: + return False + if pending is None: + return False + inflight = self.inflight / pending.name + try: + os.replace(pending, inflight) + _fsync_dir(self.pending) + _fsync_dir(self.inflight) + except (FileNotFoundError, OSError): + return True + finally: + if lock_fd is not None: + os.close(lock_fd) + if lock_path is not None: + lock_path.unlink(missing_ok=True) + try: + response = client.post( + self.sink_url, + content=inflight.read_bytes(), + headers={"Content-Type": "application/json"}, + ) + if response.status_code == 200: + status = response.json().get("status") + if status in {"accepted", "duplicate"}: + event_size = inflight.stat().st_size + inflight.unlink(missing_ok=True) + self._adjust_spool_size(-1, -event_size) + _fsync_dir(self.inflight) + self.last_error_code = None + return True + if response.status_code in {400, 409, 413, 422}: + os.replace(inflight, self.quarantine / inflight.name) + _fsync_dir(self.inflight) + _fsync_dir(self.quarantine) + self.mark_degraded(f"collector_rejected_event_{response.status_code}") + self.last_error_code = f"event_http_{response.status_code}" + return True + self.last_error_code = f"event_http_{response.status_code}" + except (httpx.HTTPError, OSError, ValueError, TypeError): + self.last_error_code = "event_send_failed" + try: + os.replace(inflight, self.pending / inflight.name) + _fsync_dir(self.inflight) + _fsync_dir(self.pending) + except (FileNotFoundError, OSError): + pass + return False + + def _run(self) -> None: + headers = {"Authorization": f"Bearer {self.token}"} + timeout = httpx.Timeout(self.request_timeout, connect=self.connect_timeout) + retry = self.retry_initial + last_heartbeat = 0.0 + collector_capable = False + next_probe = 0.0 + with httpx.Client(headers=headers, timeout=timeout) as client: + while not self._stop.is_set(): + self._recover_stale_inflight() + now = time.monotonic() + if now - self._last_cleanup_at >= 3_600: + self._clean_expired_quarantine() + self._clean_stale_local_artifacts() + self._last_cleanup_at = now + if now - last_heartbeat >= self.heartbeat_interval: + self._heartbeat(client) + last_heartbeat = now + if not collector_capable and now >= next_probe: + collector_capable = self._capable(client) + if not collector_capable: + if self.last_error_code == "capabilities_http_404": + probe_delay = self.unsupported_reprobe + elif self.last_error_code == "schema_incompatible": + probe_delay = self.schema_reprobe + else: + probe_delay = min(self.retry_max, retry) + next_probe = time.monotonic() + probe_delay + if not collector_capable: + wait_for = max(0.05, next_probe - time.monotonic()) + self._wake.wait(wait_for) + self._wake.clear() + retry = min(self.retry_max, retry * 2) + continue + progressed = self._send_one(client) + if progressed: + retry = self.retry_initial + continue + if self.last_error_code in { + "event_http_401", + "event_http_403", + "event_http_404", + "event_http_426", + }: + collector_capable = False + next_probe = time.monotonic() + ( + self.unsupported_reprobe + if self.last_error_code == "event_http_404" + else self.schema_reprobe + ) + self._wake.wait( + self.heartbeat_interval + if not any(self.pending.glob("*.json")) + else retry * random.uniform(0.8, 1.2) + ) + self._wake.clear() + retry = min(self.retry_max, retry * 2) + + def close(self) -> None: + self._stop.set() + self._wake.set() + if self._thread.is_alive(): + self._thread.join(timeout=1) + + +_singleton_lock = threading.Lock() +_singleton: tuple[int, UsageSpool] | None = None + + +def get_usage_spool() -> UsageSpool: + global _singleton + pid = os.getpid() + with _singleton_lock: + if _singleton is None or _singleton[0] != pid: + _singleton = (pid, UsageSpool()) + return _singleton[1] + + +def mark_tracking_degraded(reason: str) -> None: + if os.getenv("EVOSCIENTIST_USAGE_TRACKING", "").strip().lower() not in { + "1", + "true", + "yes", + "on", + }: + return + try: + get_usage_spool().mark_degraded(reason) + except Exception: + logger.exception("Could not mark usage tracking as degraded: %s", reason) + + +def _close_singleton() -> None: + if _singleton is not None: + _singleton[1].close() + + +atexit.register(_close_singleton) diff --git a/EvoScientist/workspace_cutover.py b/EvoScientist/workspace_cutover.py new file mode 100644 index 0000000..8a46699 --- /dev/null +++ b/EvoScientist/workspace_cutover.py @@ -0,0 +1,417 @@ +"""Administrative migration from a shared workspace to conversation scopes.""" + +from __future__ import annotations + +import hashlib +import json +import os +import uuid +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from .scope_registry import ( + ScopeRegistry, + deployment_id_for_workspace, + get_scope_registry, +) +from .workspace_scope import provision_conversation_scope, workspace_metadata + +_TERMINAL_RUN_STATUSES = frozenset( + {"success", "error", "timeout", "cancelled", "interrupted"} +) +_VALID_SCOPE_STATES = frozenset({"draft", "active"}) + + +def _now() -> str: + return datetime.now(UTC).isoformat() + + +def _write_report(workspace_root: Path, report: dict[str, Any]) -> Path: + reports = workspace_root / ".evoscientist" / "control" / "cutover-reports" + reports.mkdir(mode=0o700, parents=True, exist_ok=True) + body = dict(report) + digest = hashlib.sha256( + json.dumps(body, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + report["sha256"] = digest + encoded = json.dumps(report, sort_keys=True, indent=2).encode("utf-8") + b"\n" + target = reports / f"{report['operation_id']}.json" + for destination in (target, reports / "latest.json"): + temporary = destination.with_suffix(".json.tmp") + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "wb") as file: + file.write(encoded) + os.replace(temporary, destination) + return target + + +def _active_run_ids(client: Any, thread_id: str) -> list[str]: + return [ + str(run.get("run_id")) + for run in client.runs.list(thread_id=thread_id, limit=1000) + if str(run.get("status")) not in _TERMINAL_RUN_STATUSES + ] + + +def _scope_owner_error( + registry: ScopeRegistry, + *, + deployment_id: str, + resource_id: str, + metadata: dict[str, Any], + allowed_owner_types: frozenset[str] | None = None, +) -> str | None: + """Return a reason unless metadata names a live Registry-owned resource.""" + + scope_id = metadata.get("workspace_scope_id") + if not isinstance(scope_id, str) or not scope_id: + return "missing workspace_scope_id" + try: + scope = registry.get(deployment_id, scope_id) + owner = registry.get_owner_by_resource(deployment_id, resource_id) + except Exception as exc: + return str(exc) + if scope.state not in _VALID_SCOPE_STATES: + return f"workspace scope is {scope.state}" + if owner.scope_id != scope.scope_id: + return "resource owner belongs to another workspace scope" + if owner.state != "active": + return f"resource owner is {owner.state}" + if owner.owner_type in {"primary_thread", "primary_run"}: + return f"unexpected owner type {owner.owner_type}" + if allowed_owner_types is not None and owner.owner_type not in allowed_owner_types: + return f"unexpected owner type {owner.owner_type}" + metadata_deployment = metadata.get("workspace_deployment_id") + if ( + metadata_deployment is not None + and metadata_deployment != "" + and metadata_deployment != deployment_id + ): + return "metadata deployment does not match the active deployment" + metadata_owner = metadata.get("workspace_scope_owner_id") + if ( + metadata_owner is not None + and metadata_owner != "" + and metadata_owner != owner.owner_id + ): + return "metadata owner does not match the resource owner" + return None + + +def _quarantine_derived_thread( + client: Any, + *, + thread_id: str, + metadata: dict[str, Any], + operation_id: str, + reason: str, + report: dict[str, Any], +) -> None: + """Interrupt a non-primary thread and remove any untrusted scope claims.""" + + try: + active_runs = _active_run_ids(client, thread_id) + for run_id in active_runs: + client.runs.cancel(thread_id, run_id, wait=True, action="interrupt") + remaining_active = _active_run_ids(client, thread_id) + if remaining_active: + report["active_runs"].append( + {"thread_id": thread_id, "run_ids": remaining_active} + ) + report["unmanaged_derived_threads"].append(thread_id) + return + cleaned_metadata = { + key: value + for key, value in metadata.items() + if not key.startswith("workspace_") + } + client.threads.update( + thread_id, + metadata={ + **cleaned_metadata, + "workspace_quarantine": { + "operation_id": operation_id, + "reason": reason, + "quarantined_at": _now(), + }, + }, + ) + report["quarantined_derived_threads"].append( + { + "thread_id": thread_id, + "cancelled_run_ids": active_runs, + "reason": reason, + } + ) + except Exception as exc: + report["unmanaged_derived_threads"].append(thread_id) + report["quarantine_failures"].append( + {"thread_id": thread_id, "error": str(exc)} + ) + + +def verify_required_cutover(workspace_root: Path) -> None: + """Reject strict mode unless the current deployment has a passing report.""" + + latest = ( + workspace_root / ".evoscientist" / "control" / "cutover-reports" / "latest.json" + ) + try: + report = json.loads(latest.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise RuntimeError( + "required workspace isolation needs a passing workspace-cutover report" + ) from exc + digest = report.pop("sha256", None) + expected = hashlib.sha256( + json.dumps(report, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + if ( + digest != expected + or report.get("status") != "passed" + or report.get("browser_sdk_gate") is not True + or report.get("deployment_id") != deployment_id_for_workspace(workspace_root) + ): + raise RuntimeError("workspace-cutover report is missing, stale, or failed") + operation = get_scope_registry(workspace_root).get_operation( + str(report["deployment_id"]), str(report["operation_id"]) + ) + if ( + operation.kind != "workspace-cutover" + or operation.state != "completed" + or operation.result_sha256 != digest + ): + raise RuntimeError("workspace-cutover registry operation does not match report") + + +def run_workspace_cutover( + *, + workspace_root: Path, + client: Any, + assistant_id: str = "EvoScientist", + browser_sdk_gate: bool = False, +) -> dict[str, Any]: + """Provision primary threads and quarantine legacy unscoped work.""" + + workspace_root = workspace_root.expanduser().resolve() + deployment_id = deployment_id_for_workspace(workspace_root) + registry: ScopeRegistry = get_scope_registry(workspace_root) + operation_id = str(uuid.uuid4()) + registry.acquire_lock( + deployment_id, "workspace-lifecycle", operation_id, lease_seconds=300 + ) + try: + registry.acquire_lock( + deployment_id, "workspace-cutover", operation_id, lease_seconds=300 + ) + except Exception: + registry.release_lock(deployment_id, "workspace-lifecycle", operation_id) + raise + registry.begin_operation(deployment_id, operation_id, kind="workspace-cutover") + report: dict[str, Any] = { + "operation_id": operation_id, + "deployment_id": deployment_id, + "started_at": _now(), + "primary_threads": 0, + "scopes": 0, + "quarantined_crons": 0, + "quarantined_cron_records": [], + "quarantined_derived_threads": [], + "validated_derived_threads": [], + "validated_crons": [], + "invalid_scoped_derived_threads": [], + "invalid_scoped_crons": [], + "active_runs": [], + "unmanaged_derived_threads": [], + "quarantine_failures": [], + "metadata_mismatches": [], + "browser_sdk_gate": browser_sdk_gate, + "errors": [], + "status": "failed", + } + try: + threads: list[dict[str, Any]] = [] + offset = 0 + while True: + registry.renew_lock( + deployment_id, "workspace-cutover", operation_id, lease_seconds=300 + ) + registry.renew_lock( + deployment_id, "workspace-lifecycle", operation_id, lease_seconds=300 + ) + page = list(client.threads.search(limit=100, offset=offset)) + threads.extend(page) + if len(page) < 100: + break + offset += len(page) + for thread in threads: + registry.renew_lock( + deployment_id, "workspace-cutover", operation_id, lease_seconds=300 + ) + registry.renew_lock( + deployment_id, "workspace-lifecycle", operation_id, lease_seconds=300 + ) + metadata = dict(thread.get("metadata") or {}) + thread_id = str(thread.get("thread_id") or "") + is_primary = ( + metadata.get("graph_id") == assistant_id + or metadata.get("agent_name") == assistant_id + ) + if is_primary: + if not thread_id: + continue + active_runs = _active_run_ids(client, thread_id) + if active_runs: + report["active_runs"].append( + {"thread_id": thread_id, "run_ids": active_runs} + ) + continue + record = provision_conversation_scope( + thread_id, + deployment_id=deployment_id, + workspace_root=workspace_root, + lock_operation_id=operation_id, + ) + updated = client.threads.update( + thread_id, metadata={**metadata, **workspace_metadata(record)} + ) + if isinstance(updated, dict): + updated_metadata = dict(updated.get("metadata") or {}) + if updated_metadata.get("workspace_scope_id") != record.scope_id: + report["metadata_mismatches"].append(thread_id) + report["primary_threads"] += 1 + report["scopes"] += 1 + elif thread_id: + scope_id = metadata.get("workspace_scope_id") + if scope_id is None: + _quarantine_derived_thread( + client, + thread_id=thread_id, + metadata=metadata, + operation_id=operation_id, + reason="unscoped-derived-thread", + report=report, + ) + continue + ownership_error = _scope_owner_error( + registry, + deployment_id=deployment_id, + resource_id=thread_id, + metadata=metadata, + ) + if ownership_error is not None: + report["invalid_scoped_derived_threads"].append( + {"thread_id": thread_id, "error": ownership_error} + ) + _quarantine_derived_thread( + client, + thread_id=thread_id, + metadata=metadata, + operation_id=operation_id, + reason="invalid-scoped-derived-thread", + report=report, + ) + continue + report["validated_derived_threads"].append(thread_id) + active_runs = _active_run_ids(client, thread_id) + if active_runs: + report["active_runs"].append( + {"thread_id": thread_id, "run_ids": active_runs} + ) + for cron in client.crons.search(limit=1000): + registry.renew_lock( + deployment_id, "workspace-cutover", operation_id, lease_seconds=300 + ) + registry.renew_lock( + deployment_id, "workspace-lifecycle", operation_id, lease_seconds=300 + ) + metadata = dict(cron.get("metadata") or {}) + cron_id = str(cron.get("cron_id") or "") + scope_id = metadata.get("workspace_scope_id") + if scope_id is not None: + ownership_error = _scope_owner_error( + registry, + deployment_id=deployment_id, + resource_id=cron_id, + metadata=metadata, + allowed_owner_types=frozenset({"schedule"}), + ) + if ownership_error is None: + report["validated_crons"].append(cron_id) + continue + client.crons.update(cron_id, enabled=False) + report["quarantined_crons"] += 1 + report["invalid_scoped_crons"].append( + {"cron_id": cron_id, "error": ownership_error} + ) + report["quarantined_cron_records"].append( + {"cron_id": cron_id, "reason": "invalid-scoped-cron"} + ) + continue + if metadata.get("run_kind") != "scheduled_task": + continue + client.crons.update(cron_id, enabled=False) + report["quarantined_crons"] += 1 + report["quarantined_cron_records"].append( + {"cron_id": cron_id, "reason": "unscoped-scheduled-task"} + ) + report["status"] = ( + "passed" + if ( + report["browser_sdk_gate"] + and not report["unmanaged_derived_threads"] + and not report["quarantine_failures"] + and not report["active_runs"] + and not report["metadata_mismatches"] + ) + else "failed" + ) + report["completed_at"] = _now() + report["report_path"] = str( + workspace_root + / ".evoscientist" + / "control" + / "cutover-reports" + / f"{operation_id}.json" + ) + _write_report(workspace_root, report) + registry.finish_operation( + deployment_id, + operation_id, + state="completed" if report["status"] == "passed" else "failed", + result_sha256=str(report["sha256"]), + last_error_code=None + if report["status"] == "passed" + else "cutover-gates-failed", + ) + return report + except Exception as exc: + report["errors"].append(str(exc)) + report["completed_at"] = _now() + report["report_path"] = str( + workspace_root + / ".evoscientist" + / "control" + / "cutover-reports" + / f"{operation_id}.json" + ) + _write_report(workspace_root, report) + registry.finish_operation( + deployment_id, + operation_id, + state="failed", + result_sha256=str(report["sha256"]), + last_error_code="cutover-exception", + ) + return report + finally: + try: + registry.release_lock(deployment_id, "workspace-cutover", operation_id) + except Exception: + # A lost/expired lease is already recorded as a failed report. + pass + try: + registry.release_lock(deployment_id, "workspace-lifecycle", operation_id) + except Exception: + pass diff --git a/EvoScientist/workspace_maintenance.py b/EvoScientist/workspace_maintenance.py new file mode 100644 index 0000000..f001c2f --- /dev/null +++ b/EvoScientist/workspace_maintenance.py @@ -0,0 +1,168 @@ +"""Idempotent lifecycle maintenance for conversation workspaces.""" + +from __future__ import annotations + +import os +import shutil +import uuid +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +from .scope_registry import ( + ScopeRecord, + deployment_id_for_workspace, + get_scope_registry, +) + +_TERMINAL_RUN_STATES = frozenset( + {"success", "error", "timeout", "cancelled", "interrupted"} +) + + +def _now() -> datetime: + return datetime.now(UTC) + + +def _parse_timestamp(value: str) -> datetime: + parsed = datetime.fromisoformat(value) + return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC) + + +def _trash_root(workspace_root: Path) -> Path: + return workspace_root / ".evoscientist" / "trash" + + +def _conversation_root(workspace_root: Path, scope_id: str) -> Path: + return workspace_root / ".evoscientist" / "conversations" / scope_id + + +def _move_to_trash(workspace_root: Path, scope_id: str, now: datetime) -> None: + source = _conversation_root(workspace_root, scope_id) + target_root = _trash_root(workspace_root) + target_root.mkdir(mode=0o700, parents=True, exist_ok=True) + target = target_root / f"{scope_id}-{int(now.timestamp() * 1000)}" + try: + os.replace(source, target) + os.utime(target, (now.timestamp(), now.timestamp())) + except FileNotFoundError: + return + + +def _purge_trash(workspace_root: Path, cutoff: datetime) -> tuple[int, list[str]]: + root = _trash_root(workspace_root) + if not root.exists(): + return 0, [] + removed = 0 + errors: list[str] = [] + for candidate in root.iterdir(): + try: + if candidate.is_symlink() or not candidate.is_dir(): + continue + modified = datetime.fromtimestamp(candidate.stat().st_mtime, tz=UTC) + if modified >= cutoff: + continue + shutil.rmtree(candidate) + removed += 1 + except OSError as exc: + errors.append(f"trash:{candidate.name}:{exc}") + return removed, errors + + +def _draft_can_be_deleted(scope: ScopeRecord, registry: Any, client: Any) -> bool: + owners = registry.owners(scope.deployment_id, scope.scope_id) + if any(owner.owner_type != "primary_thread" for owner in owners): + return False + try: + runs = list(client.runs.list(thread_id=scope.primary_thread_id, limit=1000)) + if any(str(run.get("status")) not in _TERMINAL_RUN_STATES for run in runs): + return False + state = client.threads.get_state(scope.primary_thread_id) + values = state.get("values") if isinstance(state, dict) else None + messages = values.get("messages") if isinstance(values, dict) else None + return not messages + except Exception: + # Maintenance must not delete a draft whose content cannot be proven empty. + return False + + +def run_workspace_maintenance( + *, + workspace_root: Path, + client: Any, + draft_ttl_hours: int = 24, + trash_retention_days: int = 7, + now: datetime | None = None, +) -> dict[str, Any]: + """Delete safely-drained stale drafts and purge aged trash directories.""" + + if draft_ttl_hours <= 0 or trash_retention_days <= 0: + raise ValueError("workspace maintenance retention values must be positive") + workspace_root = workspace_root.expanduser().resolve() + current = now or _now() + deployment_id = deployment_id_for_workspace(workspace_root) + registry = get_scope_registry(workspace_root) + if registry.active_lock(deployment_id, "workspace-cutover") is not None: + raise RuntimeError("workspace maintenance is blocked by workspace cutover") + operation_id = str(uuid.uuid4()) + registry.acquire_lock( + deployment_id, "workspace-lifecycle", operation_id, lease_seconds=300 + ) + registry.begin_operation(deployment_id, operation_id, kind="workspace-maintenance") + report: dict[str, Any] = { + "operation_id": operation_id, + "deployment_id": deployment_id, + "started_at": current.isoformat(), + "deleted_drafts": [], + "skipped_drafts": [], + "purged_trash": 0, + "errors": [], + } + try: + draft_cutoff = current - timedelta(hours=draft_ttl_hours) + for scope in registry.list_scopes(deployment_id, state="draft"): + registry.renew_lock( + deployment_id, "workspace-lifecycle", operation_id, lease_seconds=300 + ) + if _parse_timestamp(scope.created_at) >= draft_cutoff: + continue + if not _draft_can_be_deleted(scope, registry, client): + report["skipped_drafts"].append(scope.primary_thread_id) + continue + deleting = registry.transition_scope( + deployment_id, + scope.scope_id, + expected_revision=scope.revision, + state="deleting", + ) + _move_to_trash(workspace_root, deleting.scope_id, current) + try: + client.threads.delete(deleting.primary_thread_id) + except Exception as exc: + status = getattr(exc, "status", None) + if status != 404 and "not found" not in str(exc).lower(): + raise + registry.transition_scope( + deployment_id, + deleting.scope_id, + expected_revision=deleting.revision, + state="deleted", + ) + report["deleted_drafts"].append(scope.primary_thread_id) + purged, errors = _purge_trash( + workspace_root, current - timedelta(days=trash_retention_days) + ) + report["purged_trash"] = purged + report["errors"].extend(errors) + except Exception as exc: + report["errors"].append(str(exc)) + finally: + report["completed_at"] = _now().isoformat() + registry.finish_operation( + deployment_id, + operation_id, + state="completed" if not report["errors"] else "failed", + last_error_code=None if not report["errors"] else "maintenance-failed", + ) + registry.release_lock(deployment_id, "workspace-lifecycle", operation_id) + return report diff --git a/EvoScientist/workspace_scope.py b/EvoScientist/workspace_scope.py new file mode 100644 index 0000000..95d7fc9 --- /dev/null +++ b/EvoScientist/workspace_scope.py @@ -0,0 +1,541 @@ +"""Conversation-scoped workspace resolution and DeepAgents backend factory.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import threading +import uuid +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from deepagents.backends.protocol import ( + EditResult, + ExecuteResponse, + FileDownloadResponse, + FileUploadResponse, + GlobResult, + GrepResult, + LsResult, + ReadResult, + SandboxBackendProtocol, + WriteResult, +) +from langchain.tools import ToolRuntime + +from . import paths +from .scope_registry import ( + ScopeAccessError, + ScopeRecord, + deployment_id_for_workspace, + get_scope_registry, +) + +IsolationMode = str + + +def workspace_isolation_mode() -> IsolationMode: + value = os.getenv("EVOSCIENTIST_WORKSPACE_ISOLATION", "optional").strip().lower() + if value not in {"legacy", "optional", "required"}: + raise RuntimeError("EVOSCIENTIST_WORKSPACE_ISOLATION must be legacy, optional or required") + return value + + +def is_required() -> bool: + return workspace_isolation_mode() == "required" + + +def verify_required_executor() -> None: + """Fail startup unless the pinned scope executor is locally usable.""" + + docker = shutil.which("docker") + if not docker: + raise RuntimeError("required workspace isolation needs the docker OCI runtime") + image = os.getenv("EVOSCIENTIST_STRICT_EXECUTOR_IMAGE", "").strip() + if "@sha256:" not in image: + raise RuntimeError("required workspace isolation needs an OCI image pinned by digest") + try: + probe = subprocess.run( + [docker, "image", "inspect", image], + check=False, + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise RuntimeError("required workspace isolation cannot verify the OCI executor") from exc + if probe.returncode != 0: + raise RuntimeError( + f"required workspace isolation needs local OCI image {image!r}" + ) + + +def current_deployment_id() -> str: + return deployment_id_for_workspace(paths.WORKSPACE_ROOT) + + +def conversation_root(scope_id: str, workspace_root: Path | None = None) -> Path: + scope = str(uuid.UUID(scope_id)) + # The deploy process supplies an absolute workspace root. This helper is + # called from synchronous DeepAgents backend factories on the ASGI loop. + root = (workspace_root or paths.WORKSPACE_ROOT).expanduser() + return root / ".evoscientist" / "conversations" / scope + + +def conversation_files_dir(scope_id: str, workspace_root: Path | None = None) -> Path: + return conversation_root(scope_id, workspace_root) / "files" + + +@dataclass(frozen=True, slots=True) +class ScopeContext: + deployment_id: str + scope_id: str + owner_id: str + thread_id: str + revision: int + files_dir: Path + runtime_dir: Path + primary_thread_id: str + + +@dataclass(frozen=True, slots=True) +class _RuntimeScopeConfig: + """Untrusted runtime identifiers parsed without filesystem or Registry I/O.""" + + scope_id: str + owner_id: str + thread_id: str + deployment_id: str | None + + +class ScopedContainerBackend: + """Filesystem backend whose shell commands execute in a scope-only OCI container.""" + + def __init__(self, root_dir: Path, *, timeout: int) -> None: + from .backends import CustomSandboxBackend + + # Reuse the hardened filesystem operations; only ``execute`` is + # replaced so no agent shell runs in the host process. + self._filesystem = CustomSandboxBackend( + root_dir=str(root_dir), virtual_mode=True, timeout=timeout, dangerous=False + ) + self._root_dir = root_dir + self._timeout = timeout + + def __getattr__(self, name: str) -> Any: + return getattr(self._filesystem, name) + + def execute(self, command: str, *, timeout: int | None = None) -> Any: + from .backends import ExecuteResponse, prepare_sandbox_command + + command, error = prepare_sandbox_command( + command, self._filesystem.cwd, virtual_mode=True, dangerous=False + ) + if error: + return ExecuteResponse(output=error, exit_code=1, truncated=False) + image = os.getenv("EVOSCIENTIST_STRICT_EXECUTOR_IMAGE", "").strip() + if "@sha256:" not in image: + return ExecuteResponse( + output="Required workspace isolation needs an OCI image pinned by digest.", + exit_code=125, + truncated=False, + ) + effective_timeout = max(1, min(timeout or self._timeout, 3600)) + invocation = [ + "docker", + "run", + "--rm", + "--network", + "none", + "--read-only", + "--tmpfs", + "/tmp:rw,noexec,nosuid,size=64m", + "--cap-drop", + "ALL", + "--security-opt", + "no-new-privileges", + "--pids-limit", + os.getenv("EVOSCIENTIST_STRICT_EXECUTOR_PIDS", "128"), + "--memory", + os.getenv("EVOSCIENTIST_STRICT_EXECUTOR_MEMORY", "1g"), + "--cpus", + os.getenv("EVOSCIENTIST_STRICT_EXECUTOR_CPUS", "1"), + "--mount", + f"type=bind,src={self._root_dir},dst=/workspace", + "--workdir", + "/workspace", + image, + "sh", + "-lc", + command, + ] + try: + completed = subprocess.run( + invocation, + check=False, + capture_output=True, + text=True, + timeout=effective_timeout, + ) + except FileNotFoundError: + return ExecuteResponse( + output="Required workspace isolation needs an OCI runtime (docker was not found).", + exit_code=127, + truncated=False, + ) + except subprocess.TimeoutExpired as exc: + output = (exc.stdout or "") + (exc.stderr or "") + return ExecuteResponse(output=output, exit_code=124, truncated=False) + output = completed.stdout + completed.stderr + return ExecuteResponse(output=output, exit_code=completed.returncode, truncated=False) + + +def _configurable(runtime: ToolRuntime[Any, Any] | Any | None) -> dict[str, Any]: + """Return the active runnable config, with a non-graph fallback. + + ``ToolRuntime`` deliberately does not expose ``RunnableConfig`` during a + graph execution. LangGraph keeps it in a context variable instead. The + fallback preserves direct callers and unit tests that supply a lightweight + runtime object outside a runnable context. + """ + + config: Any = None + try: + from langgraph.config import get_config + + config = get_config() + except (ImportError, LookupError, RuntimeError): + pass + if not isinstance(config, dict) and runtime is not None: + config = getattr(runtime, "config", None) or {} + if not isinstance(config, dict): + return {} + configurable = config.get("configurable") or {} + return dict(configurable) if isinstance(configurable, dict) else {} + + +def _required_string(configurable: dict[str, Any], key: str) -> str: + value = configurable.get(key) + if not isinstance(value, str) or not value: + raise ScopeAccessError(f"missing {key}") + return value + + +def _runtime_scope_config( + runtime: ToolRuntime[Any, Any] | Any | None, + *, + kind: str, +) -> _RuntimeScopeConfig | None: + """Parse scope identifiers without treating config as an authorization grant.""" + + configurable = _configurable(runtime) + scope_id = configurable.get("workspace_scope_id") + owner_id = configurable.get("workspace_scope_owner_id") + thread_id = configurable.get("thread_id") + + if scope_id is None and owner_id is None: + if workspace_isolation_mode() == "required": + raise ScopeAccessError(f"{kind} requires a workspace scope") + return None + if ( + not isinstance(scope_id, str) + or not isinstance(owner_id, str) + or not isinstance(thread_id, str) + ): + raise ScopeAccessError(f"{kind} has an incomplete workspace scope") + try: + canonical_scope_id = str(uuid.UUID(scope_id)) + canonical_owner_id = str(uuid.UUID(owner_id)) + except ValueError as exc: + raise ScopeAccessError(f"{kind} has an invalid workspace scope") from exc + deployment_id = configurable.get("workspace_deployment_id") + if deployment_id is not None and ( + not isinstance(deployment_id, str) or not deployment_id + ): + raise ScopeAccessError(f"{kind} has an invalid workspace deployment") + return _RuntimeScopeConfig( + scope_id=canonical_scope_id, + owner_id=canonical_owner_id, + thread_id=thread_id, + deployment_id=deployment_id, + ) + + +def _validated_scope_directories(scope_id: str) -> tuple[Path, Path]: + """Return canonical private directories after preventing symlink escape. + + This function intentionally resolves paths and must run only from a + filesystem-operation worker, never from the runtime backend factory. + """ + + conversations_dir = ( + paths.WORKSPACE_ROOT.expanduser() / ".evoscientist" / "conversations" + ).resolve(strict=True) + scope_root = (conversations_dir / scope_id).resolve(strict=True) + files_dir = (scope_root / "files").resolve(strict=True) + runtime_dir = (scope_root / "runtime").resolve(strict=True) + if ( + scope_root.parent != conversations_dir + or files_dir.parent != scope_root + or runtime_dir.parent != scope_root + ): + raise ScopeAccessError("workspace directory escapes its scope") + if not files_dir.is_dir() or not runtime_dir.is_dir(): + raise ScopeAccessError("workspace directory is missing") + return files_dir, runtime_dir + + +def _resolve_scope_context(config: _RuntimeScopeConfig | None) -> ScopeContext | None: + """Validate parsed scope identifiers against the active registry.""" + + if config is None: + return None + deployment_id = config.deployment_id or current_deployment_id() + registry = get_scope_registry(paths.WORKSPACE_ROOT) + if registry.active_lock(deployment_id, "workspace-cutover") is not None: + raise ScopeAccessError("workspace cutover is in progress") + record = registry.assert_runtime( + deployment_id, config.scope_id, config.thread_id, config.owner_id + ) + files_dir, runtime_dir = _validated_scope_directories(record.scope_id) + return ScopeContext( + deployment_id=deployment_id, + scope_id=record.scope_id, + owner_id=config.owner_id, + thread_id=config.thread_id, + revision=record.revision, + files_dir=files_dir, + runtime_dir=runtime_dir, + primary_thread_id=record.primary_thread_id, + ) + + +def require_scoped_runtime( + runtime: ToolRuntime[Any, Any] | Any | None, + *, + kind: str = "tool", +) -> ScopeContext | None: + """Resolve and validate a runtime scope. + + ``optional`` retains legacy CLI compatibility when no scope has been + injected. ``required`` never falls back to ``WORKSPACE_ROOT``. + """ + + config = _runtime_scope_config(runtime, kind=kind) + return _resolve_scope_context(config) + + +def provision_conversation_scope( + thread_id: str, + *, + deployment_id: str | None = None, + scope_id: str | None = None, + workspace_root: Path | None = None, + lock_operation_id: str | None = None, +) -> ScopeRecord: + """Create the registry mapping and private directory for a primary thread.""" + + root = (workspace_root or paths.WORKSPACE_ROOT).expanduser() + deployment_id = deployment_id or deployment_id_for_workspace(root) + registry = get_scope_registry(root) + record = registry.provision( + deployment_id, + thread_id, + scope_id=scope_id, + lock_operation_id=lock_operation_id, + ) + root = conversation_root(record.scope_id, root) + try: + (root / "files").mkdir(mode=0o700, parents=True, exist_ok=True) + (root / "runtime").mkdir(mode=0o700, parents=True, exist_ok=True) + for directory in (root, root / "files", root / "runtime"): + try: + directory.chmod(0o700) + except OSError: + pass + except OSError: + # Keep the durable reservation for the recovery job; it is safer than + # silently falling back to the shared deployment root. + raise + return record + + +def _build_backend(root_dir: Path, *, dangerous: bool) -> Any: + from deepagents.backends import CompositeBackend + + from .backends import ( + CustomSandboxBackend, + MemoryFilesystemBackend, + MergedSkillsBackend, + ) + from .EvoScientist import SKILLS_DIR + + cfg_timeout = int(os.getenv("EVOSCIENTIST_SANDBOX_EXECUTE_TIMEOUT", "300")) + ws_backend: Any + if is_required(): + ws_backend = ScopedContainerBackend(root_dir, timeout=cfg_timeout) + else: + ws_backend = CustomSandboxBackend( + root_dir=str(root_dir), + virtual_mode=True, + timeout=cfg_timeout, + dangerous=dangerous, + ) + return CompositeBackend( + default=ws_backend, + routes={ + "/skills/": MergedSkillsBackend( + primary_dir=str(paths.USER_SKILLS_DIR), + global_dir=str(paths.GLOBAL_SKILLS_DIR), + secondary_dir=SKILLS_DIR, + ), + "/memories/": MemoryFilesystemBackend( + root_dir=str(paths.MEMORIES_DIR), virtual_mode=True + ), + }, + ) + + +class DeferredScopedBackend(SandboxBackendProtocol): + """Resolve the scoped filesystem backend only from a worker thread. + + DeepAgents invokes its deprecated backend factory from async middleware. + Its concrete filesystem backends synchronously call ``Path.resolve()`` in + their constructors, so doing that work in the factory makes every run fail + under LangGraph's blocking-call detector. This proxy itself is I/O-free; + the inherited async methods dispatch the synchronous operations to a + thread, where Registry validation and concrete backend construction occur. + """ + + def __init__( + self, + config: _RuntimeScopeConfig, + *, + dangerous: bool, + ) -> None: + self._config = config + self._dangerous = dangerous + self._backend: Any | None = None + self._backend_key: tuple[str, str, str, int] | None = None + self._lock = threading.RLock() + + @property + def id(self) -> str: + # This is queried while composing the model request; do not initialize + # the real backend or touch the Registry here. + return f"scope-{self._config.scope_id[:8]}-{self._config.owner_id[:8]}" + + def _delegate(self) -> Any: + """Validate the current scope and return a concrete backend. + + Every operation enters here, so a deleted scope or stale owner cannot + keep using a backend constructed before the lifecycle transition. + """ + + # Async backend methods run this code in a worker thread. LangGraph's + # RunnableConfig context variable is not available there, so validate + # the immutable scope parsed by the factory on the graph thread. + context = _resolve_scope_context(self._config) + if context is None: + raise ScopeAccessError("scoped backend lost its workspace scope") + if is_required() and self._dangerous: + raise ScopeAccessError( + "dangerous_mode is incompatible with required isolation" + ) + key = (context.scope_id, context.owner_id, context.thread_id, context.revision) + with self._lock: + if self._backend is None or self._backend_key != key: + self._backend = _build_backend( + context.files_dir, dangerous=self._dangerous + ) + self._backend_key = key + return self._backend + + def ls(self, path: str) -> LsResult: + return self._delegate().ls(path) + + def read(self, file_path: str, offset: int = 0, limit: int = 2000) -> ReadResult: + return self._delegate().read(file_path, offset, limit) + + def grep( + self, pattern: str, path: str | None = None, glob: str | None = None + ) -> GrepResult: + return self._delegate().grep(pattern, path, glob) + + def glob(self, pattern: str, path: str | None = None) -> GlobResult: + return self._delegate().glob(pattern, path) + + def write(self, file_path: str, content: str) -> WriteResult: + return self._delegate().write(file_path, content) + + def edit( + self, + file_path: str, + old_string: str, + new_string: str, + replace_all: bool = False, + ) -> EditResult: + return self._delegate().edit(file_path, old_string, new_string, replace_all) + + def upload_files(self, files: list[tuple[str, bytes]]) -> list[FileUploadResponse]: + return self._delegate().upload_files(files) + + def download_files(self, paths: list[str]) -> list[FileDownloadResponse]: + return self._delegate().download_files(paths) + + def execute(self, command: str, *, timeout: int | None = None) -> ExecuteResponse: + return self._delegate().execute(command, timeout=timeout) + + +def create_workspace_backend( + runtime: ToolRuntime[Any, Any], + *, + legacy_backend: Callable[[], Any], + dangerous: bool = False, + allow_unscoped_legacy: bool = True, +) -> Any: + """Return a backend handle without blocking the Agent event loop.""" + + config = _runtime_scope_config(runtime, kind="filesystem backend") + if config is None: + if not allow_unscoped_legacy: + raise ScopeAccessError( + "deployed graph runs require a workspace scope" + ) + return legacy_backend() + if is_required() and dangerous: + raise ScopeAccessError("dangerous_mode is incompatible with required isolation") + return DeferredScopedBackend(config, dangerous=dangerous) + + +def create_workspace_backend_factory( + legacy_backend: Callable[[], Any], + *, + dangerous: bool = False, + allow_unscoped_legacy: bool = True, +) -> Callable[[ToolRuntime[Any, Any]], Any]: + def factory(runtime: ToolRuntime[Any, Any]) -> Any: + return create_workspace_backend( + runtime, + legacy_backend=legacy_backend, + dangerous=dangerous, + allow_unscoped_legacy=allow_unscoped_legacy, + ) + + return factory + + +def workspace_metadata(record: ScopeRecord) -> dict[str, str | int]: + """Metadata mirrored onto the LangGraph primary thread by trusted callers.""" + + return { + "workspace_schema_version": 1, + "workspace_scope_id": record.scope_id, + "workspace_status": record.state, + "workspace_scope_owner_id": record.primary_owner_id, + "workspace_scope_revision": record.revision, + "workspace_deployment_id": record.deployment_id, + } diff --git a/README.md b/README.md index 11a1884..3ae8704 100644 --- a/README.md +++ b/README.md @@ -428,6 +428,32 @@ Alternatively, copy the example `.env` file for project-level configuration: cp .env.example .env # then fill in your keys ``` +### WebUI workspace isolation + +`EVOSCIENTIST_WORKSPACE_DIR` identifies the deployment workspace root. It is +not a directory selected by the browser. In isolated modes, each WebUI +conversation receives its own `files/` and `runtime/` directories below: + +```text +/.evoscientist/conversations// +``` + +`EVOSCIENTIST_WORKSPACE_ISOLATION` is a deployment-level policy with three +values: + +| Value | Behavior | Use it when | +| --- | --- | --- | +| `legacy` | All WebUI conversations use the shared deployment root. Files are visible across those conversations. | Only for an explicit compatibility rollback. | +| `optional` | New WebUI conversations are given isolated scope directories. Missing Registry, token, or scope rejects the operation; it does not fall back to the shared root. | Default for single-host deployments and migration. | +| `required` | Keeps per-conversation isolation and additionally rejects missing or invalid runtime ownership. It requires a completed cutover and a verified, pinned OCI executor. | Production hardening after migration is complete. | + +Use `optional` unless you are intentionally rolling back or have completed the +`required` deployment checks. Changing this setting, the workspace root, or the +strict executor configuration requires restarting both the backend and WebUI. +It affects new operations only; do not use it to convert an existing +conversation between shared and isolated directories. Create a new conversation +and explicitly import files instead. + > ⚠️ Never commit `.env` files with real keys. It is already in `.gitignore`. @@ -481,6 +507,8 @@ EvoSci config set webui_port 4800 # change the front-end port (must differ fr Requires **Node.js 24 LTS** (for `npx`); the first launch downloads `@evoscientist/webui` and needs network. Note: the WebUI does not show your CLI/TUI chat history, and `-p` / `--resume` fall back to the classic CLI. +You may also run `EvoSci deploy --workdir ` and the WebUI separately. For processes owned by the same OS user, both sides automatically share the token-usage identity and private sink token through `~/.evoscientist`; no manual environment variables are required. The WebUI port must match the configured `webui_port` (default `4716`). +
diff --git a/README.zh-CN.md b/README.zh-CN.md index 04c3427..a4cdb23 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -406,6 +406,27 @@ export TAVILY_API_KEY="tvly-..." # app.tavily.com cp .env.example .env # 填入你的密钥 ``` +### WebUI 工作目录隔离 + +`EVOSCIENTIST_WORKSPACE_DIR` 指向部署工作目录根，而不是浏览器可选择的目录。在隔离模式下， +每个 WebUI 对话会在以下位置拥有独立的 `files/` 与 `runtime/` 目录： + +```text +/.evoscientist/conversations// +``` + +`EVOSCIENTIST_WORKSPACE_ISOLATION` 是部署级策略，支持三个值： + +| 值 | 行为 | 适用场景 | +| --- | --- | --- | +| `legacy` | 所有 WebUI 对话使用同一个部署公共根目录，文件会在这些对话之间可见。 | 仅用于明确的兼容性回退。 | +| `optional` | 新建 WebUI 对话获得独立 scope 目录；缺少 Registry、令牌或 scope 时请求失败，不会回退到公共目录。 | 默认值，适用于单机部署和迁移阶段。 | +| `required` | 保留会话隔离，并拒绝缺少或非法运行时归属的操作；需要完成全量 cutover 并验证固定 OCI 执行器。 | 迁移完成后的生产加固。 | + +除非明确需要回退，或已经完成 `required` 的部署检查，否则应使用 `optional`。修改该值、 +工作目录根或严格执行器配置后，必须同时重启后端和 WebUI。它不会转换已有对话的目录策略； +若需在共享与隔离之间转换，应新建对话并显式导入文件。 + > ⚠️ 切勿将包含真实密钥的 `.env` 文件提交到版本库。该文件已在 `.gitignore` 中。
@@ -456,6 +477,8 @@ EvoSci config set webui_port 4800 # 修改前端端口（须与 langgraph dev 需要 **Node.js 24 LTS**（提供 `npx`）；首次启动会下载 `@evoscientist/webui`，需要联网。注意：WebUI 不会显示 CLI/TUI 的历史会话，且 `-p` / `--resume` 会回退到经典 CLI。 +也可以分别运行 `EvoSci deploy --workdir <目录>` 和 WebUI。相同系统用户下，两端会通过 `~/.evoscientist` 自动共享 Token 统计身份和私有 sink token，无需手工设置环境变量。WebUI 端口必须与 `webui_port` 配置一致（默认 `4716`）。 +
diff --git a/pyproject.toml b/pyproject.toml index ccd4890..22749d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "langgraph-cli[inmem]>=0.4", "langgraph-checkpoint-sqlite>=3.0", "httpx>=0.28", + "pydantic>=2.10", "psutil>=6.0", "filelock>=3.16", "lazy-loader>=0.5", diff --git a/scripts/benchmark_usage_spool.py b/scripts/benchmark_usage_spool.py new file mode 100644 index 0000000..a61b8fe --- /dev/null +++ b/scripts/benchmark_usage_spool.py @@ -0,0 +1,127 @@ +"""Measure durable UsageEvent enqueue latency for the release platform matrix.""" + +from __future__ import annotations + +import json +import os +import platform +import tempfile +import time +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime +from pathlib import Path + +from EvoScientist.usage.schema import UsageEventV1 +from EvoScientist.usage.spool import UsageSpool + +WORKER_COUNTS = (1, 4, 16) +SAMPLES_PER_WORKER = 32 +P95_LIMIT_MS = 20.0 +P99_LIMIT_MS = 100.0 + + +class BenchmarkSpool(UsageSpool): + def _run(self) -> None: + self._stop.wait() + + +def _percentile(samples: list[float], percentile: float) -> float: + ordered = sorted(samples) + index = min(len(ordered) - 1, max(0, int(len(ordered) * percentile) - 1)) + return ordered[index] + + +def _event(index: int) -> UsageEventV1: + model_call_id = f"benchmark-{index:08d}" + now = datetime.now(UTC) + deployment_id = "11111111-1111-4111-8111-111111111111" + return UsageEventV1( + schema_version=1, + event_id=f"{deployment_id}:{model_call_id}:callback_final:1", + event_type="usage_observed", + source="callback_final", + authority_class="observed_final", + revision=1, + deployment_id=deployment_id, + workspace_id="ws1_benchmark", + model_call_id=model_call_id, + parent_run_id=None, + provider_request_id=None, + thread_id="benchmark-thread", + source_session_id=None, + turn_id="benchmark-turn", + workspace_dir=None, + scope="main", + source_agent="EvoScientist", + provider_profile_id="benchmark", + provider_revision=None, + provider_adapter="openai", + model_alias="benchmark", + upstream_model_id="benchmark", + usage_status="confirmed", + input_tokens=1, + output_tokens=1, + provider_total_tokens=2, + input_token_details={}, + output_token_details={}, + started_at=now, + observed_at=now, + completed_at=now, + ) + + +def _run_case(root: Path, workers: int) -> dict[str, float | int]: + spool_root = root / str(workers) + os.environ.update( + { + "EVOSCIENTIST_USAGE_SINK_URL": "http://127.0.0.1:1/api/usage/events", + "EVOSCIENTIST_USAGE_SINK_TOKEN": "benchmark", + "EVOSCIENTIST_DEPLOYMENT_ID": "11111111-1111-4111-8111-111111111111", + "EVOSCIENTIST_WORKSPACE_ID": "ws1_benchmark", + "EVOSCIENTIST_USAGE_SPOOL_DIR": str(spool_root), + "EVOSCIENTIST_USAGE_SPOOL_MAX_FILES": "100000", + } + ) + spool = BenchmarkSpool() + + def enqueue(index: int) -> float: + started = time.perf_counter_ns() + spool.enqueue(_event(workers * 1_000_000 + index)) + return (time.perf_counter_ns() - started) / 1_000_000 + + try: + with ThreadPoolExecutor(max_workers=workers) as pool: + samples = list(pool.map(enqueue, range(workers * SAMPLES_PER_WORKER))) + finally: + spool.close() + return { + "workers": workers, + "samples": len(samples), + "p50_ms": round(_percentile(samples, 0.50), 3), + "p95_ms": round(_percentile(samples, 0.95), 3), + "p99_ms": round(_percentile(samples, 0.99), 3), + "max_ms": round(max(samples), 3), + } + + +def main() -> None: + with tempfile.TemporaryDirectory(prefix="evosci-usage-benchmark-") as temporary: + results = [_run_case(Path(temporary), workers) for workers in WORKER_COUNTS] + report = { + "platform": platform.platform(), + "python": platform.python_version(), + "limits_ms": {"p95": P95_LIMIT_MS, "p99": P99_LIMIT_MS}, + "results": results, + } + print(json.dumps(report, indent=2, sort_keys=True)) + failed = [ + result + for result in results + if result["p95_ms"] > P95_LIMIT_MS or result["p99_ms"] > P99_LIMIT_MS + ] + if failed: + raise SystemExit("usage spool latency exceeds the release threshold") + + +if __name__ == "__main__": + main() diff --git a/scripts/workspace_cutover.py b/scripts/workspace_cutover.py new file mode 100644 index 0000000..4a7c320 --- /dev/null +++ b/scripts/workspace_cutover.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Run the durable conversation-workspace migration against a local backend.""" + +from __future__ import annotations + +import argparse +import json +import subprocess +from pathlib import Path + +from EvoScientist.langgraph_dev.sdk import get_langgraph_sync_client +from EvoScientist.workspace_cutover import run_workspace_cutover + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--workspace", required=True) + parser.add_argument("--api-url", default="http://127.0.0.1:6174") + parser.add_argument("--assistant-id", default="EvoScientist") + parser.add_argument( + "--webui-root", + default=str(Path(__file__).resolve().parents[2] / "EvoScientist-WebUI"), + ) + args = parser.parse_args() + try: + gate = subprocess.run( + ["npm", "run", "verify:workspace-isolation"], + cwd=args.webui_root, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + timeout=120, + ) + browser_sdk_gate = gate.returncode == 0 + except (OSError, subprocess.TimeoutExpired): + browser_sdk_gate = False + report = run_workspace_cutover( + workspace_root=Path(args.workspace), + client=get_langgraph_sync_client(url=args.api_url), + assistant_id=args.assistant_id, + browser_sdk_gate=browser_sdk_gate, + ) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 if report["status"] == "passed" else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/workspace_maintenance.py b/scripts/workspace_maintenance.py new file mode 100644 index 0000000..9e316b7 --- /dev/null +++ b/scripts/workspace_maintenance.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Run periodic draft and conversation-trash cleanup for one deployment.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from EvoScientist.config import load_config +from EvoScientist.langgraph_dev.sdk import get_langgraph_sync_client +from EvoScientist.workspace_maintenance import run_workspace_maintenance + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--workspace", required=True) + parser.add_argument("--api-url", default="http://127.0.0.1:6174") + parser.add_argument("--draft-ttl-hours", type=int) + parser.add_argument("--trash-retention-days", type=int) + args = parser.parse_args() + config = load_config() + report = run_workspace_maintenance( + workspace_root=Path(args.workspace), + client=get_langgraph_sync_client(url=args.api_url), + draft_ttl_hours=args.draft_ttl_hours or config.draft_workspace_ttl_hours, + trash_retention_days=args.trash_retention_days + or config.workspace_trash_retention_days, + ) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 if not report["errors"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_ask_user.py b/tests/test_ask_user.py index cc21f71..7f8c494 100644 --- a/tests/test_ask_user.py +++ b/tests/test_ask_user.py @@ -1,9 +1,23 @@ """Tests for the ask_user middleware, stream events, state, and UI helpers.""" +from contextlib import contextmanager from unittest.mock import MagicMock, patch import pytest + +@contextmanager +def _patched_runtime_config(config): + import langgraph.config as langgraph_config + + if isinstance(config, Exception): + with patch.object(langgraph_config, "get_config", side_effect=config): + yield + else: + with patch.object(langgraph_config, "get_config", return_value=config): + yield + + # --------------------------------------------------------------------------- # Middleware data types # --------------------------------------------------------------------------- @@ -292,6 +306,129 @@ class TestAskUserMiddleware: assert "resource" in ASK_USER_TOOL_DESCRIPTION.lower() + @pytest.mark.parametrize("mode", ["manual", "auto"]) + def test_manual_and_auto_keep_ask_user(self, mode): + from langchain.agents.middleware.types import ModelRequest + + from EvoScientist.middleware.ask_user import ( + ASK_USER_SYSTEM_PROMPT, + AskUserMiddleware, + ) + + middleware = AskUserMiddleware() + request = ModelRequest( + model=MagicMock(), + messages=[], + tools=[middleware.tools[0], {"name": "execute"}], + ) + handler = MagicMock(return_value="response") + with _patched_runtime_config({"configurable": {"review_mode": mode}}): + result = middleware.wrap_model_call(request, handler) + + assert result == "response" + forwarded = handler.call_args.args[0] + assert [ + tool.name if hasattr(tool, "name") else tool["name"] + for tool in forwarded.tools + ] == [ + "ask_user", + "execute", + ] + assert ASK_USER_SYSTEM_PROMPT in forwarded.system_message.text + + def test_full_removes_ask_user_and_adds_unattended_prompt(self): + from langchain.agents.middleware.types import ModelRequest + + from EvoScientist.middleware.ask_user import ( + FULL_APPROVE_SYSTEM_PROMPT, + AskUserMiddleware, + ) + + middleware = AskUserMiddleware() + request = ModelRequest( + model=MagicMock(), + messages=[], + tools=[ + middleware.tools[0], + {"name": "execute"}, + {"type": "function", "function": {"name": "ask_user"}}, + ], + ) + handler = MagicMock(return_value="response") + with _patched_runtime_config({"configurable": {"review_mode": "full"}}): + result = middleware.wrap_model_call(request, handler) + + assert result == "response" + forwarded = handler.call_args.args[0] + assert forwarded.tools == [{"name": "execute"}] + assert FULL_APPROVE_SYSTEM_PROMPT in forwarded.system_message.text + + async def test_full_async_wrapper_matches_sync_behavior(self): + from langchain.agents.middleware.types import ModelRequest + + from EvoScientist.middleware.ask_user import AskUserMiddleware + + middleware = AskUserMiddleware() + request = ModelRequest( + model=MagicMock(), + messages=[], + tools=[middleware.tools[0], {"name": "execute"}], + ) + + async def handler(forwarded): + assert forwarded.tools == [{"name": "execute"}] + return "response" + + with _patched_runtime_config({"configurable": {"review_mode": "full"}}): + result = await middleware.awrap_model_call(request, handler) + assert result == "response" + + def test_full_tool_defense_does_not_interrupt(self): + from EvoScientist.middleware.ask_user import ( + FULL_APPROVE_TOOL_MESSAGE, + AskUserMiddleware, + ) + + middleware = AskUserMiddleware() + with ( + _patched_runtime_config({"configurable": {"review_mode": "full"}}), + patch("EvoScientist.middleware.ask_user.interrupt") as mock_interrupt, + ): + result = middleware.tools[0].func( + questions=[], + tool_call_id="tool-call-1", + ) + + mock_interrupt.assert_not_called() + message = result.update["messages"][0] + assert message.content == FULL_APPROVE_TOOL_MESSAGE + assert message.tool_call_id == "tool-call-1" + + +class TestReviewMode: + @pytest.mark.parametrize( + ("config", "expected"), + [ + ({"configurable": {"review_mode": "manual"}}, "manual"), + ({"configurable": {"review_mode": "auto"}}, "auto"), + ({"configurable": {"review_mode": "full"}}, "full"), + ({"configurable": {"review_mode": "invalid"}}, "manual"), + ({"configurable": "invalid"}, "manual"), + ("invalid", "manual"), + ], + ) + def test_review_mode_parsing(self, config, expected): + from EvoScientist.middleware.ask_user import _review_mode + + with _patched_runtime_config(config): + assert _review_mode() == expected + + def test_outside_runtime_defaults_to_manual(self): + from EvoScientist.middleware.ask_user import _review_mode + + with _patched_runtime_config(RuntimeError("outside runtime")): + assert _review_mode() == "manual" + # --------------------------------------------------------------------------- # Stream event emitter diff --git a/tests/test_cli_deploy.py b/tests/test_cli_deploy.py index 27578ff..5a87515 100644 --- a/tests/test_cli_deploy.py +++ b/tests/test_cli_deploy.py @@ -1,7 +1,7 @@ """Tests for ``EvoSci deploy`` command flow. Verifies the orchestration: -- workspace resolution (CLI > config > cwd) +- workspace resolution (CLI > config > stable default) - port resolution (CLI > config > default) - port collision pre-flight - ccproxy lifecycle (only if OAuth configured) @@ -11,6 +11,9 @@ Verifies the orchestration: from __future__ import annotations +import io +import threading +from pathlib import Path from types import SimpleNamespace from typing import Any @@ -24,6 +27,7 @@ def _make_config( *, default_workdir: str = "", langgraph_dev_port: int = 6174, + webui_port: int = 4716, anthropic_auth_mode: str = "api_key", openai_auth_mode: str = "api_key", log_level: str = "warning", @@ -34,6 +38,7 @@ def _make_config( return SimpleNamespace( default_workdir=default_workdir, langgraph_dev_port=langgraph_dev_port, + webui_port=webui_port, anthropic_auth_mode=anthropic_auth_mode, openai_auth_mode=openai_auth_mode, log_level=log_level, @@ -76,6 +81,7 @@ def _run_deploy_once( langgraph_dev_running: bool = True, # health-check passes after start tunnel: bool = False, tunnel_url: str | None = None, + default_workspace: str = "/tmp/evoscientist-default-workspace", ): """Run ``deploy()`` end-to-end with all external dependencies mocked. Returns a ``captured`` dict with observation points.""" @@ -90,8 +96,23 @@ def _run_deploy_once( "workspace_passed": None, "port_passed": None, "atexit_callbacks": [], + "log_follower_started": False, + "log_follower_stopped": False, + "usage_environment_prepared": False, } + def _fake_prepare_usage_environment(workspace_dir, *, webui_port): + captured["usage_environment_prepared"] = True + captured["usage_workspace"] = str(workspace_dir) + captured["usage_webui_port"] = webui_port + return {} + + monkeypatch.setattr( + deploy_server, + "prepare_usage_environment", + _fake_prepare_usage_environment, + ) + def _fake_get_effective_config(cli_overrides=None): captured["cli_overrides"] = dict(cli_overrides or {}) merged = vars(config).copy() @@ -100,9 +121,32 @@ def _run_deploy_once( monkeypatch.setattr(config_mod, "get_effective_config", _fake_get_effective_config) monkeypatch.setattr(config_mod, "apply_config_to_env", lambda _cfg: None) + monkeypatch.setattr( + config_mod, "get_default_workspace_dir", lambda: Path(default_workspace) + ) monkeypatch.setattr(deploy_server, "console", _SilentConsole()) + class _FakeFollowerStop: + def set(self): + captured["log_follower_stopped"] = True + + class _FakeFollowerThread: + def join(self, timeout=None): + captured["log_follower_join_timeout"] = timeout + + def _fake_start_gateway_log_follower(log_path, start_offset): + captured["log_follower_started"] = True + captured["log_follower_path"] = log_path + captured["log_follower_offset"] = start_offset + return _FakeFollowerStop(), _FakeFollowerThread() + + monkeypatch.setattr( + deploy_server, + "_start_gateway_log_follower", + _fake_start_gateway_log_follower, + ) + # Workspace setup mocks from EvoScientist import paths as paths_mod @@ -242,6 +286,74 @@ def test_deploy_starts_langgraph_dev_with_deploy_mode_true(monkeypatch, tmp_path ) +def test_deploy_prepares_usage_for_separately_started_webui(monkeypatch, tmp_path): + workspace = tmp_path / "workspace" + config = _make_config( + default_workdir=str(workspace), + webui_port=4816, + ) + + captured = _run_deploy_once(monkeypatch, config) + + assert captured["usage_environment_prepared"] is True + assert captured["usage_workspace"] == str(workspace) + assert captured["usage_webui_port"] == 4816 + + +def test_deploy_streams_and_stops_gateway_log_follower(monkeypatch, tmp_path): + config = _make_config(default_workdir=str(tmp_path)) + captured = _run_deploy_once(monkeypatch, config) + + assert captured["log_follower_started"] is True + assert captured["log_follower_stopped"] is True + assert captured["log_follower_join_timeout"] == 2.0 + + +def test_follow_gateway_log_only_replays_current_session(tmp_path): + log_path = tmp_path / "langgraph_dev.log" + stale = "old session\n" + current = "Gateway ready\n模型调用完成\n" + log_path.write_bytes((stale + current).encode("utf-8")) + stop_event = threading.Event() + stop_event.set() + output = io.StringIO() + + deploy_server._follow_gateway_log( + log_path, + len(stale.encode("utf-8")), + stop_event, + output=output, + ) + + assert output.getvalue() == current + + +def test_follow_gateway_log_streams_appended_output(tmp_path): + log_path = tmp_path / "langgraph_dev.log" + log_path.write_text("Gateway ready\n", encoding="utf-8") + output = io.StringIO() + + class _AppendThenStop: + stopped = False + + def is_set(self): + return self.stopped + + def wait(self, _timeout): + with log_path.open("a", encoding="utf-8") as log_file: + log_file.write("new request\n") + self.stopped = True + + deploy_server._follow_gateway_log( + log_path, + 0, + _AppendThenStop(), + output=output, + ) + + assert output.getvalue() == "Gateway ready\nnew request\n" + + def test_deploy_tunnel_default_off(monkeypatch, tmp_path): """Without ``--tunnel``, start_langgraph_dev is called with tunnel=False.""" config = _make_config(default_workdir=str(tmp_path)) @@ -280,12 +392,17 @@ def test_deploy_workdir_config_beats_cwd(monkeypatch, tmp_path): assert captured["workspace_passed"] == str(cfg_ws) -def test_deploy_workdir_falls_back_to_cwd(monkeypatch, tmp_path): +def test_deploy_workdir_falls_back_to_stable_default(monkeypatch, tmp_path): config = _make_config(default_workdir="") - cwd = str(tmp_path / "cwd") - captured = _run_deploy_once(monkeypatch, config, cwd=cwd) + default_workspace = str(tmp_path / "default-workspace") + captured = _run_deploy_once( + monkeypatch, + config, + cwd=str(tmp_path / "cwd"), + default_workspace=default_workspace, + ) - assert captured["workspace_passed"] == cwd + assert captured["workspace_passed"] == default_workspace def test_deploy_port_cli_arg_beats_config(monkeypatch, tmp_path): diff --git a/tests/test_config.py b/tests/test_config.py index 8799da5..81a9844 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -17,6 +17,7 @@ from EvoScientist.config import ( get_config_dir, get_config_path, get_config_value, + get_default_workspace_dir, get_effective_config, list_config, load_config, @@ -192,6 +193,26 @@ class TestEvoScientistConfig: assert config.dangerous_mode is True assert config.auto_approve is True + def test_workspace_isolation_defaults_to_optional(self): + config = EvoScientistConfig() + assert config.workspace_isolation == "optional" + + def test_required_isolation_needs_a_pinned_executor_image(self): + with pytest.raises(ValueError, match="pinned by digest"): + EvoScientistConfig(workspace_isolation="required") + + config = EvoScientistConfig( + workspace_isolation="required", + strict_executor_image="registry.example/runtime@sha256:" + "a" * 64, + ) + assert config.strict_executor_image.endswith("a" * 64) + + def test_workspace_maintenance_retention_values_must_be_positive(self): + with pytest.raises(ValueError, match="draft_workspace_ttl_hours"): + EvoScientistConfig(draft_workspace_ttl_hours=0) + with pytest.raises(ValueError, match="workspace_trash_retention_days"): + EvoScientistConfig(workspace_trash_retention_days=0) + # ============================================================================= # Test config path functions @@ -216,6 +237,11 @@ class TestConfigPaths: config_path = get_config_path() assert config_path == temp_config_dir / "config.yaml" + def test_default_workspace_dir_is_stable_under_home(self, monkeypatch, tmp_path): + monkeypatch.setattr(Path, "home", lambda: tmp_path) + + assert get_default_workspace_dir() == tmp_path / ".evoscientist" / "workspace" + # ============================================================================= # Test load/save/reset diff --git a/tests/test_cron_schedule.py b/tests/test_cron_schedule.py index 2aa9679..46beda6 100644 --- a/tests/test_cron_schedule.py +++ b/tests/test_cron_schedule.py @@ -201,10 +201,10 @@ requirements = [ monkeypatch.setattr(manager, "RUNTIME", runtime) workspace = tmp_path / "workspace-écho" manager._write_workspace_sidecar(workspace, 12345) - assert manager._read_workspace_sidecar() == { - "workspace": str(workspace), - "pid": 12345, - } + sidecar = manager._read_workspace_sidecar() + assert sidecar is not None + assert sidecar["workspace"] == str(workspace) + assert sidecar["pid"] == 12345 runtime.pid_file.write_text(str(os.getpid()), encoding="utf-8") diff --git a/tests/test_langgraph_dev_http.py b/tests/test_langgraph_dev_http.py index a33d280..8089a67 100644 --- a/tests/test_langgraph_dev_http.py +++ b/tests/test_langgraph_dev_http.py @@ -7,15 +7,31 @@ from __future__ import annotations from unittest.mock import patch +import pytest from langchain_core.messages import AIMessage, HumanMessage from starlette.testclient import TestClient -from EvoScientist.config import EvoScientistConfig +from EvoScientist.config import EvoScientistConfig, load_config, save_config +from EvoScientist.config.provider_profiles import ( + load_provider_profiles, + replace_provider_profiles, +) from EvoScientist.langgraph_dev.http import app +from EvoScientist.llm.provider_operations import ( + DiscoveredProviderModel, + ProviderModelTestResult, + ProviderOperationError, +) client = TestClient(app) +@pytest.fixture(autouse=True) +def isolate_xdg_config(tmp_path, monkeypatch): + """Keep HTTP tests independent from the developer's saved providers.""" + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + + def test_get_models_returns_entries_and_default(): mock_cfg = EvoScientistConfig( model="claude-sonnet-4-6", provider="custom-anthropic" @@ -45,6 +61,695 @@ def test_get_models_returns_entries_and_default(): assert entry["provider"] +def test_get_models_only_returns_enabled_catalog_entries_when_configured(): + mock_cfg = EvoScientistConfig( + model="chat-main", + provider="openai", + model_catalog=[ + { + "provider": "openai", + "id": "chat-main", + "name": "Chat Main", + "model_id": "gpt-upstream", + "enabled": True, + }, + { + "provider": "anthropic", + "id": "hidden-model", + "name": "Hidden", + "model_id": "claude-hidden", + "enabled": False, + }, + ], + ) + + with patch( + "EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg + ): + body = client.get("/api/models").json() + + assert body["entries"] == [ + {"name": "chat-main", "model_id": "gpt-upstream", "provider": "openai"} + ] + + +def test_get_models_empty_catalog_hides_all_builtin_models(): + mock_cfg = EvoScientistConfig(model_catalog=[]) + + with patch( + "EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg + ): + body = client.get("/api/models").json() + + assert body["entries"] == [] + + +def test_provider_profiles_api_requires_admin_token_header(tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False) + response = client.get("/api/provider-profiles") + assert response.status_code == 403 + + +def test_provider_profiles_api_round_trip_redacts_secret(tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + headers = {"X-EvoScientist-Admin-Token": "admin-secret"} + payload = { + "providers": [ + { + "id": "lab-openai", + "name": "Lab OpenAI", + "adapter": "openai-compatible", + "base_url": "https://llm.example.test/v1", + "api_key": "provider-secret", + "enabled": True, + "models": [ + { + "id": "lab-model", + "name": "Lab Model", + "model_id": "vendor/model", + "enabled": True, + } + ], + } + ] + } + + put_response = client.put("/api/provider-profiles", headers=headers, json=payload) + assert put_response.status_code == 200 + assert "provider-secret" not in put_response.text + assert put_response.json()["providers"][0]["api_key_configured"] is True + assert "openai" in put_response.json()["reserved_provider_ids"] + + get_response = client.get("/api/provider-profiles", headers=headers) + assert get_response.status_code == 200 + assert get_response.json()["providers"][0]["models"][0]["id"] == "lab-model" + assert "provider-secret" not in get_response.text + + +def test_llm_config_api_requires_admin_token_header(monkeypatch): + monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False) + + response = client.get("/api/config") + + assert response.status_code == 403 + + +def test_llm_config_api_redacts_secrets_and_reports_env_overrides(monkeypatch): + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + monkeypatch.setenv("OPENAI_API_KEY", "environment-secret") + save_config( + EvoScientistConfig( + provider="openai", + model="gpt-5.4", + openai_api_key="file-secret-1234", + default_workdir="/tmp/research", + ) + ) + + response = client.get( + "/api/config", + headers={"X-EvoScientist-Admin-Token": "admin-secret"}, + ) + + assert response.status_code == 200 + body = response.json() + assert body["values"]["provider"] == "openai" + assert body["values"]["model"] == "gpt-5.4" + assert "openai_api_key" not in body["values"] + assert body["secrets"]["openai_api_key"] == { + "configured": True, + "hint": "...1234", + } + assert body["env_overrides"]["openai_api_key"] == "OPENAI_API_KEY" + assert "file-secret-1234" not in response.text + assert "environment-secret" not in response.text + assert "default_workdir" not in body["values"] + assert body["model_catalog"] is None + assert body["builtin_model_candidates"] + openai = next( + provider for provider in body["builtin_providers"] if provider["id"] == "openai" + ) + assert openai["managed"] is False + assert openai["api_key_configured"] is True + assert "environment-secret" not in response.text + + +def test_llm_config_api_saves_builtin_registry_without_rewriting_legacy_secret( + monkeypatch, +): + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + headers = {"X-EvoScientist-Admin-Token": "admin-secret"} + save_config( + EvoScientistConfig( + provider="openai", + model="chat-main", + openai_api_key="legacy-secret", + default_workdir="/tmp/research", + ) + ) + loaded = client.get("/api/config", headers=headers).json() + openai = next( + provider + for provider in loaded["builtin_providers"] + if provider["id"] == "openai" + ) + openai.update( + { + "managed": True, + "base_url": "https://proxy.example.test/v1", + "models": [ + { + "id": "chat-main", + "name": "Chat Main", + "model_id": "gpt-upstream", + "enabled": True, + } + ], + } + ) + + response = client.patch( + "/api/config", + headers=headers, + json={ + "revision": loaded["revision"], + "values": loaded["values"], + "secrets": {}, + "clear_secrets": [], + "builtin_providers": [openai], + }, + ) + + assert response.status_code == 200 + body = response.json() + assert body["restart_required"] is False + assert "builtin_providers" in body["changed_fields"] + saved_profile = load_provider_profiles().builtins[0] + assert saved_profile.id == "openai" + assert saved_profile.api_key == "legacy-secret" + assert saved_profile.base_url == "https://proxy.example.test/v1" + saved_config = load_config() + assert saved_config.openai_api_key == "legacy-secret" + assert saved_config.default_workdir == "/tmp/research" + assert saved_config.model_catalog is None + + models = client.get("/api/models").json()["entries"] + assert models == [ + {"name": "chat-main", "model_id": "gpt-upstream", "provider": "openai"} + ] + + +def test_first_builtin_registry_save_migrates_legacy_default_model(monkeypatch): + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://environment.example.test") + headers = {"X-EvoScientist-Admin-Token": "admin-secret"} + save_config(EvoScientistConfig(provider="anthropic", model="claude-sonnet-4-6")) + loaded = client.get("/api/config", headers=headers).json() + ollama = next( + provider + for provider in loaded["builtin_providers"] + if provider["id"] == "ollama" + ) + ollama.update( + { + "managed": True, + "base_url": "http://127.0.0.1:11434", + "models": [], + } + ) + + response = client.patch( + "/api/config", + headers=headers, + json={ + "revision": loaded["revision"], + "values": loaded["values"], + "secrets": {}, + "clear_secrets": [], + "builtin_providers": [ollama], + }, + ) + + assert response.status_code == 200 + builtins = {profile.id: profile for profile in load_provider_profiles().builtins} + assert set(builtins) == {"anthropic", "ollama"} + assert builtins["anthropic"].models[0].id == "claude-sonnet-4-6" + assert builtins["anthropic"].base_url == "" + + +def test_llm_config_api_persists_model_catalog(monkeypatch): + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + headers = {"X-EvoScientist-Admin-Token": "admin-secret"} + save_config(EvoScientistConfig(provider="openai", model="chat-main")) + revision = client.get("/api/config", headers=headers).json()["revision"] + catalog = [ + { + "provider": "openai", + "id": "chat-main", + "name": "Chat Main", + "model_id": "gpt-upstream", + "enabled": True, + } + ] + + response = client.patch( + "/api/config", + headers=headers, + json={ + "revision": revision, + "values": {}, + "secrets": {}, + "clear_secrets": [], + "model_catalog": catalog, + }, + ) + + assert response.status_code == 200 + assert response.json()["model_catalog"] == catalog + assert response.json()["restart_required"] is False + assert load_config().model_catalog == catalog + + +def test_llm_config_api_rejects_default_outside_model_catalog(monkeypatch): + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + headers = {"X-EvoScientist-Admin-Token": "admin-secret"} + save_config(EvoScientistConfig(provider="openai", model="gpt-default")) + revision = client.get("/api/config", headers=headers).json()["revision"] + + response = client.patch( + "/api/config", + headers=headers, + json={ + "revision": revision, + "values": {}, + "secrets": {}, + "clear_secrets": [], + "model_catalog": [ + { + "provider": "openai", + "id": "different-model", + "name": "Different", + "model_id": "gpt-different", + "enabled": True, + } + ], + }, + ) + + assert response.status_code == 400 + assert "Default model" in response.json()["error"] + assert load_config().model_catalog is None + + +def test_llm_config_api_patch_preserves_unrelated_fields_and_secret_by_default( + monkeypatch, +): + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + headers = {"X-EvoScientist-Admin-Token": "admin-secret"} + save_config( + EvoScientistConfig( + openai_api_key="saved-openai-key", + anthropic_api_key="saved-anthropic-key", + default_workdir="/tmp/research", + ) + ) + revision = client.get("/api/config", headers=headers).json()["revision"] + + response = client.patch( + "/api/config", + headers=headers, + json={ + "revision": revision, + "values": { + "provider": "openai", + "model": "gpt-5.4", + "ollama_base_url": "http://127.0.0.1:11434", + }, + "secrets": {"openai_api_key": "replacement-openai-key"}, + "clear_secrets": ["anthropic_api_key"], + }, + ) + + assert response.status_code == 200 + body = response.json() + assert body["restart_required"] is True + assert set(body["changed_fields"]) == { + "anthropic_api_key", + "model", + "ollama_base_url", + "openai_api_key", + "provider", + } + assert "replacement-openai-key" not in response.text + saved = load_config() + assert saved.provider == "openai" + assert saved.model == "gpt-5.4" + assert saved.ollama_base_url == "http://127.0.0.1:11434" + assert saved.openai_api_key == "replacement-openai-key" + assert saved.anthropic_api_key == "" + assert saved.default_workdir == "/tmp/research" + + +def test_llm_config_api_rejects_stale_revision(monkeypatch): + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + headers = {"X-EvoScientist-Admin-Token": "admin-secret"} + save_config(EvoScientistConfig(model="claude-sonnet-4-6")) + revision = client.get("/api/config", headers=headers).json()["revision"] + save_config(EvoScientistConfig(model="gpt-5.4", provider="openai")) + + response = client.patch( + "/api/config", + headers=headers, + json={ + "revision": revision, + "values": {"model": "claude-opus-4-8"}, + }, + ) + + assert response.status_code == 409 + assert "Reload and try again" in response.json()["error"] + assert load_config().model == "gpt-5.4" + + +@pytest.mark.parametrize( + ("payload", "message"), + [ + ( + {"values": {"openai_auth_mode": "password"}}, + "openai_auth_mode must be 'api_key' or 'oauth'", + ), + ( + {"values": {"default_workdir": "/tmp/other"}}, + "Unsupported config fields: default_workdir", + ), + ( + {"values": {"ollama_base_url": "localhost:11434"}}, + "ollama_base_url must use http:// or https://", + ), + ], +) +def test_llm_config_api_validates_updates(monkeypatch, payload, message): + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + headers = {"X-EvoScientist-Admin-Token": "admin-secret"} + revision = client.get("/api/config", headers=headers).json()["revision"] + + response = client.patch( + "/api/config", + headers=headers, + json={"revision": revision, **payload}, + ) + + assert response.status_code == 400 + assert message in response.json()["error"] + + +def test_provider_actions_api_requires_admin_token_header(tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False) + + response = client.post("/api/provider-actions", json={}) + + assert response.status_code == 403 + + +def test_provider_actions_lists_models_with_saved_api_key(tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + headers = {"X-EvoScientist-Admin-Token": "admin-secret"} + provider = { + "id": "lab-openai", + "name": "Lab OpenAI", + "adapter": "openai-compatible", + "base_url": "https://llm.example.test/v1", + "api_key": "provider-secret", + "enabled": True, + "models": [], + } + assert ( + client.put( + "/api/provider-profiles", + headers=headers, + json={"providers": [provider]}, + ).status_code + == 200 + ) + provider["api_key"] = "" + + async def fake_discover(profile): + assert profile.api_key == "provider-secret" + return [DiscoveredProviderModel("vendor/model", "Vendor Model")] + + with patch( + "EvoScientist.langgraph_dev.http.discover_provider_models", + new=fake_discover, + ): + response = client.post( + "/api/provider-actions", + headers=headers, + json={"action": "list_models", "provider": provider}, + ) + + assert response.status_code == 200 + assert response.json() == { + "models": [{"model_id": "vendor/model", "name": "Vendor Model"}] + } + + +def test_provider_actions_tests_model(tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + headers = {"X-EvoScientist-Admin-Token": "admin-secret"} + + async def fake_test(profile, model): + assert profile.adapter == "grok" + assert model.model_id == "grok-4" + return ProviderModelTestResult(latency_ms=123, response="OK") + + with patch("EvoScientist.langgraph_dev.http.test_provider_model", new=fake_test): + response = client.post( + "/api/provider-actions", + headers=headers, + json={ + "action": "test_model", + "provider": { + "id": "lab-grok", + "name": "Lab Grok", + "adapter": "grok", + "base_url": "", + "api_key": "xai-secret", + "enabled": True, + "models": [], + }, + "model": { + "id": "grok", + "name": "Grok", + "model_id": "grok-4", + }, + }, + ) + + assert response.status_code == 200 + assert response.json() == {"ok": True, "latency_ms": 123, "response": "OK"} + + +def test_provider_actions_maps_provider_failures_to_bad_gateway(tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + + async def fake_discover(_profile): + raise ProviderOperationError("Provider returned HTTP 401") + + with patch( + "EvoScientist.langgraph_dev.http.discover_provider_models", + new=fake_discover, + ): + response = client.post( + "/api/provider-actions", + headers={"X-EvoScientist-Admin-Token": "admin-secret"}, + json={ + "action": "list_models", + "provider": { + "id": "lab-openai", + "name": "Lab OpenAI", + "adapter": "openai-compatible", + "base_url": "https://llm.example.test/v1", + "api_key": "bad-key", + "enabled": True, + "models": [], + }, + }, + ) + + assert response.status_code == 502 + assert response.json() == {"error": "Provider returned HTTP 401"} + + +def test_llm_config_action_discovers_models_with_effective_builtin_secret( + monkeypatch, +): + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + monkeypatch.setenv("OPENAI_API_KEY", "environment-secret") + headers = {"X-EvoScientist-Admin-Token": "admin-secret"} + + async def fake_discover(profile): + assert profile.id == "openai" + assert profile.adapter == "openai" + assert profile.api_key == "environment-secret" + return [DiscoveredProviderModel("gpt-discovered", "GPT Discovered")] + + with patch( + "EvoScientist.langgraph_dev.http.discover_provider_models", + new=fake_discover, + ): + response = client.post( + "/api/config", + headers=headers, + json={"action": "list_models", "provider": {"id": "openai"}}, + ) + + assert response.status_code == 200 + assert response.json() == { + "models": [{"model_id": "gpt-discovered", "name": "GPT Discovered"}] + } + + +def test_llm_config_action_tests_builtin_model_with_draft_connection(monkeypatch): + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + headers = {"X-EvoScientist-Admin-Token": "admin-secret"} + + async def fake_test(profile, model): + assert profile.id == "custom-openai" + assert profile.adapter == "openai-compatible" + assert profile.base_url == "https://proxy.example.test/v1" + assert profile.api_key == "draft-secret" + assert model.model_id == "vendor/model" + return ProviderModelTestResult(latency_ms=42, response="OK") + + with patch("EvoScientist.langgraph_dev.http.test_provider_model", new=fake_test): + response = client.post( + "/api/config", + headers=headers, + json={ + "action": "test_model", + "provider": { + "id": "custom-openai", + "base_url": "https://proxy.example.test/v1", + "api_key": "draft-secret", + }, + "model": { + "id": "chat-main", + "name": "Chat Main", + "model_id": "vendor/model", + }, + }, + ) + + assert response.status_code == 200 + assert response.json() == {"ok": True, "latency_ms": 42, "response": "OK"} + + +def test_default_model_api_requires_admin_token_header(tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False) + response = client.put( + "/api/default-model", + json={"model": "gpt-5.4", "provider": "openai"}, + ) + assert response.status_code == 403 + + +def test_default_model_api_persists_pair_and_preserves_config(tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + save_config(EvoScientistConfig(default_workdir="/tmp/research")) + + response = client.put( + "/api/default-model", + headers={"X-EvoScientist-Admin-Token": "admin-secret"}, + json={"model": "gpt-5.4", "provider": "openai"}, + ) + + assert response.status_code == 200 + assert response.json() == {"default": {"name": "gpt-5.4", "provider": "openai"}} + saved = load_config() + assert saved.model == "gpt-5.4" + assert saved.provider == "openai" + assert saved.default_workdir == "/tmp/research" + + +def test_default_model_api_accepts_configured_dynamic_model(tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + headers = {"X-EvoScientist-Admin-Token": "admin-secret"} + provider_payload = { + "providers": [ + { + "id": "lab-openai", + "name": "Lab OpenAI", + "adapter": "openai-compatible", + "base_url": "https://llm.example.test/v1", + "api_key": "provider-secret", + "enabled": True, + "models": [ + { + "id": "lab-model", + "name": "Lab Model", + "model_id": "vendor/model", + "enabled": True, + } + ], + } + ] + } + assert ( + client.put( + "/api/provider-profiles", headers=headers, json=provider_payload + ).status_code + == 200 + ) + + response = client.put( + "/api/default-model", + headers=headers, + json={"model": "lab-model", "provider": "lab-openai"}, + ) + + assert response.status_code == 200 + saved = load_config() + assert (saved.model, saved.provider) == ("lab-model", "lab-openai") + + +def test_default_model_api_rejects_unconfigured_pair(tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + + response = client.put( + "/api/default-model", + headers={"X-EvoScientist-Admin-Token": "admin-secret"}, + json={"model": "missing-model", "provider": "missing-provider"}, + ) + + assert response.status_code == 400 + assert "is not configured" in response.json()["error"] + + +def test_default_model_api_rejects_invalid_json(monkeypatch): + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "admin-secret") + response = client.put( + "/api/default-model", + headers={ + "X-EvoScientist-Admin-Token": "admin-secret", + "Content-Type": "application/json", + }, + content="{invalid", + ) + assert response.status_code == 400 + assert response.json() == {"error": "Request body must be valid JSON."} + + def test_entries_preserve_registry_order(): """The picker uses position-in-list to rank providers per short name — the JSON must preserve the order returned by ``list_models_by_provider``. @@ -67,22 +772,55 @@ def test_entries_preserve_registry_order(): assert resp.json()["entries"] == expected -def test_default_passes_through_arbitrary_config_pair(): - """If config.yaml names a (name, provider) pair that isn't in the - registry (typo, retired model), still report it as default — the - picker labels it as the active selection regardless. - """ +def test_unavailable_default_falls_back_to_first_picker_entry(): mock_cfg = EvoScientistConfig(model="some-retired-name", provider="some-provider") with patch( "EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg ): resp = client.get("/api/models") + first = resp.json()["entries"][0] assert resp.json()["default"] == { - "name": "some-retired-name", - "provider": "some-provider", + "name": first["name"], + "provider": first["provider"], } +def test_custom_registry_default_does_not_reuse_stale_builtin_provider(): + replace_provider_profiles( + { + "providers": [ + { + "id": "open", + "name": "Open proxy", + "adapter": "openai", + "base_url": "https://proxy.example.test/v1", + "api_key": "provider-secret", + "enabled": True, + "models": [ + { + "id": "gpt-5.5", + "name": "GPT 5.5", + "model_id": "gpt-5.5", + "enabled": True, + } + ], + } + ] + } + ) + mock_cfg = EvoScientistConfig(model="gpt-5.5", provider="openai") + + with patch( + "EvoScientist.langgraph_dev.http.get_effective_config", return_value=mock_cfg + ): + body = client.get("/api/models").json() + + assert body["entries"] == [ + {"name": "gpt-5.5", "model_id": "gpt-5.5", "provider": "open"} + ] + assert body["default"] == {"name": "gpt-5.5", "provider": "open"} + + def test_ollama_models_appended_when_base_url_configured(): """Mirrors the TUI ``/model`` picker: when ``ollama_base_url`` is set, locally-pulled Ollama models are appended after the static registry diff --git a/tests/test_langgraph_dev_workspace_sidecar.py b/tests/test_langgraph_dev_workspace_sidecar.py index ed14109..1eb1e6d 100644 --- a/tests/test_langgraph_dev_workspace_sidecar.py +++ b/tests/test_langgraph_dev_workspace_sidecar.py @@ -112,7 +112,11 @@ def test_read_workspace_sidecar_round_trip(tmp_path, monkeypatch, runtime_paths) workspace = tmp_path / "x" / "y" manager._write_workspace_sidecar(workspace_dir=workspace, pid=42) data = manager._read_workspace_sidecar() - assert data == {"workspace": str(workspace), "pid": 42} + assert data is not None + assert data["workspace"] == str(workspace) + assert data["pid"] == 42 + assert data["api_url"] == "http://localhost:6174" + assert isinstance(data["deployment_id"], str) def test_workspace_mismatch_error_is_runtime_error_subclass(): diff --git a/tests/test_langgraph_manager.py b/tests/test_langgraph_manager.py index 353013c..18e19f6 100644 --- a/tests/test_langgraph_manager.py +++ b/tests/test_langgraph_manager.py @@ -294,6 +294,7 @@ class TestEnsureLanggraphDev: with ( patch.object(manager, "is_langgraph_dev_running", return_value=False), patch.object(manager, "start_langgraph_dev", return_value=proc) as start, + patch.object(manager.atexit, "register") as register_exit, patch.object( manager, "RUNTIME", @@ -303,10 +304,14 @@ class TestEnsureLanggraphDev: ), ), ): + runtime = manager.RUNTIME result = manager.ensure_langgraph_dev(cfg, workspace_dir=tmp_path) assert result is proc start.assert_called_once() + register_exit.assert_called_once_with( + manager.stop_langgraph_dev, proc, runtime=runtime + ) assert manager.is_async_subagents_available() is True def test_skips_when_async_and_memory_workers_disabled( diff --git a/tests/test_model_command.py b/tests/test_model_command.py index 35141f7..02c21b6 100644 --- a/tests/test_model_command.py +++ b/tests/test_model_command.py @@ -740,6 +740,10 @@ class TestModelCommandOllamaPicker: """Verify Ollama discovery augments the picker entries and the sentinel is always present when Ollama is configured.""" + @pytest.fixture(autouse=True) + def _isolate_provider_registry(self, tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + def _make_ctx_and_cfg(self, *, ollama_base_url: str | None): cfg = SimpleNamespace( model="claude-sonnet-4-6", diff --git a/tests/test_observation_memory.py b/tests/test_observation_memory.py index 5989034..4c8662e 100644 --- a/tests/test_observation_memory.py +++ b/tests/test_observation_memory.py @@ -121,6 +121,7 @@ def _memory_source_context( project_id: str = "P-project", source_agent: str = "EvoScientist", session_id: str = "thread-1", + turn_id: str | None = None, trajectory: list[source_context.CompactMessage] | None = None, ) -> source_context.MemorySourceContext: context_trajectory = trajectory or [{"role": "human", "content": "hi"}] @@ -131,6 +132,7 @@ def _memory_source_context( project_id=project_id, source_agent=source_agent, session_id=session_id, + turn_id=turn_id, trajectory=context_trajectory, trajectory_digest=source_context._trajectory_digest(context_trajectory), ) @@ -1781,6 +1783,7 @@ def test_subagent_summary_writer_uses_worker_metadata(tmp_path, monkeypatch): def test_memory_worker_run_payload_use_server_thread_id_and_source_metadata( monkeypatch, ): + monkeypatch.setenv("EVOSCIENTIST_USAGE_TRACKING", "true") monkeypatch.setattr( memory_launch, "_worker_workspace_dir", @@ -1805,6 +1808,7 @@ def test_memory_worker_run_payload_use_server_thread_id_and_source_metadata( assert kwargs["assistant_id"] == memory_launch.SUBAGENT_MEMORY_WORKER_GRAPH_ID assert kwargs["metadata"] == { "run_kind": "evomemory_subagent_worker", + "usage_scope": "memory", "source_session_id": "thread-1", "source_agent": "writing-agent", "project_id": "P-project", @@ -1825,6 +1829,29 @@ def test_memory_worker_run_payload_use_server_thread_id_and_source_metadata( } +def test_memory_worker_propagates_optional_usage_turn(monkeypatch): + monkeypatch.setenv("EVOSCIENTIST_USAGE_TRACKING", "true") + monkeypatch.setattr( + memory_launch, + "_worker_workspace_dir", + lambda _workspace_dir: "/tmp/ws", + ) + context = _memory_source_context( + memory_dir="/memories", + workspace_dir="/active/workspace", + turn_id="human-message-fixture", + ) + kwargs = memory_launch._memory_worker_run_payload( + context=context, + thread_id="worker-thread", + ) + assert kwargs["metadata"]["turn_id"] == "human-message-fixture" + assert ( + kwargs["config"]["configurable"]["evomemory_source_turn_id"] + == "human-message-fixture" + ) + + def test_memory_worker_finish_launches_linker_for_new_observations( tmp_path, ): @@ -2440,6 +2467,7 @@ def test_memory_worker_skips_when_langgraph_dev_unavailable(tmp_path, monkeypatc def test_memory_worker_marks_active_status(tmp_path, monkeypatch): + monkeypatch.setenv("EVOSCIENTIST_USAGE_TRACKING", "true") monkeypatch.setattr( background_runs, "default_background_run_url", lambda: "http://x" ) @@ -2480,6 +2508,7 @@ def test_memory_worker_marks_active_status(tmp_path, monkeypatch): assert worker_activity.memory_worker_status().is_running is True expected_metadata = { "run_kind": "evomemory_turn_worker", + "usage_scope": "memory", "source_session_id": "thread-1", "source_agent": "EvoScientist", "project_id": "P-project", diff --git a/tests/test_provider_admin.py b/tests/test_provider_admin.py new file mode 100644 index 0000000..c50d7d9 --- /dev/null +++ b/tests/test_provider_admin.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import os + +from EvoScientist.config.provider_admin import ( + get_provider_admin_token, + get_provider_admin_token_path, +) + + +def test_explicit_provider_admin_token_wins(monkeypatch, tmp_path): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + monkeypatch.setenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", "explicit-token") + + assert get_provider_admin_token() == "explicit-token" + assert not get_provider_admin_token_path().exists() + + +def test_provider_admin_token_is_created_and_reused(monkeypatch, tmp_path): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False) + + first = get_provider_admin_token() + second = get_provider_admin_token() + path = get_provider_admin_token_path() + + assert first + assert first == second + assert path.read_text(encoding="utf-8").strip() == first + if os.name != "nt": + assert path.parent.stat().st_mode & 0o777 == 0o700 + assert path.stat().st_mode & 0o777 == 0o600 + + +def test_provider_admin_token_reads_existing_file(monkeypatch, tmp_path): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + monkeypatch.delenv("EVOSCIENTIST_PROVIDER_ADMIN_TOKEN", raising=False) + path = get_provider_admin_token_path() + path.parent.mkdir(parents=True) + path.write_text("shared-token\n", encoding="utf-8") + + assert get_provider_admin_token() == "shared-token" diff --git a/tests/test_provider_operations.py b/tests/test_provider_operations.py new file mode 100644 index 0000000..55b4b74 --- /dev/null +++ b/tests/test_provider_operations.py @@ -0,0 +1,280 @@ +"""Tests for provider-specific model discovery and connectivity checks.""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import patch + +import httpx +import pytest + +from EvoScientist.config.provider_profiles import ProviderModel, ProviderProfile +from EvoScientist.llm.provider_operations import ( + ProviderOperationError, + discover_provider_models, +) +from EvoScientist.llm.provider_operations import ( + test_provider_model as run_provider_model_test, +) + + +def _profile( + adapter: str, + *, + base_url: str = "", + api_key: str = "provider-secret", +) -> ProviderProfile: + return ProviderProfile( + id="test-provider", + name="Test provider", + adapter=adapter, + base_url=base_url, + api_key=api_key, + enabled=True, + models=(), + ) + + +@pytest.mark.asyncio +async def test_openai_discovery_uses_bearer_token_and_models_endpoint(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.url == "https://api.openai.com/v1/models" + assert request.headers["Authorization"] == "Bearer provider-secret" + return httpx.Response( + 200, + json={ + "data": [ + {"id": "gpt-z"}, + {"id": "gpt-a", "name": "GPT A"}, + ] + }, + ) + + models = await discover_provider_models( + _profile("openai"), transport=httpx.MockTransport(handler) + ) + + assert [(model.model_id, model.name) for model in models] == [ + ("gpt-a", "GPT A"), + ("gpt-z", "gpt-z"), + ] + + +@pytest.mark.asyncio +async def test_anthropic_discovery_uses_native_headers_and_pagination(): + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + assert request.url.path == "/v1/models" + assert request.headers["x-api-key"] == "provider-secret" + assert request.headers["anthropic-version"] == "2023-06-01" + if request.url.params.get("after_id"): + return httpx.Response( + 200, + json={ + "data": [{"id": "claude-opus", "display_name": "Claude Opus"}], + "has_more": False, + }, + ) + return httpx.Response( + 200, + json={ + "data": [{"id": "claude-sonnet"}], + "has_more": True, + "last_id": "claude-sonnet", + }, + ) + + models = await discover_provider_models( + _profile("anthropic"), transport=httpx.MockTransport(handler) + ) + + assert len(requests) == 2 + assert requests[1].url.params["after_id"] == "claude-sonnet" + assert [model.model_id for model in models] == ["claude-opus", "claude-sonnet"] + + +@pytest.mark.asyncio +async def test_google_discovery_filters_non_generation_models_and_paginates(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/v1beta/models" + assert request.headers["x-goog-api-key"] == "provider-secret" + if request.url.params.get("pageToken"): + return httpx.Response( + 200, + json={ + "models": [ + { + "name": "models/gemini-2.5-pro", + "displayName": "Gemini 2.5 Pro", + "supportedGenerationMethods": ["generateContent"], + } + ] + }, + ) + return httpx.Response( + 200, + json={ + "models": [ + { + "name": "models/text-embedding-004", + "supportedGenerationMethods": ["embedContent"], + }, + { + "name": "models/gemini-2.5-flash", + "supportedGenerationMethods": ["generateContent"], + }, + ], + "nextPageToken": "next-page", + }, + ) + + models = await discover_provider_models( + _profile("google-genai"), transport=httpx.MockTransport(handler) + ) + + assert [(model.model_id, model.name) for model in models] == [ + ("gemini-2.5-flash", "models/gemini-2.5-flash"), + ("gemini-2.5-pro", "Gemini 2.5 Pro"), + ] + + +@pytest.mark.asyncio +async def test_grok_uses_xai_default_and_antigravity_uses_configured_proxy(): + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json={"data": [{"id": "grok-4"}]}) + + transport = httpx.MockTransport(handler) + await discover_provider_models(_profile("grok"), transport=transport) + await discover_provider_models( + _profile("antigravity", base_url="http://127.0.0.1:8317/v1"), + transport=transport, + ) + + assert str(requests[0].url) == "https://api.x.ai/v1/models" + assert str(requests[1].url) == "http://127.0.0.1:8317/v1/models" + + +@pytest.mark.asyncio +async def test_ollama_discovery_uses_tags_without_api_key(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.url == "http://127.0.0.1:11434/api/tags" + assert "Authorization" not in request.headers + return httpx.Response( + 200, + json={"models": [{"name": "qwen3:8b"}, {"model": "llama3.3:latest"}]}, + ) + + models = await discover_provider_models( + _profile("ollama", api_key=""), transport=httpx.MockTransport(handler) + ) + + assert [model.model_id for model in models] == ["llama3.3:latest", "qwen3:8b"] + + +@pytest.mark.asyncio +async def test_discovery_requires_native_provider_api_key(): + with pytest.raises(ProviderOperationError, match="API key is required"): + await discover_provider_models(_profile("openai", api_key="")) + + +@pytest.mark.asyncio +async def test_provider_http_error_surfaces_message_without_response_body_dump(): + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response( + 401, + json={"error": {"message": "Invalid credential"}, "secret": "hidden"}, + ) + + with pytest.raises( + ProviderOperationError, match="Provider returned HTTP 401: Invalid credential" + ): + await discover_provider_models( + _profile("openai"), transport=httpx.MockTransport(handler) + ) + + +@pytest.mark.asyncio +async def test_bare_openai_origin_uses_v1_and_reports_non_json_content_type(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.url == "https://api.example.test/v1/models" + return httpx.Response( + 200, + text="", + headers={"content-type": "text/html; charset=utf-8"}, + ) + + with pytest.raises(ProviderOperationError) as raised: + await discover_provider_models( + _profile("openai", base_url="https://api.example.test"), + transport=httpx.MockTransport(handler), + ) + + message = str(raised.value) + assert "https://api.example.test/v1/models" in message + assert "Content-Type: text/html" in message + assert "normally end in /v1" in message + + +@pytest.mark.asyncio +async def test_discovery_retries_a_transient_connection_failure(): + attempts = 0 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal attempts + attempts += 1 + if attempts == 1: + raise httpx.ConnectTimeout("temporary timeout", request=request) + return httpx.Response(200, json={"data": [{"id": "gpt-recovered"}]}) + + models = await discover_provider_models( + _profile("openai"), transport=httpx.MockTransport(handler) + ) + + assert attempts == 2 + assert [model.model_id for model in models] == ["gpt-recovered"] + + +@pytest.mark.asyncio +async def test_model_test_builds_draft_model_and_returns_text(): + class FakeChatModel: + async def ainvoke(self, prompt: str): + assert prompt == "Reply with exactly: OK" + return SimpleNamespace(content=[{"type": "text", "text": "OK"}]) + + profile = _profile("grok") + model = ProviderModel( + id="grok-test", name="Grok test", model_id="grok-4", enabled=True + ) + with patch( + "EvoScientist.llm.models.get_profile_chat_model", return_value=FakeChatModel() + ) as build_model: + result = await run_provider_model_test(profile, model) + + assert result.response == "OK" + assert result.latency_ms >= 1 + build_model.assert_called_once_with( + profile, + model, + max_tokens=64, + skip_auto_config=True, + ) + + +@pytest.mark.asyncio +async def test_model_test_redacts_api_key_from_errors(): + profile = _profile("openai", api_key="secret-value") + model = ProviderModel(id="test", name="Test", model_id="gpt-test") + with patch( + "EvoScientist.llm.models.get_profile_chat_model", + side_effect=ValueError("request rejected for secret-value"), + ): + with pytest.raises(ProviderOperationError) as raised: + await run_provider_model_test(profile, model) + + assert "secret-value" not in str(raised.value) + assert "[redacted]" in str(raised.value) diff --git a/tests/test_provider_profiles.py b/tests/test_provider_profiles.py new file mode 100644 index 0000000..aef9370 --- /dev/null +++ b/tests/test_provider_profiles.py @@ -0,0 +1,453 @@ +"""Tests for runtime-editable provider profiles.""" + +from __future__ import annotations + +import os +from unittest.mock import patch + +import pytest + +from EvoScientist.config import EvoScientistConfig, apply_config_to_env, save_config +from EvoScientist.config.provider_profiles import ( + ProviderProfileError, + get_builtin_provider_profile, + get_provider_profile_revision, + get_provider_profiles_path, + list_configured_model_entries, + load_provider_profiles, + provider_profiles_public, + replace_provider_profiles, + resolve_provider_profile_draft, +) +from EvoScientist.llm.models import get_chat_model, list_model_picker_entries + + +@pytest.fixture +def provider_config_dir(tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + return tmp_path / "evoscientist" + + +def _document(api_key: str = "secret-key") -> dict: + return { + "providers": [ + { + "id": "lab-openai", + "name": "Lab OpenAI", + "adapter": "openai-compatible", + "base_url": "https://llm.example.test/v1/", + "api_key": api_key, + "enabled": True, + "models": [ + { + "id": "research-model", + "name": "Research Model", + "model_id": "vendor/research-1", + "enabled": True, + }, + { + "id": "disabled-model", + "name": "Disabled Model", + "model_id": "vendor/disabled", + "enabled": False, + }, + ], + } + ] + } + + +def _builtin_document(api_key: str = "builtin-secret") -> dict: + return { + "builtins": [ + { + "id": "openai", + "name": "OpenAI", + "adapter": "openai", + "base_url": "https://proxy.example.test/v1", + "api_key": api_key, + "auth_mode": "api_key", + "enabled": True, + "models": [ + { + "id": "chat-main", + "name": "Chat Main", + "model_id": "gpt-upstream", + "enabled": True, + } + ], + } + ] + } + + +def test_replace_round_trip_and_redacts_api_key(provider_config_dir): + saved = replace_provider_profiles(_document()) + + assert saved.providers[0].base_url == "https://llm.example.test/v1" + assert load_provider_profiles() == saved + public = provider_profiles_public(saved) + assert "api_key" not in public["providers"][0] + assert public["providers"][0]["api_key_configured"] is True + assert public["providers"][0]["api_key_hint"] == "sec...-key" + assert "openai" in public["reserved_provider_ids"] + + path = get_provider_profiles_path() + assert path == provider_config_dir / "providers.yaml" + if os.name != "nt": + assert path.parent.stat().st_mode & 0o777 == 0o700 + assert path.stat().st_mode & 0o777 == 0o600 + + +def test_v1_document_loads_and_saves_as_v2(provider_config_dir): + path = get_provider_profiles_path() + path.parent.mkdir(parents=True) + path.write_text( + "version: 1\nproviders:\n" + " - id: lab-openai\n" + " name: Lab OpenAI\n" + " adapter: openai-compatible\n" + " base_url: https://llm.example.test/v1\n" + " api_key: old-secret\n" + " enabled: true\n" + " models: []\n", + encoding="utf-8", + ) + + loaded = load_provider_profiles() + assert loaded.version == 2 + assert loaded.builtins == () + assert loaded.providers[0].id == "lab-openai" + + replace_provider_profiles({"providers": _document()["providers"]}) + assert path.read_text(encoding="utf-8").startswith("version: 2\nbuiltins:") + + +def test_builtin_profiles_round_trip_and_redact_secret(provider_config_dir): + saved = replace_provider_profiles(_builtin_document()) + + assert saved.builtins[0].id == "openai" + assert get_builtin_provider_profile("openai") == saved.builtins[0] + public = provider_profiles_public(saved) + assert public["builtins"][0]["api_key_configured"] is True + assert "builtin-secret" not in str(public) + + +def test_builtin_private_revision_changes_with_connection_secret( + provider_config_dir, +): + replace_provider_profiles(_builtin_document("first-secret")) + first = get_provider_profile_revision("openai") + + replace_provider_profiles(_builtin_document("second-secret")) + second = get_provider_profile_revision("openai") + + assert first + assert second + assert first != second + + +def test_custom_update_preserves_builtin_profiles(provider_config_dir): + replace_provider_profiles(_builtin_document()) + + saved = replace_provider_profiles(_document()) + + assert saved.builtins[0].id == "openai" + assert saved.providers[0].id == "lab-openai" + + +def test_builtin_update_preserves_custom_profiles(provider_config_dir): + replace_provider_profiles(_document()) + + saved = replace_provider_profiles(_builtin_document()) + + assert saved.builtins[0].id == "openai" + assert saved.providers[0].id == "lab-openai" + + +def test_empty_api_key_keeps_existing_secret(provider_config_dir): + replace_provider_profiles(_document("first-secret")) + update = _document("") + update["providers"][0]["name"] = "Renamed" + + saved = replace_provider_profiles(update) + + assert saved.providers[0].name == "Renamed" + assert saved.providers[0].api_key == "first-secret" + + +def test_public_revision_does_not_fingerprint_api_key(provider_config_dir): + first = provider_profiles_public( + replace_provider_profiles(_document("abc-first-1234")) + )["revision"] + second = provider_profiles_public( + replace_provider_profiles(_document("abc-second-1234")) + )["revision"] + + assert first == second + + +def test_clear_api_key_is_explicit(provider_config_dir): + replace_provider_profiles(_document("first-secret")) + update = _document("") + update["providers"][0]["clear_api_key"] = True + + saved = replace_provider_profiles(update) + + assert saved.providers[0].api_key == "" + + +def test_draft_reuses_saved_api_key_without_exposing_it(provider_config_dir): + replace_provider_profiles(_document("saved-secret")) + draft = _document("")["providers"][0] + draft["models"] = [] + + resolved = resolve_provider_profile_draft(draft) + + assert resolved.api_key == "saved-secret" + assert resolved.models == () + + +def test_draft_can_explicitly_clear_saved_api_key(provider_config_dir): + replace_provider_profiles(_document("saved-secret")) + draft = _document("")["providers"][0] + draft.update({"models": [], "clear_api_key": True}) + + resolved = resolve_provider_profile_draft(draft) + + assert resolved.api_key == "" + + +def test_reserved_provider_id_is_rejected(provider_config_dir): + raw = _document() + raw["providers"][0]["id"] = "openai" + with pytest.raises(ProviderProfileError, match="openai-custom"): + replace_provider_profiles(raw) + + +@pytest.mark.parametrize( + ("adapter", "expected"), + [ + ("openai", "https://llm.example.test/v1"), + ("openai-compatible", "https://llm.example.test/v1"), + ("openrouter", "https://llm.example.test/api/v1"), + ("google-genai", "https://llm.example.test/v1beta"), + ("anthropic", "https://llm.example.test"), + ], +) +def test_bare_provider_origin_gets_adapter_api_path( + provider_config_dir, adapter, expected +): + raw = _document() + raw["providers"][0].update( + { + "id": "lab-provider", + "adapter": adapter, + "base_url": "https://llm.example.test", + } + ) + + saved = replace_provider_profiles(raw) + + assert saved.providers[0].base_url == expected + + +def test_dynamic_models_are_listed_and_disabled_models_are_hidden( + provider_config_dir, +): + replace_provider_profiles(_document()) + + assert list_configured_model_entries() == [ + ("research-model", "vendor/research-1", "lab-openai") + ] + + +@pytest.mark.asyncio +async def test_custom_registry_replaces_static_picker_entries(provider_config_dir): + replace_provider_profiles(_document()) + + entries = await list_model_picker_entries(None, include_custom_ollama=False) + + assert entries == [("research-model", "vendor/research-1", "lab-openai")] + + +@pytest.mark.asyncio +async def test_empty_registry_hides_static_picker_entries(provider_config_dir): + replace_provider_profiles({"providers": []}) + + entries = await list_model_picker_entries(None, include_custom_ollama=False) + + assert entries == [] + + +@pytest.mark.asyncio +async def test_managed_builtins_replace_static_picker_entries(provider_config_dir): + replace_provider_profiles(_builtin_document()) + + entries = await list_model_picker_entries(None, include_custom_ollama=False) + + assert entries == [("chat-main", "gpt-upstream", "openai")] + + +def test_dynamic_profile_routes_through_selected_adapter(provider_config_dir): + replace_provider_profiles(_document()) + + with patch("EvoScientist.llm.models.init_chat_model") as init_model: + get_chat_model("research-model", provider="lab-openai") + + _, kwargs = init_model.call_args + assert kwargs["model"] == "vendor/research-1" + assert kwargs["model_provider"] == "openai" + assert kwargs["base_url"] == "https://llm.example.test/v1" + assert kwargs["api_key"] == "secret-key" + assert kwargs["default_headers"]["User-Agent"] == "codex_cli_rs/0.0.0" + + +def test_dynamic_native_openai_adapter_uses_gateway_compatible_user_agent( + provider_config_dir, +): + raw = _document() + raw["providers"][0]["adapter"] = "openai" + replace_provider_profiles(raw) + + with patch("EvoScientist.llm.models.init_chat_model") as init_model: + get_chat_model("research-model", provider="lab-openai") + + _, kwargs = init_model.call_args + assert kwargs["model_provider"] == "openai" + assert kwargs["default_headers"]["User-Agent"] == "codex_cli_rs/0.0.0" + + +def test_builtin_catalog_alias_routes_to_upstream_model(provider_config_dir): + save_config( + EvoScientistConfig( + provider="openai", + model="chat-main", + model_catalog=[ + { + "provider": "openai", + "id": "chat-main", + "name": "Chat Main", + "model_id": "gpt-upstream", + "enabled": True, + } + ], + ) + ) + + with patch("EvoScientist.llm.models.init_chat_model") as init_model: + get_chat_model("chat-main", provider="openai") + + _, kwargs = init_model.call_args + assert kwargs["model"] == "gpt-upstream" + assert kwargs["model_provider"] == "openai" + + +def test_managed_builtin_routes_connection_without_restart( + provider_config_dir, monkeypatch +): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + replace_provider_profiles(_builtin_document()) + + with patch("EvoScientist.llm.models.init_chat_model") as init_model: + get_chat_model("chat-main", provider="openai") + + _, kwargs = init_model.call_args + assert kwargs["model"] == "gpt-upstream" + assert kwargs["model_provider"] == "openai" + assert kwargs["base_url"] == "https://proxy.example.test/v1" + assert kwargs["api_key"] == "builtin-secret" + + +def test_environment_overrides_managed_builtin_connection( + provider_config_dir, monkeypatch +): + replace_provider_profiles(_builtin_document()) + monkeypatch.setenv("OPENAI_API_KEY", "environment-secret") + monkeypatch.setenv("OPENAI_BASE_URL", "https://env.example.test/v1") + + with patch("EvoScientist.llm.models.init_chat_model") as init_model: + get_chat_model("chat-main", provider="openai") + + _, kwargs = init_model.call_args + assert kwargs["base_url"] == "https://env.example.test/v1" + assert kwargs["api_key"] == "environment-secret" + + +def test_managed_builtin_overrides_legacy_config_injected_environment( + provider_config_dir, monkeypatch +): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + apply_config_to_env( + EvoScientistConfig( + openai_api_key="legacy-secret", + ) + ) + replace_provider_profiles(_builtin_document("registry-secret")) + + with patch("EvoScientist.llm.models.init_chat_model") as init_model: + get_chat_model("chat-main", provider="openai") + + _, kwargs = init_model.call_args + assert kwargs["api_key"] == "registry-secret" + + +def test_dynamic_ollama_profile_does_not_pass_api_key(provider_config_dir): + raw = _document("unused-secret") + raw["providers"][0].update( + { + "id": "local-ollama", + "adapter": "ollama", + "base_url": "http://127.0.0.1:11434", + } + ) + replace_provider_profiles(raw) + + with patch("EvoScientist.llm.models.init_chat_model") as init_model: + get_chat_model("research-model", provider="local-ollama") + + _, kwargs = init_model.call_args + assert kwargs["model_provider"] == "ollama" + assert kwargs["base_url"] == "http://127.0.0.1:11434" + assert "api_key" not in kwargs + + +@pytest.mark.parametrize( + ("adapter", "base_url"), + [ + ("grok", ""), + ("antigravity", "http://127.0.0.1:8317/v1"), + ], +) +def test_openai_protocol_adapters_route_through_openai( + provider_config_dir, adapter, base_url +): + raw = _document() + raw["providers"][0].update( + { + "id": f"lab-{adapter}", + "adapter": adapter, + "base_url": base_url, + } + ) + replace_provider_profiles(raw) + + with patch("EvoScientist.llm.models.init_chat_model") as init_model: + get_chat_model("research-model", provider=f"lab-{adapter}") + + _, kwargs = init_model.call_args + assert kwargs["model_provider"] == "openai" + assert kwargs["base_url"] == (base_url or "https://api.x.ai/v1") + + +def test_revision_changes_when_credentials_change(provider_config_dir): + replace_provider_profiles(_document("first-secret")) + first = get_provider_profile_revision("lab-openai") + replace_provider_profiles(_document("second-secret")) + second = get_provider_profile_revision("lab-openai") + + assert first + assert second + assert first != second diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 96ff698..a0f3c71 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -19,7 +19,7 @@ def test_schedules_block_lists_active_crons(monkeypatch): monkeypatch.setattr( crons, "list_schedules", - lambda: [ + lambda scope=None: [ { "cron_id": "abc12345-xyz", "schedule": "*/10 * * * *", @@ -48,7 +48,7 @@ def test_schedules_block_empty_on_error(monkeypatch): monkeypatch.setattr(crons, "is_available", lambda: True) - def _boom(): + def _boom(scope=None): raise RuntimeError("backend died") monkeypatch.setattr(crons, "list_schedules", _boom) diff --git a/tests/test_scheduler_tools.py b/tests/test_scheduler_tools.py index 91975e4..014f6d0 100644 --- a/tests/test_scheduler_tools.py +++ b/tests/test_scheduler_tools.py @@ -3,6 +3,20 @@ from unittest.mock import patch +def test_scheduler_tool_schemas_are_json_serializable(): + from langchain_core.utils.function_calling import convert_to_openai_tool + + from EvoScientist.middleware.scheduler import ( + cancel_scheduled_task, + list_scheduled_tasks, + schedule_task, + ) + + for tool in (schedule_task, list_scheduled_tasks, cancel_scheduled_task): + schema = convert_to_openai_tool(tool) + assert "runtime" not in schema["function"]["parameters"].get("properties", {}) + + def test_schedule_task_translates_and_creates(): from EvoScientist.middleware.scheduler import schedule_task diff --git a/tests/test_usage_tracking.py b/tests/test_usage_tracking.py new file mode 100644 index 0000000..006b248 --- /dev/null +++ b/tests/test_usage_tracking.py @@ -0,0 +1,552 @@ +from __future__ import annotations + +import json +import os +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import MagicMock, patch +from uuid import uuid4 + +import pytest +from langchain_core.language_models.fake_chat_models import FakeListChatModel +from langchain_core.messages import AIMessage, AIMessageChunk +from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, LLMResult +from pydantic import ValidationError + +from EvoScientist.usage.callback import ( + UsageCaptureCallback, + UsageModelIdentity, + _scope, + attach_usage_callback, +) +from EvoScientist.usage.identity import ( + _load_or_create, + normalize_workspace_path_v1, + prepare_usage_environment, + workspace_identity, + workspace_identity_from_normalized, +) +from EvoScientist.usage.schema import UsageEventV1 +from EvoScientist.usage.spool import UsageSpool + +IDENTITY = UsageModelIdentity( + provider_profile_id="profile-a", + provider_revision="revision-a", + provider_adapter="openai", + model_alias="chat-main", + upstream_model_id="upstream-a", +) +FIXTURES = Path( + os.getenv( + "EVOSCIENTIST_USAGE_FIXTURES", + str( + Path(__file__).parents[2] + / "EvoScientist-WebUI" + / "docs" + / "schemas" + / "fixtures" + ), + ) +) + + +def _fixture_event() -> UsageEventV1: + fixture = FIXTURES / "accepted" / "confirmed.json" + return UsageEventV1.model_validate_json(fixture.read_text(encoding="utf-8")) + + +def _usage_environment(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + values = { + "EVOSCIENTIST_USAGE_TRACKING": "true", + "EVOSCIENTIST_USAGE_SINK_URL": "http://127.0.0.1:1/api/usage/events", + "EVOSCIENTIST_USAGE_SINK_TOKEN": "test-token", + "EVOSCIENTIST_DEPLOYMENT_ID": "11111111-1111-4111-8111-111111111111", + "EVOSCIENTIST_WORKSPACE_ID": "ws1_fixture", + "EVOSCIENTIST_USAGE_SPOOL_DIR": str(tmp_path / "spool"), + "EVOSCIENTIST_WORKSPACE_DIR": str(tmp_path), + } + for key, value in values.items(): + monkeypatch.setenv(key, value) + + +def test_prepare_usage_environment_rejects_invalid_webui_port(tmp_path: Path) -> None: + with pytest.raises(ValueError, match="webui_port"): + prepare_usage_environment(tmp_path, webui_port=0) + + +@pytest.mark.parametrize("name", ["confirmed", "unknown"]) +def test_shared_accepted_fixtures_match_python_contract(name: str) -> None: + fixture = FIXTURES / "accepted" / f"{name}.json" + event = UsageEventV1.model_validate_json(fixture.read_text(encoding="utf-8")) + assert event.usage_status == name + assert event.model_dump(mode="json")["completed_at"].endswith("Z") + + +def test_shared_rejected_fixtures_match_python_contract() -> None: + rejected = json.loads( + (FIXTURES / "rejected" / "unsafe-token-sum.json").read_text(encoding="utf-8") + ) + base = json.loads( + (FIXTURES / "rejected" / rejected["base_fixture"]).read_text(encoding="utf-8") + ) + base.update(rejected["patch"]) + with pytest.raises(ValidationError, match="input plus output tokens"): + UsageEventV1.model_validate(base) + + +def test_unknown_usage_rejects_numeric_tokens() -> None: + data = _fixture_event().model_dump() + data.update( + usage_status="unknown", + input_tokens=0, + output_tokens=None, + provider_total_tokens=None, + ) + with pytest.raises(ValidationError, match="unknown usage"): + UsageEventV1.model_validate(data) + + +def test_python_contract_rejects_missing_and_coerced_fields() -> None: + data = _fixture_event().model_dump(mode="json") + data.pop("turn_id") + with pytest.raises(ValidationError, match="turn_id"): + UsageEventV1.model_validate(data) + data = _fixture_event().model_dump(mode="json") + data["input_tokens"] = "1000" + with pytest.raises(ValidationError, match="input_tokens"): + UsageEventV1.model_validate(data) + + data = _fixture_event().model_dump(mode="json") + data.update(started_at=None, observed_at=0, completed_at=0) + with pytest.raises(ValidationError, match="observed_at"): + UsageEventV1.model_validate(data) + + +def test_model_copy_preflight_attaches_callback_without_mutating_source( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _usage_environment(monkeypatch, tmp_path) + source = FakeListChatModel(responses=["ok"]) + with patch("EvoScientist.usage.callback.get_usage_spool") as get_spool: + tracked = attach_usage_callback(source, IDENTITY) + get_spool.assert_called_once_with() + assert tracked is not source + assert not source.callbacks + assert any( + isinstance(item, UsageCaptureCallback) for item in tracked.callbacks or [] + ) + selector = tracked.model_copy( + update={ + "metadata": {**(tracked.metadata or {}), "usage_scope": "tool_selector"} + } + ) + assert selector.metadata["usage_scope"] == "tool_selector" + + +def test_async_run_inherits_usage_context_only_when_tracking_enabled( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from EvoScientist.llm import patches + + monkeypatch.setenv("EVOSCIENTIST_USAGE_TRACKING", "true") + monkeypatch.setattr(patches, "_read_cfg_configurable", lambda: {}) + active = { + "metadata": { + "turn_id": "turn-a", + "thread_id": "thread-a", + "source_agent": "EvoScientist", + }, + "configurable": {"thread_id": "thread-a"}, + } + with patch("langgraph.config.get_config", return_value=active): + merged = patches._merge_runs_config_kwargs({"metadata": {"name": "writer"}}) + assert merged["metadata"] == { + "turn_id": "turn-a", + "source_agent": "EvoScientist", + "source_session_id": "thread-a", + "name": "writer", + "usage_scope": "async_subagent", + } + + +def test_callback_emits_one_confirmed_terminal_event( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _usage_environment(monkeypatch, tmp_path) + spool = MagicMock() + callback = UsageCaptureCallback(IDENTITY) + run_id = uuid4() + callback.on_chat_model_start( + {}, + [], + run_id=run_id, + metadata={ + "thread_id": "thread-a", + "turn_id": "turn-a", + "usage_scope": "tool_selector", + }, + ) + message = AIMessage( + content="ok", + usage_metadata={"input_tokens": 12, "output_tokens": 3, "total_tokens": 15}, + response_metadata={"request_id": "provider-request"}, + ) + result = LLMResult(generations=[[ChatGeneration(message=message)]]) + with patch("EvoScientist.usage.callback.get_usage_spool", return_value=spool): + callback.on_llm_end(result, run_id=run_id) + callback.on_llm_end(result, run_id=run_id) + spool.enqueue.assert_called_once() + event = spool.enqueue.call_args.args[0] + assert event.model_call_id == str(run_id) + assert event.scope == "tool_selector" + assert event.turn_id == "turn-a" + assert event.provider_request_id == "provider-request" + assert event.input_tokens == 12 + assert event.output_tokens == 3 + + +def test_callback_error_emits_unknown( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _usage_environment(monkeypatch, tmp_path) + spool = MagicMock() + callback = UsageCaptureCallback(IDENTITY) + run_id = uuid4() + callback.on_chat_model_start( + {}, [], run_id=run_id, metadata={"run_kind": "scheduled_task"} + ) + with patch("EvoScientist.usage.callback.get_usage_spool", return_value=spool): + callback.on_llm_error(RuntimeError("provider failed"), run_id=run_id) + event = spool.enqueue.call_args.args[0] + assert event.usage_status == "unknown" + assert event.input_tokens is None + assert event.scope == "scheduler" + + +def test_stream_usage_survives_terminal_error( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _usage_environment(monkeypatch, tmp_path) + spool = MagicMock() + callback = UsageCaptureCallback(IDENTITY) + run_id = uuid4() + callback.on_chat_model_start({}, [], run_id=run_id, metadata={}) + chunk = ChatGenerationChunk( + message=AIMessageChunk( + content="", + usage_metadata={"input_tokens": 8, "output_tokens": 2, "total_tokens": 10}, + ) + ) + callback.on_llm_new_token("", run_id=run_id, chunk=chunk) + with patch("EvoScientist.usage.callback.get_usage_spool", return_value=spool): + callback.on_llm_error(RuntimeError("stream interrupted"), run_id=run_id) + event = spool.enqueue.call_args.args[0] + assert event.usage_status == "confirmed" + assert event.input_tokens == 8 + assert event.output_tokens == 2 + + +def test_callback_uses_deepagents_name_for_sync_subagent_scope( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _usage_environment(monkeypatch, tmp_path) + spool = MagicMock() + callback = UsageCaptureCallback(IDENTITY) + run_id = uuid4() + callback.on_chat_model_start( + {}, [], run_id=run_id, metadata={"lc_agent_name": "writing-agent"} + ) + with patch("EvoScientist.usage.callback.get_usage_spool", return_value=spool): + callback.on_llm_error(RuntimeError("provider failed"), run_id=run_id) + assert spool.enqueue.call_args.args[0].scope == "sync_subagent" + + +@pytest.mark.parametrize( + ("metadata", "expected"), + [ + ({"usage_scope": "tool_selector"}, "tool_selector"), + ({"usage_scope": "diagnostic"}, "diagnostic"), + ({"usage_scope": "skill_eval"}, "skill_eval"), + ({"lc_source": "summarization"}, "summarizer"), + ({"run_kind": "scheduled_task"}, "scheduler"), + ({"run_kind": "evomemory_autoskills"}, "autoskills"), + ({"run_kind": "evomemory_turn_worker"}, "memory"), + ({"lc_agent_name": "EvoScientist"}, "main"), + ({"lc_agent_name": "writing-agent"}, "sync_subagent"), + ({"source_session_id": "origin-thread"}, "async_subagent"), + ({"thread_id": "thread-a"}, "main"), + ({}, "unattributed"), + ], +) +def test_scope_mapping_contract(metadata: dict, expected: str) -> None: + assert _scope(metadata) == expected + + +def test_provider_compatibility_replays_all_terminal_cases( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _usage_environment(monkeypatch, tmp_path) + compatibility = json.loads( + (FIXTURES / "providers" / "compatibility.json").read_text(encoding="utf-8") + ) + for provider in compatibility["providers"]: + for case in compatibility["cases"]: + identity = UsageModelIdentity( + provider_profile_id=provider["name"], + provider_revision="fixture", + provider_adapter=provider["adapter"], + model_alias="fixture", + upstream_model_id="fixture", + ) + callback = UsageCaptureCallback(identity) + spool = MagicMock() + run_id = uuid4() + callback.on_chat_model_start({}, [], run_id=run_id, metadata={}) + request_field = provider["request_id_field"].split(".")[-1] + if case.get("stream_usage"): + chunk = ChatGenerationChunk( + message=AIMessageChunk( + content="", + usage_metadata=case["stream_usage"], + response_metadata={request_field: "request-fixture"}, + ) + ) + callback.on_llm_new_token("", run_id=run_id, chunk=chunk) + with patch( + "EvoScientist.usage.callback.get_usage_spool", return_value=spool + ): + if case["terminal"] == "error": + callback.on_llm_error(RuntimeError("fixture error"), run_id=run_id) + else: + message = AIMessage( + content="ok", + usage_metadata=case.get("usage"), + response_metadata={request_field: "request-fixture"}, + ) + callback.on_llm_end( + LLMResult(generations=[[ChatGeneration(message=message)]]), + run_id=run_id, + ) + event = spool.enqueue.call_args.args[0] + assert event.usage_status == case["expected_status"] + if case["expected_status"] == "confirmed": + assert event.input_tokens is not None + assert event.output_tokens is not None + + +def test_workspace_identity_is_stable_and_deployment_scoped(tmp_path: Path) -> None: + first = workspace_identity("11111111-1111-4111-8111-111111111111", tmp_path) + second = workspace_identity("11111111-1111-4111-8111-111111111111", tmp_path) + other = workspace_identity("22222222-2222-4222-8222-222222222222", tmp_path) + assert first == second + assert first.startswith("ws1_") + assert first != other + + +def test_identity_file_creation_is_atomic_across_launchers(tmp_path: Path) -> None: + identity_path = tmp_path / "deployment-id" + with ThreadPoolExecutor(max_workers=16) as pool: + values = list( + pool.map( + lambda index: _load_or_create( + identity_path, lambda: f"launcher-{index}" + ), + range(64), + ) + ) + assert len(set(values)) == 1 + assert identity_path.read_text(encoding="utf-8").strip() == values[0] + if os.name != "nt": + assert identity_path.stat().st_mode & 0o777 == 0o600 + + +def test_callback_remains_fail_open_when_spool_write_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _usage_environment(monkeypatch, tmp_path) + callback = UsageCaptureCallback(IDENTITY) + run_id = uuid4() + callback.on_chat_model_start({}, [], run_id=run_id, metadata={}) + message = AIMessage( + content="ok", + usage_metadata={"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + ) + result = LLMResult(generations=[[ChatGeneration(message=message)]]) + spool = MagicMock() + spool.enqueue.side_effect = OSError("disk full") + with patch("EvoScientist.usage.callback.get_usage_spool", return_value=spool): + callback.on_llm_end(result, run_id=run_id) + + +def test_workspace_identity_normalization_contract(tmp_path: Path) -> None: + deployment = "11111111-1111-4111-8111-111111111111" + assert ( + normalize_workspace_path_v1("/tmp/research/", windows=False) == "/tmp/research" + ) + assert normalize_workspace_path_v1("/", windows=False) == "/" + assert normalize_workspace_path_v1("/tmp/Cafe\u0301", windows=False) == "/tmp/Café" + assert normalize_workspace_path_v1("C:\\Research\\", windows=True) == "c:/research" + assert normalize_workspace_path_v1("C:\\", windows=True) == "c:/" + assert ( + normalize_workspace_path_v1("\\\\Server\\Share\\Research\\", windows=True) + == "//server/share/research" + ) + normalized = normalize_workspace_path_v1("C:\\Research\\", windows=True) + assert workspace_identity_from_normalized(deployment, normalized).startswith("ws1_") + + real = tmp_path / "real" + real.mkdir() + link = tmp_path / "link" + link.symlink_to(real, target_is_directory=True) + assert workspace_identity(deployment, real) == workspace_identity(deployment, link) + + posix = json.loads( + (FIXTURES / "identity" / "workspace-posix.json").read_text(encoding="utf-8") + ) + assert ( + workspace_identity_from_normalized(deployment, posix["normalized_path"]) + == posix["expected_workspace_id"] + ) + windows = json.loads( + (FIXTURES / "identity" / "workspace-windows.json").read_text(encoding="utf-8") + ) + for case in windows["cases"]: + normalized_case = normalize_workspace_path_v1(case["input"], windows=True) + assert normalized_case == case["normalized_path"] + assert ( + workspace_identity_from_normalized(deployment, normalized_case) + == case["expected_workspace_id"] + ) + unicode_case = json.loads( + (FIXTURES / "identity" / "workspace-unicode.json").read_text(encoding="utf-8") + ) + normalized_unicode = normalize_workspace_path_v1( + unicode_case["input"], windows=False + ) + assert normalized_unicode == unicode_case["normalized_path"] + assert ( + workspace_identity_from_normalized(deployment, normalized_unicode) + == unicode_case["expected_workspace_id"] + ) + root_case = json.loads( + (FIXTURES / "identity" / "workspace-root-and-symlink.json").read_text( + encoding="utf-8" + ) + )["posix_root"] + assert ( + workspace_identity_from_normalized(deployment, root_case["normalized_path"]) + == root_case["expected_workspace_id"] + ) + + +def test_spool_enqueue_is_durable_without_waiting_for_http( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _usage_environment(monkeypatch, tmp_path) + monkeypatch.setattr(UsageSpool, "_run", lambda self: self._stop.wait()) + spool = UsageSpool() + try: + spool.enqueue(_fixture_event()) + pending = list(spool.pending.glob("*.json")) + assert len(pending) == 1 + assert json.loads(pending[0].read_text(encoding="utf-8"))["schema_version"] == 1 + finally: + spool.close() + + +def _event_with_id(event: UsageEventV1, model_call_id: str) -> UsageEventV1: + return event.model_copy( + update={ + "model_call_id": model_call_id, + "event_id": (f"{event.deployment_id}:{model_call_id}:callback_final:1"), + } + ) + + +def test_spool_multi_worker_same_event_is_idempotent( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _usage_environment(monkeypatch, tmp_path) + monkeypatch.setattr(UsageSpool, "_run", lambda self: self._stop.wait()) + first = UsageSpool() + second = UsageSpool() + try: + event = _fixture_event() + with ThreadPoolExecutor(max_workers=2) as pool: + list(pool.map(lambda spool: spool.enqueue(event), (first, second))) + assert len(list(first.pending.glob("*.json"))) == 1 + assert list(first.quarantine.glob("*.json")) == [] + finally: + first.close() + second.close() + + +def test_spool_recovers_stale_inflight_and_rejects_at_soft_limit( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + _usage_environment(monkeypatch, tmp_path) + monkeypatch.setenv("EVOSCIENTIST_USAGE_INFLIGHT_LEASE_SECONDS", "1") + monkeypatch.setenv("EVOSCIENTIST_USAGE_SPOOL_MAX_FILES", "1") + monkeypatch.setattr(UsageSpool, "_run", lambda self: self._stop.wait()) + spool = UsageSpool() + try: + event = _fixture_event() + spool.enqueue(event) + pending = next(spool.pending.glob("*.json")) + inflight = spool.inflight / pending.name + pending.replace(inflight) + old = time.time() - 5 + os.utime(inflight, (old, old)) + spool._recover_stale_inflight() + assert (spool.pending / inflight.name).exists() + + second = _event_with_id(event, "77777777-7777-4777-8777-777777777777") + spool.enqueue(second) + assert len(list(spool.pending.glob("*.json"))) == 1 + assert spool.first_loss_at is not None + assert spool.degraded_reason == "spool_soft_limit_reached" + finally: + spool.close() + + +@pytest.mark.parametrize( + ("status_code", "response_status", "expected_location", "progressed"), + [ + (200, "accepted", "deleted", True), + (200, "duplicate", "deleted", True), + (409, "conflict", "quarantine", True), + (422, "rejected", "quarantine", True), + (401, "unauthorized", "pending", False), + (429, "rate_limited", "pending", False), + (500, "error", "pending", False), + ], +) +def test_sender_response_state_machine( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + status_code: int, + response_status: str, + expected_location: str, + progressed: bool, +) -> None: + _usage_environment(monkeypatch, tmp_path) + monkeypatch.setattr(UsageSpool, "_run", lambda self: self._stop.wait()) + spool = UsageSpool() + response = SimpleNamespace( + status_code=status_code, + json=lambda: {"status": response_status}, + ) + client = SimpleNamespace(post=lambda *args, **kwargs: response) + try: + spool.enqueue(_fixture_event()) + assert spool._send_one(client) is progressed + if expected_location == "deleted": + assert list(spool.pending.glob("*.json")) == [] + assert list(spool.inflight.glob("*.json")) == [] + else: + directory = getattr(spool, expected_location) + assert len(list(directory.glob("*.json"))) == 1 + finally: + spool.close() diff --git a/tests/test_workspace_cutover.py b/tests/test_workspace_cutover.py new file mode 100644 index 0000000..4446116 --- /dev/null +++ b/tests/test_workspace_cutover.py @@ -0,0 +1,223 @@ +"""Tests for the required-mode migration gate.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from EvoScientist.scope_registry import deployment_id_for_workspace, get_scope_registry +from EvoScientist.workspace_cutover import ( + run_workspace_cutover, + verify_required_cutover, +) + + +class _Threads: + def __init__(self, threads: list[dict] | None = None) -> None: + self.updated: list[tuple[str, dict]] = [] + self._threads = threads or [ + { + "thread_id": "primary-thread", + "metadata": {"graph_id": "EvoScientist"}, + } + ] + + def search(self, *, limit: int, offset: int = 0): + assert limit == 100 + assert offset == 0 + return self._threads + + def update(self, thread_id: str, *, metadata: dict): + self.updated.append((thread_id, metadata)) + + +class _Crons: + def __init__(self, crons: list[dict] | None = None) -> None: + self._crons = crons or [] + self.updated: list[tuple[str, bool]] = [] + + def search(self, *, limit: int): + assert limit == 1000 + return self._crons + + def update(self, cron_id: str, *, enabled: bool): + self.updated.append((cron_id, enabled)) + + +class _Runs: + def __init__(self, runs: dict[str, list[dict]] | None = None) -> None: + self.runs = runs or {} + self.cancelled: list[tuple[str, str]] = [] + + def list(self, *, thread_id: str, limit: int): + assert limit == 1000 + return self.runs.get(thread_id, []) + + def cancel(self, thread_id: str, run_id: str, *, wait: bool, action: str): + assert wait is True + assert action == "interrupt" + self.cancelled.append((thread_id, run_id)) + for run in self.runs.get(thread_id, []): + if run["run_id"] == run_id: + run["status"] = "interrupted" + + +def test_cutover_writes_a_verified_passing_report(tmp_path, monkeypatch): + import EvoScientist.scope_registry as registry_module + + registry_module._registry_cache.clear() + monkeypatch.delenv("EVOSCIENTIST_DEPLOYMENT_ID", raising=False) + client = SimpleNamespace(threads=_Threads(), crons=_Crons(), runs=_Runs()) + report = run_workspace_cutover( + workspace_root=tmp_path, client=client, browser_sdk_gate=True + ) + + assert report["status"] == "passed" + assert client.threads.updated[0][1]["workspace_scope_id"] + verify_required_cutover(tmp_path) + + +def test_required_cutover_rejects_missing_report(tmp_path): + with pytest.raises(RuntimeError, match="workspace-cutover"): + verify_required_cutover(tmp_path) + + +def test_cutover_quarantines_unscoped_derived_threads(tmp_path, monkeypatch): + import EvoScientist.scope_registry as registry_module + + registry_module._registry_cache.clear() + monkeypatch.delenv("EVOSCIENTIST_DEPLOYMENT_ID", raising=False) + threads = _Threads( + [ + {"thread_id": "primary-thread", "metadata": {"graph_id": "EvoScientist"}}, + {"thread_id": "legacy-child", "metadata": {"agent_name": "worker"}}, + ] + ) + runs = _Runs({"legacy-child": [{"run_id": "run-child", "status": "running"}]}) + + report = run_workspace_cutover( + workspace_root=tmp_path, + client=SimpleNamespace(threads=threads, crons=_Crons(), runs=runs), + browser_sdk_gate=True, + ) + + assert report["status"] == "passed" + assert runs.cancelled == [("legacy-child", "run-child")] + assert report["quarantined_derived_threads"] == [ + { + "thread_id": "legacy-child", + "cancelled_run_ids": ["run-child"], + "reason": "unscoped-derived-thread", + } + ] + derived_update = next( + update for update in threads.updated if update[0] == "legacy-child" + ) + assert ( + derived_update[1]["workspace_quarantine"]["reason"] == "unscoped-derived-thread" + ) + + +def test_cutover_accepts_registered_scoped_derived_thread(tmp_path, monkeypatch): + import EvoScientist.scope_registry as registry_module + + registry_module._registry_cache.clear() + monkeypatch.delenv("EVOSCIENTIST_DEPLOYMENT_ID", raising=False) + deployment_id = deployment_id_for_workspace(tmp_path) + registry = get_scope_registry(tmp_path) + scope = registry.provision(deployment_id, "primary-thread") + owner = registry.register_owner( + deployment_id, + scope.scope_id, + owner_type="async_thread", + resource_id="registered-child", + parent_owner_id=scope.primary_owner_id, + state="active", + ) + threads = _Threads( + [ + {"thread_id": "primary-thread", "metadata": {"graph_id": "EvoScientist"}}, + { + "thread_id": "registered-child", + "metadata": { + "workspace_scope_id": scope.scope_id, + "workspace_scope_owner_id": owner.owner_id, + "workspace_deployment_id": deployment_id, + }, + }, + ] + ) + + report = run_workspace_cutover( + workspace_root=tmp_path, + client=SimpleNamespace(threads=threads, crons=_Crons(), runs=_Runs()), + browser_sdk_gate=True, + ) + + assert report["status"] == "passed" + assert report["validated_derived_threads"] == ["registered-child"] + assert report["quarantined_derived_threads"] == [] + + +def test_cutover_quarantines_scoped_thread_without_registry_owner( + tmp_path, monkeypatch +): + import EvoScientist.scope_registry as registry_module + + registry_module._registry_cache.clear() + monkeypatch.delenv("EVOSCIENTIST_DEPLOYMENT_ID", raising=False) + threads = _Threads( + [ + {"thread_id": "primary-thread", "metadata": {"graph_id": "EvoScientist"}}, + { + "thread_id": "forged-child", + "metadata": { + "workspace_scope_id": "00000000-0000-4000-8000-000000000001", + "workspace_scope_owner_id": "00000000-0000-4000-8000-000000000002", + }, + }, + ] + ) + + report = run_workspace_cutover( + workspace_root=tmp_path, + client=SimpleNamespace(threads=threads, crons=_Crons(), runs=_Runs()), + browser_sdk_gate=True, + ) + + assert report["status"] == "passed" + assert report["invalid_scoped_derived_threads"][0]["thread_id"] == "forged-child" + update = next(item for item in threads.updated if item[0] == "forged-child") + assert "workspace_scope_id" not in update[1] + assert ( + update[1]["workspace_quarantine"]["reason"] == "invalid-scoped-derived-thread" + ) + + +def test_cutover_disables_scoped_cron_without_registry_owner(tmp_path, monkeypatch): + import EvoScientist.scope_registry as registry_module + + registry_module._registry_cache.clear() + monkeypatch.delenv("EVOSCIENTIST_DEPLOYMENT_ID", raising=False) + crons = _Crons( + [ + { + "cron_id": "forged-cron", + "metadata": { + "run_kind": "scheduled_task", + "workspace_scope_id": "00000000-0000-4000-8000-000000000001", + }, + } + ] + ) + + report = run_workspace_cutover( + workspace_root=tmp_path, + client=SimpleNamespace(threads=_Threads(), crons=crons, runs=_Runs()), + browser_sdk_gate=True, + ) + + assert report["status"] == "passed" + assert crons.updated == [("forged-cron", False)] + assert report["invalid_scoped_crons"][0]["cron_id"] == "forged-cron" diff --git a/tests/test_workspace_maintenance.py b/tests/test_workspace_maintenance.py new file mode 100644 index 0000000..2336a21 --- /dev/null +++ b/tests/test_workspace_maintenance.py @@ -0,0 +1,116 @@ +"""Lifecycle maintenance tests for scoped conversation workspaces.""" + +from __future__ import annotations + +import os +import sqlite3 +from datetime import UTC, datetime, timedelta +from pathlib import Path +from types import SimpleNamespace + +from EvoScientist.scope_registry import deployment_id_for_workspace, get_scope_registry +from EvoScientist.workspace_maintenance import run_workspace_maintenance +from EvoScientist.workspace_scope import provision_conversation_scope + + +class _Threads: + def __init__(self) -> None: + self.deleted: list[str] = [] + + def delete(self, thread_id: str) -> None: + self.deleted.append(thread_id) + + def get_state(self, thread_id: str) -> dict: + return {"values": {"messages": []}} + + +class _Runs: + def list(self, *, thread_id: str, limit: int): + assert limit == 1000 + return [] + + +def _age_scope(workspace_root: Path, scope_id: str, when: datetime) -> None: + registry = get_scope_registry(workspace_root) + with sqlite3.connect(registry.path) as database: + database.execute( + "UPDATE scopes SET created_at = ?, updated_at = ? WHERE scope_id = ?", + (when.isoformat(), when.isoformat(), scope_id), + ) + + +def test_maintenance_deletes_stale_draft_and_purges_old_trash(tmp_path, monkeypatch): + import EvoScientist.scope_registry as registry_module + + registry_module._registry_cache.clear() + monkeypatch.delenv("EVOSCIENTIST_DEPLOYMENT_ID", raising=False) + scope = provision_conversation_scope("draft-thread", workspace_root=tmp_path) + _age_scope(tmp_path, scope.scope_id, datetime(2026, 7, 1, tzinfo=UTC)) + files = tmp_path / ".evoscientist" / "conversations" / scope.scope_id / "files" + files.mkdir(parents=True, exist_ok=True) + old_trash = tmp_path / ".evoscientist" / "trash" / "old-scope" + old_trash.mkdir(parents=True) + old_timestamp = datetime(2026, 7, 1, tzinfo=UTC).timestamp() + os.utime(old_trash, (old_timestamp, old_timestamp)) + threads = _Threads() + + report = run_workspace_maintenance( + workspace_root=tmp_path, + client=SimpleNamespace(threads=threads, runs=_Runs()), + now=datetime(2026, 7, 10, tzinfo=UTC), + ) + + assert report["errors"] == [] + assert report["deleted_drafts"] == ["draft-thread"] + assert report["purged_trash"] == 1 + assert threads.deleted == ["draft-thread"] + registry = get_scope_registry(tmp_path) + deployment_id = deployment_id_for_workspace(tmp_path) + assert registry.get(deployment_id, scope.scope_id).state == "deleted" + assert not old_trash.exists() + assert any((tmp_path / ".evoscientist" / "trash").iterdir()) + + +def test_maintenance_leaves_draft_with_non_primary_owner(tmp_path, monkeypatch): + import EvoScientist.scope_registry as registry_module + + registry_module._registry_cache.clear() + monkeypatch.delenv("EVOSCIENTIST_DEPLOYMENT_ID", raising=False) + scope = provision_conversation_scope("draft-thread", workspace_root=tmp_path) + _age_scope(tmp_path, scope.scope_id, datetime.now(UTC) - timedelta(days=2)) + registry = get_scope_registry(tmp_path) + registry.register_owner( + deployment_id_for_workspace(tmp_path), + scope.scope_id, + owner_type="unknown-legacy-owner", + state="active", + ) + + report = run_workspace_maintenance( + workspace_root=tmp_path, + client=SimpleNamespace(threads=_Threads(), runs=_Runs()), + ) + + assert report["deleted_drafts"] == [] + assert report["skipped_drafts"] == ["draft-thread"] + + +def test_maintenance_leaves_draft_that_has_messages(tmp_path, monkeypatch): + import EvoScientist.scope_registry as registry_module + + class _ThreadWithMessages(_Threads): + def get_state(self, thread_id: str) -> dict: + return {"values": {"messages": [{"content": "keep this draft"}]}} + + registry_module._registry_cache.clear() + monkeypatch.delenv("EVOSCIENTIST_DEPLOYMENT_ID", raising=False) + scope = provision_conversation_scope("draft-thread", workspace_root=tmp_path) + _age_scope(tmp_path, scope.scope_id, datetime.now(UTC) - timedelta(days=2)) + + report = run_workspace_maintenance( + workspace_root=tmp_path, + client=SimpleNamespace(threads=_ThreadWithMessages(), runs=_Runs()), + ) + + assert report["deleted_drafts"] == [] + assert report["skipped_drafts"] == ["draft-thread"] diff --git a/tests/test_workspace_scope.py b/tests/test_workspace_scope.py new file mode 100644 index 0000000..92de198 --- /dev/null +++ b/tests/test_workspace_scope.py @@ -0,0 +1,448 @@ +"""Regression tests for conversation workspace ownership and isolation.""" + +from __future__ import annotations + +import sqlite3 +from types import SimpleNamespace + +import pytest +from blockbuster import BlockBuster + +from EvoScientist import paths +from EvoScientist.scope_registry import ( + ScopeAccessError, + ScopeConflictError, + ScopeRegistry, +) +from EvoScientist.workspace_scope import ( + DeferredScopedBackend, + create_workspace_backend, + provision_conversation_scope, + require_scoped_runtime, + verify_required_executor, + workspace_isolation_mode, +) + + +def test_workspace_isolation_environment_defaults_to_optional(monkeypatch): + monkeypatch.delenv("EVOSCIENTIST_WORKSPACE_ISOLATION", raising=False) + assert workspace_isolation_mode() == "optional" + + +def test_deployed_backend_rejects_an_unscoped_runtime(): + with pytest.raises(ScopeAccessError, match="require a workspace scope"): + create_workspace_backend( + SimpleNamespace(config={}), + legacy_backend=lambda: pytest.fail("deployed run used legacy backend"), + allow_unscoped_legacy=False, + ) + + +def test_non_deployed_backend_keeps_legacy_unscoped_runtime(): + legacy_backend = object() + backend = create_workspace_backend( + SimpleNamespace(config={}), + legacy_backend=lambda: legacy_backend, + ) + + assert backend is legacy_backend + + +def test_provision_is_idempotent_and_rejects_remap(tmp_path): + registry = ScopeRegistry(tmp_path / "scope-registry.sqlite3") + first = registry.provision("deployment-a", "thread-a") + second = registry.provision("deployment-a", "thread-a") + + assert first.scope_id == second.scope_id + assert first.primary_owner_id == second.primary_owner_id + + with pytest.raises(ScopeConflictError): + registry.provision( + "deployment-a", "thread-a", scope_id="00000000-0000-4000-8000-000000000001" + ) + + +def test_scope_transition_uses_compare_and_set(tmp_path): + registry = ScopeRegistry(tmp_path / "scope-registry.sqlite3") + record = registry.provision("deployment-a", "thread-a") + active = registry.transition_scope( + "deployment-a", + record.scope_id, + expected_revision=record.revision, + state="active", + ) + assert active.revision == record.revision + 1 + + with pytest.raises(ScopeConflictError): + registry.transition_scope( + "deployment-a", + record.scope_id, + expected_revision=record.revision, + state="deleting", + ) + + +def test_derived_owner_cannot_bind_resource_from_other_scope(tmp_path): + registry = ScopeRegistry(tmp_path / "scope-registry.sqlite3") + first = registry.provision("deployment-a", "thread-a") + second = registry.provision("deployment-a", "thread-b") + owner_a = registry.register_owner( + "deployment-a", + first.scope_id, + owner_type="async_thread", + resource_id="child-thread", + parent_owner_id=first.primary_owner_id, + state="active", + ) + assert owner_a.resource_id == "child-thread" + + with pytest.raises(ScopeConflictError): + registry.register_owner( + "deployment-a", + second.scope_id, + owner_type="async_thread", + resource_id="child-thread", + parent_owner_id=second.primary_owner_id, + state="active", + ) + + +def test_turn_reservation_is_idempotent_and_primary_run_validates_runtime(tmp_path): + registry = ScopeRegistry(tmp_path / "scope-registry.sqlite3") + record = registry.provision("deployment-a", "thread-a") + first = registry.reserve_turn( + "deployment-a", + record.scope_id, + "00000000-0000-4000-8000-000000000001", + "hash-a", + ) + second = registry.reserve_turn( + "deployment-a", + record.scope_id, + "00000000-0000-4000-8000-000000000001", + "hash-a", + ) + assert first.run_owner_id == second.run_owner_id + assert ( + registry.assert_runtime( + "deployment-a", record.scope_id, "thread-a", first.run_owner_id + ).primary_thread_id + == "thread-a" + ) + bound = registry.bind_turn("deployment-a", record.scope_id, first.turn_id, "run-a") + assert bound.run_id == "run-a" + assert ( + registry.assert_runtime( + "deployment-a", record.scope_id, "thread-a", bound.run_owner_id + ).primary_thread_id + == "thread-a" + ) + + with pytest.raises(ScopeConflictError): + registry.reserve_turn( + "deployment-a", + record.scope_id, + first.turn_id, + "another-request", + ) + + +def test_run_reservations_allow_resume_within_one_logical_turn(tmp_path): + registry = ScopeRegistry(tmp_path / "scope-registry.sqlite3") + record = registry.provision("deployment-a", "thread-a") + turn_id = "00000000-0000-4000-8000-000000000001" + initial_request_id = "00000000-0000-4000-8000-000000000002" + resume_request_id = "00000000-0000-4000-8000-000000000003" + + initial = registry.reserve_run( + "deployment-a", record.scope_id, initial_request_id, turn_id, "initial" + ) + retry = registry.reserve_run( + "deployment-a", record.scope_id, initial_request_id, turn_id, "initial" + ) + resume = registry.reserve_run( + "deployment-a", + record.scope_id, + resume_request_id, + turn_id, + "resume", + interrupt_key="interrupt-a", + ) + + assert retry.run_owner_id == initial.run_owner_id + assert resume.turn_id == initial.turn_id == turn_id + assert resume.run_owner_id != initial.run_owner_id + assert ( + registry.bind_run( + "deployment-a", record.scope_id, resume_request_id, "resume-run" + ).run_id + == "resume-run" + ) + + with pytest.raises(ScopeConflictError, match="run_request_id"): + registry.reserve_run( + "deployment-a", record.scope_id, initial_request_id, turn_id, "changed" + ) + with pytest.raises(ScopeConflictError, match="interrupt"): + registry.reserve_run( + "deployment-a", + record.scope_id, + "00000000-0000-4000-8000-000000000004", + turn_id, + "another-resume", + interrupt_key="interrupt-a", + ) + + +def test_registry_migrates_turn_keyed_reservations_to_run_request_ids(tmp_path): + database_path = tmp_path / "scope-registry.sqlite3" + scope_id = "00000000-0000-4000-8000-000000000010" + turn_id = "00000000-0000-4000-8000-000000000011" + with sqlite3.connect(database_path) as connection: + connection.executescript( + """ + CREATE TABLE scopes ( + deployment_id TEXT NOT NULL, + scope_id TEXT NOT NULL, + primary_thread_id TEXT NOT NULL, + state TEXT NOT NULL, + revision INTEGER NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + deleted_at TEXT, + PRIMARY KEY (deployment_id, scope_id) + ); + CREATE TABLE scope_turns ( + deployment_id TEXT NOT NULL, + scope_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + request_hash TEXT NOT NULL, + run_owner_id TEXT NOT NULL, + run_id TEXT, + state TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (deployment_id, scope_id, turn_id) + ); + """ + ) + connection.execute( + """ + INSERT INTO scopes VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + "deployment-a", + scope_id, + "thread-a", + "active", + 1, + "created", + "updated", + None, + ), + ) + connection.execute( + """ + INSERT INTO scope_turns VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + "deployment-a", + scope_id, + turn_id, + "old-hash", + "owner-a", + "run-a", + "active", + "created", + "updated", + ), + ) + connection.execute("PRAGMA user_version = 1") + + ScopeRegistry(database_path).initialize() + + with sqlite3.connect(database_path) as connection: + version = connection.execute("PRAGMA user_version").fetchone()[0] + row = connection.execute( + """ + SELECT run_request_id, turn_id, run_id + FROM scope_run_requests WHERE deployment_id = ? AND scope_id = ? + """, + ("deployment-a", scope_id), + ).fetchone() + assert version == 2 + assert row == (turn_id, turn_id, "run-a") + + +def test_required_runtime_never_falls_back_to_shared_root(tmp_path, monkeypatch): + monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path) + monkeypatch.setenv("EVOSCIENTIST_WORKSPACE_ISOLATION", "required") + import EvoScientist.scope_registry as registry_module + + registry_module._registry_cache.clear() + record = provision_conversation_scope("thread-a", deployment_id="deployment-a") + runtime = SimpleNamespace( + config={ + "configurable": { + "thread_id": "thread-a", + "workspace_scope_id": record.scope_id, + "workspace_scope_owner_id": record.primary_owner_id, + "workspace_scope_revision": record.revision, + "workspace_deployment_id": "deployment-a", + } + } + ) + + context = require_scoped_runtime(runtime) + assert context is not None + assert ( + context.files_dir + == tmp_path / ".evoscientist" / "conversations" / record.scope_id / "files" + ) + + with pytest.raises(ScopeAccessError): + require_scoped_runtime(SimpleNamespace(config={})) + + +async def test_deferred_backend_does_not_block_the_agent_event_loop( + tmp_path, monkeypatch +): + monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path) + monkeypatch.setenv("EVOSCIENTIST_WORKSPACE_ISOLATION", "optional") + import EvoScientist.scope_registry as registry_module + import EvoScientist.workspace_scope as workspace_scope_module + + registry_module._registry_cache.clear() + record = provision_conversation_scope("thread-a", deployment_id="deployment-a") + from langchain_core.runnables.config import var_child_runnable_config + + token = var_child_runnable_config.set( + { + "configurable": { + "thread_id": "thread-a", + "workspace_scope_id": record.scope_id, + "workspace_scope_owner_id": record.primary_owner_id, + "workspace_scope_revision": record.revision, + "workspace_deployment_id": "deployment-a", + } + } + ) + blocker = BlockBuster(scanned_modules=[workspace_scope_module, registry_module]) + blocker.activate() + try: + backend = create_workspace_backend( + SimpleNamespace(), + legacy_backend=lambda: pytest.fail("scoped runtime used legacy backend"), + ) + # The filesystem operation is dispatched to a thread, which does not + # inherit LangGraph's context variable. + var_child_runnable_config.reset(token) + token = None + result = await backend.als("/") + finally: + if token is not None: + var_child_runnable_config.reset(token) + blocker.deactivate() + + assert isinstance(backend, DeferredScopedBackend) + assert result.error is None + + +def test_deferred_backend_revalidates_a_deleting_scope(tmp_path, monkeypatch): + monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path) + monkeypatch.setenv("EVOSCIENTIST_WORKSPACE_ISOLATION", "optional") + import EvoScientist.scope_registry as registry_module + + registry_module._registry_cache.clear() + record = provision_conversation_scope("thread-a", deployment_id="deployment-a") + registry = registry_module.get_scope_registry(tmp_path) + active = registry.transition_scope( + "deployment-a", + record.scope_id, + expected_revision=record.revision, + state="active", + ) + runtime = SimpleNamespace( + config={ + "configurable": { + "thread_id": "thread-a", + "workspace_scope_id": active.scope_id, + "workspace_scope_owner_id": active.primary_owner_id, + "workspace_scope_revision": active.revision, + "workspace_deployment_id": "deployment-a", + } + } + ) + backend = create_workspace_backend( + runtime, + legacy_backend=lambda: pytest.fail("scoped runtime used legacy backend"), + ) + assert backend.ls("/").error is None + + registry.transition_scope( + "deployment-a", + active.scope_id, + expected_revision=active.revision, + state="deleting", + ) + with pytest.raises(ScopeAccessError): + backend.ls("/") + + +def test_required_executor_needs_an_oci_runtime(monkeypatch): + import EvoScientist.workspace_scope as scope_module + + monkeypatch.setattr(scope_module.shutil, "which", lambda _: None) + with pytest.raises(RuntimeError, match="OCI runtime"): + verify_required_executor() + + +def test_active_lock_can_only_be_renewed_by_the_current_owner(tmp_path): + registry = ScopeRegistry(tmp_path / "scope-registry.sqlite3") + operation_id = "00000000-0000-4000-8000-000000000001" + lock = registry.acquire_lock( + "deployment-a", "workspace-cutover", operation_id, lease_seconds=30 + ) + assert registry.active_lock("deployment-a", "workspace-cutover") == lock + renewed = registry.renew_lock( + "deployment-a", "workspace-cutover", operation_id, lease_seconds=60 + ) + assert renewed.expires_at > lock.expires_at + registry.release_lock("deployment-a", "workspace-cutover", operation_id) + assert registry.active_lock("deployment-a", "workspace-cutover") is None + + +def test_cutover_lock_blocks_scoped_runtime_and_owner_creation(tmp_path, monkeypatch): + monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path) + monkeypatch.setenv("EVOSCIENTIST_WORKSPACE_ISOLATION", "required") + import EvoScientist.scope_registry as registry_module + + registry_module._registry_cache.clear() + record = provision_conversation_scope("thread-a", deployment_id="deployment-a") + registry = registry_module.get_scope_registry(tmp_path) + registry.acquire_lock( + "deployment-a", + "workspace-cutover", + "00000000-0000-4000-8000-000000000001", + ) + with pytest.raises(ScopeAccessError, match="cutover"): + registry.provision("deployment-a", "new-thread") + runtime = SimpleNamespace( + config={ + "configurable": { + "thread_id": "thread-a", + "workspace_scope_id": record.scope_id, + "workspace_scope_owner_id": record.primary_owner_id, + "workspace_deployment_id": "deployment-a", + } + } + ) + with pytest.raises(ScopeAccessError, match="cutover"): + require_scoped_runtime(runtime) + with pytest.raises(ScopeAccessError, match="cutover"): + registry.register_owner( + "deployment-a", + record.scope_id, + owner_type="derived_run", + parent_owner_id=record.primary_owner_id, + ) diff --git a/tests/test_workspace_scope_http.py b/tests/test_workspace_scope_http.py new file mode 100644 index 0000000..990fe57 --- /dev/null +++ b/tests/test_workspace_scope_http.py @@ -0,0 +1,261 @@ +"""Tests for the server-only workspace scope registry routes.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +import pytest +from blockbuster import BlockBuster +from starlette.testclient import TestClient + +from EvoScientist import paths +from EvoScientist.langgraph_dev.http import app +from EvoScientist.workspace_scope import DeferredScopedBackend, create_workspace_backend + + +def test_scope_routes_require_service_token(monkeypatch): + monkeypatch.delenv("EVOSCIENTIST_BACKEND_SERVICE_TOKEN", raising=False) + with TestClient(app) as client: + response = client.post( + "/internal/workspace-scopes/provision", json={"thread_id": "thread-a"} + ) + assert response.status_code == 503 + + +def test_scope_provision_and_lookup(tmp_path, monkeypatch): + monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path) + monkeypatch.setenv("EVOSCIENTIST_BACKEND_SERVICE_TOKEN", "scope-test-token") + monkeypatch.setenv("EVOSCIENTIST_DEPLOYMENT_ID", "deployment-test") + import EvoScientist.scope_registry as registry_module + + registry_module._registry_cache.clear() + headers = {"Authorization": "Bearer scope-test-token"} + with TestClient(app) as client: + created = client.post( + "/internal/workspace-scopes/provision", + json={"thread_id": "thread-a"}, + headers=headers, + ) + assert created.status_code == 201 + payload = created.json() + looked_up = client.get( + "/internal/workspace-scopes/by-thread/thread-a", headers=headers + ) + + assert looked_up.status_code == 200 + assert looked_up.json()["scope_id"] == payload["scope_id"] + assert ( + tmp_path / ".evoscientist" / "conversations" / payload["scope_id"] / "files" + ).is_dir() + + +def test_scope_registry_is_constructed_off_the_asgi_event_loop(tmp_path, monkeypatch): + monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path) + monkeypatch.setenv("EVOSCIENTIST_BACKEND_SERVICE_TOKEN", "scope-test-token") + monkeypatch.setenv("EVOSCIENTIST_DEPLOYMENT_ID", "deployment-test") + import EvoScientist.scope_registry as registry_module + + registry_module._registry_cache.clear() + original = registry_module.get_scope_registry + + def off_event_loop_registry(*args, **kwargs): + with pytest.raises(RuntimeError): + asyncio.get_running_loop() + return original(*args, **kwargs) + + monkeypatch.setattr(registry_module, "get_scope_registry", off_event_loop_registry) + headers = {"Authorization": "Bearer scope-test-token"} + with TestClient(app) as client: + created = client.post( + "/internal/workspace-scopes/provision", + json={"thread_id": "thread-a"}, + headers=headers, + ) + looked_up = client.get( + "/internal/workspace-scopes/by-thread/thread-a", headers=headers + ) + + assert created.status_code == 201 + assert looked_up.status_code == 200 + + +def test_scope_turn_routes_are_idempotent(tmp_path, monkeypatch): + monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path) + monkeypatch.setenv("EVOSCIENTIST_BACKEND_SERVICE_TOKEN", "scope-test-token") + monkeypatch.setenv("EVOSCIENTIST_DEPLOYMENT_ID", "deployment-test") + import EvoScientist.scope_registry as registry_module + + registry_module._registry_cache.clear() + headers = {"Authorization": "Bearer scope-test-token"} + turn_id = "00000000-0000-4000-8000-000000000001" + with TestClient(app) as client: + scope = client.post( + "/internal/workspace-scopes/provision", + json={"thread_id": "thread-a"}, + headers=headers, + ).json() + reserved = client.post( + f"/internal/workspace-scopes/{scope['scope_id']}/turns/reserve", + json={"turn_id": turn_id, "request_hash": "hash-a"}, + headers=headers, + ) + assert reserved.status_code == 201 + bound = client.patch( + f"/internal/workspace-scopes/{scope['scope_id']}/turns/{turn_id}", + json={"run_id": "run-a"}, + headers=headers, + ) + assert bound.status_code == 200 + retried = client.post( + f"/internal/workspace-scopes/{scope['scope_id']}/turns/reserve", + json={"turn_id": turn_id, "request_hash": "hash-a"}, + headers=headers, + ) + + assert retried.status_code == 201 + assert retried.json()["run_id"] == "run-a" + + +def test_scope_run_routes_support_approval_resume_and_dedupe_interrupts( + tmp_path, monkeypatch +): + monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path) + monkeypatch.setenv("EVOSCIENTIST_BACKEND_SERVICE_TOKEN", "scope-test-token") + monkeypatch.setenv("EVOSCIENTIST_DEPLOYMENT_ID", "deployment-test") + import EvoScientist.scope_registry as registry_module + + registry_module._registry_cache.clear() + headers = {"Authorization": "Bearer scope-test-token"} + turn_id = "00000000-0000-4000-8000-000000000001" + initial_request_id = "00000000-0000-4000-8000-000000000002" + resume_request_id = "00000000-0000-4000-8000-000000000003" + duplicate_interrupt_request_id = "00000000-0000-4000-8000-000000000004" + with TestClient(app) as client: + scope = client.post( + "/internal/workspace-scopes/provision", + json={"thread_id": "thread-a"}, + headers=headers, + ).json() + initial = client.post( + f"/internal/workspace-scopes/{scope['scope_id']}/runs/reserve", + json={ + "run_request_id": initial_request_id, + "turn_id": turn_id, + "request_hash": "initial", + }, + headers=headers, + ) + resume = client.post( + f"/internal/workspace-scopes/{scope['scope_id']}/runs/reserve", + json={ + "run_request_id": resume_request_id, + "turn_id": turn_id, + "request_hash": "approve", + "interrupt_key": "interrupt-a", + }, + headers=headers, + ) + retried_resume = client.post( + f"/internal/workspace-scopes/{scope['scope_id']}/runs/reserve", + json={ + "run_request_id": resume_request_id, + "turn_id": turn_id, + "request_hash": "approve", + "interrupt_key": "interrupt-a", + }, + headers=headers, + ) + duplicate_interrupt = client.post( + f"/internal/workspace-scopes/{scope['scope_id']}/runs/reserve", + json={ + "run_request_id": duplicate_interrupt_request_id, + "turn_id": turn_id, + "request_hash": "another-decision", + "interrupt_key": "interrupt-a", + }, + headers=headers, + ) + + assert initial.status_code == 201 + assert resume.status_code == 201 + assert initial.json()["run_owner_id"] != resume.json()["run_owner_id"] + assert retried_resume.status_code == 201 + assert retried_resume.json()["run_owner_id"] == resume.json()["run_owner_id"] + assert duplicate_interrupt.status_code == 409 + assert duplicate_interrupt.json()["code"] == "interrupt_already_resolved" + + +async def test_reserved_turn_creates_an_async_safe_scoped_backend( + tmp_path, monkeypatch +): + monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path) + monkeypatch.setenv("EVOSCIENTIST_BACKEND_SERVICE_TOKEN", "scope-test-token") + monkeypatch.setenv("EVOSCIENTIST_DEPLOYMENT_ID", "deployment-test") + import EvoScientist.scope_registry as registry_module + import EvoScientist.workspace_scope as workspace_scope_module + + registry_module._registry_cache.clear() + headers = {"Authorization": "Bearer scope-test-token"} + turn_id = "00000000-0000-4000-8000-000000000001" + with TestClient(app) as client: + scope = client.post( + "/internal/workspace-scopes/provision", + json={"thread_id": "thread-a"}, + headers=headers, + ).json() + reservation = client.post( + f"/internal/workspace-scopes/{scope['scope_id']}/turns/reserve", + json={"turn_id": turn_id, "request_hash": "hash-a"}, + headers=headers, + ).json() + + runtime = SimpleNamespace( + config={ + "configurable": { + "thread_id": "thread-a", + "workspace_scope_id": scope["scope_id"], + "workspace_scope_owner_id": reservation["run_owner_id"], + "workspace_scope_revision": scope["revision"], + "workspace_deployment_id": scope["deployment_id"], + } + } + ) + blocker = BlockBuster(scanned_modules=[workspace_scope_module, registry_module]) + blocker.activate() + try: + backend = create_workspace_backend( + runtime, + legacy_backend=lambda: pytest.fail("scoped runtime used legacy backend"), + ) + result = await backend.als("/") + finally: + blocker.deactivate() + + assert isinstance(backend, DeferredScopedBackend) + assert result.error is None + + +def test_scope_owner_list_and_cutover_lock_routes(tmp_path, monkeypatch): + monkeypatch.setattr(paths, "WORKSPACE_ROOT", tmp_path) + monkeypatch.setenv("EVOSCIENTIST_BACKEND_SERVICE_TOKEN", "scope-test-token") + monkeypatch.setenv("EVOSCIENTIST_DEPLOYMENT_ID", "deployment-test") + import EvoScientist.scope_registry as registry_module + + registry_module._registry_cache.clear() + headers = {"Authorization": "Bearer scope-test-token"} + with TestClient(app) as client: + created = client.post( + "/internal/workspace-scopes/provision", + json={"thread_id": "thread-a"}, + headers=headers, + ).json() + owners = client.get( + f"/internal/workspace-scopes/{created['scope_id']}/owners", headers=headers + ) + lock = client.get("/internal/workspace-scopes/cutover-lock", headers=headers) + + assert owners.status_code == 200 + assert owners.json()["owners"][0]["owner_type"] == "primary_thread" + assert lock.status_code == 200 + assert lock.json()["active"] is False diff --git a/uv.lock b/uv.lock index 947e9fa..3e463a7 100644 --- a/uv.lock +++ b/uv.lock @@ -965,6 +965,7 @@ dependencies = [ { name = "nest-asyncio" }, { name = "prompt-toolkit" }, { name = "psutil" }, + { name = "pydantic" }, { name = "python-dotenv" }, { name = "pyyaml" }, { name = "questionary" }, @@ -1076,6 +1077,7 @@ requires-dist = [ { name = "psutil", specifier = ">=6.0" }, { name = "pycryptodome", marker = "extra == 'all-channels'", specifier = ">=3.20" }, { name = "pycryptodome", marker = "extra == 'wechat'", specifier = ">=3.20" }, + { name = "pydantic", specifier = ">=2.10" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=1.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0" },