diff --git a/.superpowers/sdd/briefs/task-1-report.md b/.superpowers/sdd/briefs/task-1-report.md new file mode 100644 index 0000000..2e7996c --- /dev/null +++ b/.superpowers/sdd/briefs/task-1-report.md @@ -0,0 +1,132 @@ +# Task 1 报告 — RegistryV4 schema + SQLite 存储层 + 统一错误码 + +## 实现摘要 + +按简报与设计文档 v1.1.0（4.2、4.3、5.1、8.2、9.5 节）在仓库内新建独立子包 +`EvoScientist/model_registry/`，未改动任何现有文件（顶层 `EvoScientist/__init__.py` +采用惰性导出，无需修改）。 + +1. **`schemas.py` — 唯一 RegistryV4 schema（Pydantic v2）** + - `ProviderId`/`ModelKey`/`CredentialId`/`AdapterId` 共用锚定全匹配模式 + `\A[a-z0-9][a-z0-9._-]{0,63}\z`（pydantic 的 pattern 约束默认是子串搜索，必须锚定）。 + - `upstream_model_id` 仅限长 1–300、保留大小写；`ValidatedEndpoint` 限长 2048 且要求 + http(s) scheme（EndpointPolicy 属后续任务）。 + - `RegistryV4 { version: Literal[4], revision: PositiveInt, state, defaults, providers }`； + schema 级校验：Provider ID 唯一、Provider 内模型 key 唯一、defaults 引用必须存在、 + `active` 状态下 primary 非空且引用已启用模型（auxiliary 同理）。 + - `ProviderConfig.runtime`：`timeout_seconds` [10,600]（缺省 120）、`max_retries` [0,5] + （缺省 2）、`default_temperature` [0,2]|null、`default_top_p` (0,1]|null、 + `default_reasoning_effort` 缺省 `auto`。 + - `ModelConfig.runtime`：`limit_mode` combined 要求 `context_window_tokens`、input_only + 要求 `max_input_tokens`（model_validator）；`min_effective_input_tokens` 缺省 4096、 + 下限 1024；三个 `fixed_*_reserve_tokens` 非负；temperature/top_p/reasoning_effort 与 + `declared_capabilities` 按 4.3 定义。 + - `AuthConfig`：`mode=none` 时 `credential_id` 必须为 null（model_validator）。 + - 6.2/6.4/9.1 结构：`AdapterParameterSpec`（含 `connection` 可选块，对应 6.2 示例 + `chat_model`/`model_field`/`base_url_field`）、`AuthSpec`、`ParameterRule`、 + `ModelAvailability`（`VerificationInfo`）、`ResolvedModelConfig` + （`auth_ref={mode, credential_id?, credential_revision?}`，无任何 secret 字段）、 + `CredentialStatus`、`CredentialWrite`（9.2 的 `operation: replace`）。 +2. **`errors.py` — 统一错误码与载荷** + - 23 个稳定错误码常量 + `ERROR_HTTP_STATUS` 映射，覆盖 9.5 总表全部 17 组 + （409×6、401×1、404×1、422×15）。 + - `ErrorPayload {code, message, details:[{path, code}], request_id}` 与 9.5 结构一致； + `ModelRegistryError` 携带 code/`http_status`/`payload()`，未知 code 直接拒绝。 +3. **`store.py` — `ModelRuntimeStore`** + - 数据库 `/model-runtime.sqlite3`（默认 `~/.config/evoscientist`，可注入）； + 目录 0700、文件 0600、WAL、外键、`busy_timeout=30000`。 + - 手写 DDL（`CREATE TABLE IF NOT EXISTS`，无 alembic）：`registry_state`（单行）、 + `credential_pointers`、`credential_versions`（(credential_id, revision) 主键）、 + `model_verifications`（五元组主键，upsert 只留最近一次）、`run_runtime_snapshots` + （含部分唯一索引 `UNIQUE(deployment_id, thread_id, run_request_id) + WHERE status IN ('prepared','bound')`）、`delegation_jtis`。 + - `load_registry()` 无行时返回 bootstrap/revision=1 空 RegistryV4； + `save_registry(expected_revision=, registry=, credential_writes=)` 在 + `BEGIN IMMEDIATE` 事务内校验 revision（不符抛 `REGISTRY_REVISION_CONFLICT`）、 + 写入不可变凭据版本、registry revision+1；首次同时具备已启用模型+有效 primary+已配置 + 凭据（或 `mode=none`）时原子转为 `active`；任一失败整体回滚。每次保存强制执行 + 9.2 第 7 条（defaults 必须引用已启用模型，违反抛 `MODEL_DISABLED`）。 + - 凭据：`write_credential_version`（递增 revision；重写相同当前密钥幂等返回原 + revision）、`resolve_credential`（不存在/已销毁抛 + `RUN_CREDENTIAL_REVISION_UNAVAILABLE`）、`retire_credential_version`、 + `credential_status`（hint 末 4 位 `...abcd`，短于 4 字符的密钥 hint 为 null 绝不泄露； + 绝不返回明文）。 + - 快照：`insert_run_snapshot`/`set_run_snapshot_status`/`get_run_snapshot` + （状态机 prepared|bound|expired|aborted，部分唯一索引行为由测试覆盖）。 + - `check_shared_storage()`：探测 `BEGIN IMMEDIATE` 写锁能力，失败抛 + `SharedStorageError`（多节点不共享持久卷时启动失败）。 +4. **`hashing.py` — `configuration_hash(provider, model)`** + - 覆盖 adapter、base_url、upstream_model_id、Provider 与 Model 全部运行参数（含声明 + 能力与限制），`json.dumps(sort_keys=True, separators=(",", ":"))` 规范化后 SHA-256。 + +## 文件清单 + +新增（无修改既有文件）： +- `EvoScientist/model_registry/__init__.py` +- `EvoScientist/model_registry/schemas.py` +- `EvoScientist/model_registry/errors.py` +- `EvoScientist/model_registry/store.py` +- `EvoScientist/model_registry/hashing.py` +- `tests/test_model_registry_schemas.py` +- `tests/test_model_registry_store.py` +- `.superpowers/sdd/briefs/task-1-report.md`（本文件） + +## 测试命令与输出 + +TDD 流程：先写两个测试文件并确认失败（`ModuleNotFoundError: No module named +'EvoScientist.model_registry'`），再实现。 + +``` +$ .venv/bin/python -m pytest tests/test_model_registry_schemas.py tests/test_model_registry_store.py -x -q +........................................................................ [ 80%] +................. [100%] +89 passed in 0.22s +``` + +全量回归（无既有失败，无回归）： + +``` +$ .venv/bin/python -m pytest tests/ -x -q +........sssss....................................... [100%] +2922 passed, 10 skipped, 1 warning in 73.00s +``` + +（warning 为 `test_langgraph_dev_http.py` 的 StarletteDeprecationWarning，既有、与本任务无关。） + +lint 与格式： + +``` +$ .venv/bin/ruff check EvoScientist/model_registry tests/test_model_registry_schemas.py tests/test_model_registry_store.py +All checks passed! +$ .venv/bin/ruff format --check ... # 已格式化 +``` + +## 自我审查发现（已处理） + +1. **测试副作用污染真实配置目录**：初版 `test_default_config_dir` 未注入路径，运行时在 + 真实 `~/.config/evoscientist/` 创建了空的 `model-runtime.sqlite3`。已确认该库所有表 + 为空（确为测试副产物）后删除（含 -wal/-shm），并把测试改为 monkeypatch + `store.DEFAULT_CONFIG_DIR` 到 tmp_path，此后测试不再触碰真实 home。 +2. **共享 `Field()` 实例**：初版 `_TEMPERATURE`/`_TOP_P` 在三个模型间复用同一 FieldInfo， + 已改为各字段独立 `Field(...)`，规避 pydantic 共享元数据的潜在风险。 +3. **docstring 混入中文**：`save_registry` 一处 docstring 误用中文，已改为英文以符合 + 仓库注释惯例。 +4. **并发 CAS 断言**：`sorted()` 大小写排序导致误报，改为不区分大小写排序。 +5. ruff 修复：`datetime.UTC` 别名、导入排序、`pytest.raises` 增加 `match=`。 + +## 遗留疑虑 + +1. **`connection` 块为可选**：6.2 正文的"至少包含"清单未列 `connection`，但 glm-5.2 + 示例契约包含它。schema 将其建模为可选字段，Task 2 落地五种 Adapter 契约时若确认 + 每个契约都有 connection，可考虑收紧为必填。 +2. **激活就绪判定中的"满足 AuthSpec"**：4.3 要求激活时认证状态满足 AuthSpec，但 + Adapter 契约属 Task 2。当前 store 仅做存储层判定（`mode=none` 或凭据已配置）； + AuthSpec 级别校验（如 credential_kind 匹配）需在 Task 2/3 的 API 层补充。 +3. **幂等语义解释**：简报称 `write_credential_version` 为"幂等 prepare"，实现为"重写 + 与当前版本完全相同的密钥时返回现有 revision"；不同密钥轮换仍产生新 revision。若 + 后续任务对幂等键有不同约定（如客户端提供 request id），需再对齐。 +4. **`save_registry` 参数为 keyword-only**：与简报签名 + `save_registry(expected_revision, registry, credential_writes=[])` 语义一致，仅调用 + 形式略异。 +5. `MODEL_DISABLED` 用于 defaults 引用未启用模型的保存错误（9.2 第 7 条属 422 校验， + 总表无更贴切码）；引用不存在模型由 schema 层先行拒绝，store 内同名分支仅作防御。 diff --git a/EvoScientist/model_registry/__init__.py b/EvoScientist/model_registry/__init__.py new file mode 100644 index 0000000..7c9e583 --- /dev/null +++ b/EvoScientist/model_registry/__init__.py @@ -0,0 +1,58 @@ +"""Unified model registry: RegistryV4 schema, error taxonomy, and SQLite store. + +This subpackage is the single authority for the version 4 model registry +(design doc sections 4.2, 4.3, 5.1, 8.2, 9.5). HTTP JSON, SQLite JSON, import +tooling, and test fixtures all reuse these Pydantic models. +""" + +from __future__ import annotations + +from .errors import ERROR_HTTP_STATUS, ErrorDetail, ErrorPayload, ModelRegistryError +from .hashing import configuration_hash +from .schemas import ( + AdapterParameterSpec, + AuthConfig, + AuthRef, + AuthSpec, + Capabilities, + CredentialStatus, + CredentialWrite, + ModelAvailability, + ModelConfig, + ModelRef, + ModelRuntimeConfig, + ParameterRule, + ProviderConfig, + ProviderRuntimeConfig, + RegistryV4, + ResolvedModelConfig, + VerificationInfo, +) +from .store import ModelRuntimeStore, SharedStorageError + +__all__ = [ + "ERROR_HTTP_STATUS", + "AdapterParameterSpec", + "AuthConfig", + "AuthRef", + "AuthSpec", + "Capabilities", + "CredentialStatus", + "CredentialWrite", + "ErrorDetail", + "ErrorPayload", + "ModelAvailability", + "ModelConfig", + "ModelRef", + "ModelRegistryError", + "ModelRuntimeConfig", + "ModelRuntimeStore", + "ParameterRule", + "ProviderConfig", + "ProviderRuntimeConfig", + "RegistryV4", + "ResolvedModelConfig", + "SharedStorageError", + "VerificationInfo", + "configuration_hash", +] diff --git a/EvoScientist/model_registry/errors.py b/EvoScientist/model_registry/errors.py new file mode 100644 index 0000000..75bea91 --- /dev/null +++ b/EvoScientist/model_registry/errors.py @@ -0,0 +1,129 @@ +"""Stable error codes and the unified error payload (design doc section 9.5). + +Every failure response uses the same structure:: + + { "code": "STABLE_ERROR_CODE", "message": "safe human-readable message", + "details": [{ "path": "...", "code": "..." }], "request_id": "..." } + +Messages and details must never contain secrets, full provider errors, or +unredacted request headers. +""" + +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field + +# 409 — version or idempotency conflicts. +REGISTRY_REVISION_CONFLICT = "REGISTRY_REVISION_CONFLICT" +THREAD_MODEL_SELECTION_CONFLICT = "THREAD_MODEL_SELECTION_CONFLICT" +RUN_REQUEST_CONFLICT = "RUN_REQUEST_CONFLICT" +SNAPSHOT_ALREADY_BOUND = "SNAPSHOT_ALREADY_BOUND" +SNAPSHOT_EXPIRED = "SNAPSHOT_EXPIRED" +MODEL_CONFIGURATION_CHANGED = "MODEL_CONFIGURATION_CHANGED" + +# 401 — invalid identity. +DELEGATION_REPLAYED = "DELEGATION_REPLAYED" + +# 404 — missing resource. +MODEL_NOT_FOUND = "MODEL_NOT_FOUND" + +# 422 — configuration and runtime precondition failures. +MODEL_REGISTRY_NOT_READY = "MODEL_REGISTRY_NOT_READY" +MODEL_DISABLED = "MODEL_DISABLED" +MODEL_NOT_AVAILABLE = "MODEL_NOT_AVAILABLE" +MODEL_CONFIG_OUTSIDE_SNAPSHOT = "MODEL_CONFIG_OUTSIDE_SNAPSHOT" +CREDENTIAL_NOT_CONFIGURED = "CREDENTIAL_NOT_CONFIGURED" +CREDENTIAL_REJECTED = "CREDENTIAL_REJECTED" +RUN_CREDENTIAL_REVISION_UNAVAILABLE = "RUN_CREDENTIAL_REVISION_UNAVAILABLE" +AUTH_MODE_UNSUPPORTED = "AUTH_MODE_UNSUPPORTED" +ADAPTER_NOT_SUPPORTED = "ADAPTER_NOT_SUPPORTED" +CAPABILITY_UNSUPPORTED_BY_ADAPTER = "CAPABILITY_UNSUPPORTED_BY_ADAPTER" +MODEL_CAPABILITY_UNAVAILABLE = "MODEL_CAPABILITY_UNAVAILABLE" +UNSUPPORTED_RUNTIME_PARAMETER = "UNSUPPORTED_RUNTIME_PARAMETER" +MODEL_LIMITS_UNCONFIRMED = "MODEL_LIMITS_UNCONFIRMED" +CONTEXT_BUDGET_UNSATISFIABLE = "CONTEXT_BUDGET_UNSATISFIABLE" +PROVIDER_UNREACHABLE = "PROVIDER_UNREACHABLE" + +ERROR_HTTP_STATUS: dict[str, int] = { + REGISTRY_REVISION_CONFLICT: 409, + THREAD_MODEL_SELECTION_CONFLICT: 409, + RUN_REQUEST_CONFLICT: 409, + SNAPSHOT_ALREADY_BOUND: 409, + SNAPSHOT_EXPIRED: 409, + MODEL_CONFIGURATION_CHANGED: 409, + DELEGATION_REPLAYED: 401, + MODEL_NOT_FOUND: 404, + MODEL_REGISTRY_NOT_READY: 422, + MODEL_DISABLED: 422, + MODEL_NOT_AVAILABLE: 422, + MODEL_CONFIG_OUTSIDE_SNAPSHOT: 422, + CREDENTIAL_NOT_CONFIGURED: 422, + CREDENTIAL_REJECTED: 422, + RUN_CREDENTIAL_REVISION_UNAVAILABLE: 422, + AUTH_MODE_UNSUPPORTED: 422, + ADAPTER_NOT_SUPPORTED: 422, + CAPABILITY_UNSUPPORTED_BY_ADAPTER: 422, + MODEL_CAPABILITY_UNAVAILABLE: 422, + UNSUPPORTED_RUNTIME_PARAMETER: 422, + MODEL_LIMITS_UNCONFIRMED: 422, + CONTEXT_BUDGET_UNSATISFIABLE: 422, + PROVIDER_UNREACHABLE: 422, +} + + +class ErrorDetail(BaseModel): + """One field-level failure entry inside an error payload.""" + + path: str + code: str + + +class ErrorPayload(BaseModel): + """The single failure response shape defined in section 9.5.""" + + code: str + message: str + details: list[ErrorDetail] = Field(default_factory=list) + request_id: str + + +class ModelRegistryError(Exception): + """An error carrying a stable code from the section 9.5 taxonomy. + + ``message`` must be safe to return to a browser: no secrets, no full + provider errors, no unredacted request headers. + """ + + def __init__( + self, + code: str, + message: str, + *, + details: list[dict[str, str] | ErrorDetail] | None = None, + ) -> None: + if code not in ERROR_HTTP_STATUS: + raise ValueError(f"Unknown model registry error code: {code!r}") + super().__init__(message) + self.code = code + self.message = message + self.details = [ + detail + if isinstance(detail, ErrorDetail) + else ErrorDetail.model_validate(detail) + for detail in (details or []) + ] + + @property + def http_status(self) -> int: + return ERROR_HTTP_STATUS[self.code] + + def payload(self, *, request_id: str = "") -> dict[str, Any]: + """Return the section 9.5 failure response body.""" + return ErrorPayload( + code=self.code, + message=self.message, + details=self.details, + request_id=request_id, + ).model_dump() diff --git a/EvoScientist/model_registry/hashing.py b/EvoScientist/model_registry/hashing.py new file mode 100644 index 0000000..b7f8f2d --- /dev/null +++ b/EvoScientist/model_registry/hashing.py @@ -0,0 +1,26 @@ +"""Configuration hashing for model verification records (section 4.3). + +The hash covers the provider's adapter, base URL, the model's +``upstream_model_id``, all runtime parameters, declared capabilities, and +limits. Any change to these inputs invalidates prior verification records. +""" + +from __future__ import annotations + +import hashlib +import json + +from .schemas import ModelConfig, ProviderConfig + + +def configuration_hash(provider: ProviderConfig, model: ModelConfig) -> str: + """Return the normalized SHA-256 hash of one model's full configuration.""" + payload = { + "adapter": provider.adapter, + "base_url": provider.base_url, + "upstream_model_id": model.upstream_model_id, + "provider_runtime": provider.runtime.model_dump(mode="json"), + "model_runtime": model.runtime.model_dump(mode="json"), + } + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() diff --git a/EvoScientist/model_registry/schemas.py b/EvoScientist/model_registry/schemas.py new file mode 100644 index 0000000..28ba92f --- /dev/null +++ b/EvoScientist/model_registry/schemas.py @@ -0,0 +1,355 @@ +"""The single RegistryV4 schema (design doc sections 4.3, 6.2, 6.4, 9.1). + +HTTP JSON, SQLite JSON, import tooling, and test fixtures must all reuse +these Pydantic models instead of maintaining separate shapes. +""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + NonNegativeInt, + PositiveInt, + StringConstraints, + field_validator, + model_validator, +) + +# ProviderId, ModelKey, CredentialId, and AdapterId share the lowercase ID +# rule. The anchors enforce a full match (the pattern constraint alone only +# searches for a matching substring). +_ID_PATTERN = r"\A[a-z0-9][a-z0-9._-]{0,63}\z" + +ProviderId = Annotated[str, StringConstraints(pattern=_ID_PATTERN)] +ModelKey = Annotated[str, StringConstraints(pattern=_ID_PATTERN)] +CredentialId = Annotated[str, StringConstraints(pattern=_ID_PATTERN)] +AdapterId = Annotated[str, StringConstraints(pattern=_ID_PATTERN)] + +NonEmptyString = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] + +# `upstream_model_id` keeps the case and characters the provider requires; +# only its length is bounded. +UpstreamModelId = Annotated[str, StringConstraints(min_length=1, max_length=300)] + +# `base_url` is normalized and authorized by the backend EndpointPolicy; the +# schema only enforces a plausible http(s) endpoint string. +ValidatedEndpoint = Annotated[str, StringConstraints(min_length=1, max_length=2048)] + +ReasoningEffort = Literal["auto", "low", "medium", "high"] +AuthMode = Literal["none", "api_key", "bearer", "adapter_managed"] +RegistryState = Literal["bootstrap", "active"] +LimitMode = Literal["combined", "input_only"] +LimitsStatus = Literal["confirmed", "needs_confirmation"] +LimitsSource = Literal["provider", "tested_contract", "user"] +ModelRole = Literal["primary", "auxiliary", "summary", "tool_selector"] + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(frozen=True) + + +class ModelRef(_FrozenModel): + """The only identifier of an enabled model: ``{provider_id, model_key}``.""" + + provider_id: ProviderId + model_key: ModelKey + + +class RegistryDefaults(BaseModel): + primary: ModelRef | None = None + auxiliary: ModelRef | None = None + + +class Capabilities(BaseModel): + tools: bool = False + vision: bool = False + structured_output: bool = False + + +class AuthConfig(BaseModel): + """The adapter-allowed authentication mode plus an optional credential ref.""" + + mode: AuthMode + credential_id: CredentialId | None = None + + @model_validator(mode="after") + def _mode_none_has_no_credential(self) -> AuthConfig: + if self.mode == "none" and self.credential_id is not None: + raise ValueError( + "auth.credential_id must be null when auth.mode is 'none'." + ) + return self + + +class ProviderRuntimeConfig(BaseModel): + timeout_seconds: int = Field(default=120, ge=10, le=600) + max_retries: int = Field(default=2, ge=0, le=5) + default_temperature: float | None = Field(default=None, ge=0, le=2) + default_top_p: float | None = Field(default=None, gt=0, le=1) + default_reasoning_effort: ReasoningEffort = "auto" + + +class ModelRuntimeConfig(BaseModel): + limit_mode: LimitMode + context_window_tokens: PositiveInt | None = None + max_input_tokens: PositiveInt | None = None + max_output_tokens: PositiveInt + min_effective_input_tokens: int = Field(default=4096, ge=1024) + fixed_system_reserve_tokens: NonNegativeInt + fixed_tools_reserve_tokens: NonNegativeInt + fixed_attachments_reserve_tokens: NonNegativeInt + limits_status: LimitsStatus + limits_source: LimitsSource + temperature: float | None = Field(default=None, ge=0, le=2) + top_p: float | None = Field(default=None, gt=0, le=1) + reasoning_effort: ReasoningEffort = "auto" + declared_capabilities: Capabilities = Field(default_factory=Capabilities) + + @model_validator(mode="after") + def _limit_mode_requires_its_limit(self) -> ModelRuntimeConfig: + if self.limit_mode == "combined" and self.context_window_tokens is None: + raise ValueError( + "context_window_tokens is required when limit_mode is 'combined'." + ) + if self.limit_mode == "input_only" and self.max_input_tokens is None: + raise ValueError( + "max_input_tokens is required when limit_mode is 'input_only'." + ) + return self + + +class ModelConfig(BaseModel): + key: ModelKey + name: NonEmptyString + upstream_model_id: UpstreamModelId + enabled: bool = True + runtime: ModelRuntimeConfig + + +class ProviderConfig(BaseModel): + id: ProviderId + name: NonEmptyString + adapter: AdapterId + base_url: ValidatedEndpoint + auth: AuthConfig + enabled: bool = True + runtime: ProviderRuntimeConfig = Field(default_factory=ProviderRuntimeConfig) + models: list[ModelConfig] = Field(default_factory=list) + + @field_validator("base_url") + @classmethod + def _base_url_uses_http(cls, value: str) -> str: + if not value.startswith(("http://", "https://")): + raise ValueError("base_url must use http:// or https://.") + return value + + @model_validator(mode="after") + def _model_keys_unique(self) -> ProviderConfig: + keys = [model.key for model in self.models] + if len(keys) != len(set(keys)): + raise ValueError(f"Provider {self.id!r} contains duplicate model keys.") + return self + + def find_model(self, model_key: str) -> ModelConfig | None: + for model in self.models: + if model.key == model_key: + return model + return None + + +class RegistryV4(BaseModel): + """The version 4 model registry document (section 4.3).""" + + version: Literal[4] = 4 + revision: PositiveInt = 1 + state: RegistryState = "bootstrap" + defaults: RegistryDefaults = Field(default_factory=RegistryDefaults) + providers: list[ProviderConfig] = Field(default_factory=list) + + @model_validator(mode="after") + def _check_document(self) -> RegistryV4: + provider_ids = [provider.id for provider in self.providers] + if len(provider_ids) != len(set(provider_ids)): + raise ValueError("Provider IDs must be unique.") + for role, ref in ( + ("primary", self.defaults.primary), + ("auxiliary", self.defaults.auxiliary), + ): + if ref is None: + continue + target = self._locate(ref) + if target is None: + raise ValueError( + f"defaults.{role} references an unknown ModelRef " + f"{ref.provider_id!r}/{ref.model_key!r}." + ) + provider, model = target + if self.state == "active" and not (provider.enabled and model.enabled): + raise ValueError( + f"defaults.{role} must reference an enabled model while " + "the registry is active." + ) + if self.state == "active" and self.defaults.primary is None: + raise ValueError("defaults.primary must not be null while active.") + return self + + def _locate(self, ref: ModelRef) -> tuple[ProviderConfig, ModelConfig] | None: + provider = self.find_provider(ref.provider_id) + if provider is None: + return None + model = provider.find_model(ref.model_key) + if model is None: + return None + return provider, model + + def find_provider(self, provider_id: str) -> ProviderConfig | None: + for provider in self.providers: + if provider.id == provider_id: + return provider + return None + + +# --- Adapter parameter contracts (section 6.2) --- + +CredentialKind = Literal["api_key", "bearer_token", "adapter_managed", "none"] + + +class AuthSpec(BaseModel): + credential_required: bool + credential_kind: CredentialKind + target: Literal["client_option", "request_header", "adapter_internal"] + target_name: str | None = None + + +class ParameterRule(BaseModel): + supported: bool + value_type: Literal["integer", "number", "enum"] + nullable: Literal["inherit", "omit", "forbidden"] + minimum: float | None = None + maximum: float | None = None + enum_values: list[str] | None = None + target: Literal["client_option", "request_option", "extra_body_path"] + target_name: str + conflicts_with: list[str] = Field(default_factory=list) + normalizer: NonEmptyString + + +class ConnectionSpec(BaseModel): + chat_model: NonEmptyString + model_field: NonEmptyString + base_url_field: NonEmptyString + + +class AdapterParameterSpec(BaseModel): + adapter_id: AdapterId + spec_revision: PositiveInt + model_selector: NonEmptyString + auth_specs: dict[AuthMode, AuthSpec] = Field(default_factory=dict) + parameters: dict[str, ParameterRule] = Field(default_factory=dict) + protocol_capabilities: Capabilities + connection: ConnectionSpec | None = None + + +# --- Availability and resolved configuration (sections 4.3, 6.4, 9.1) --- + + +class VerificationInfo(BaseModel): + status: Literal["none", "passed", "failed", "stale"] + verified_at: str | None = None + adapter_spec_revision: PositiveInt | None = None + + +class ModelAvailability(BaseModel): + model_ref: ModelRef + state: Literal[ + "unavailable", + "configured", + "verification_failed", + "verification_stale", + "verified", + "enabled", + ] + selectable: bool + reason_code: str | None = None + verification: VerificationInfo = Field( + default_factory=lambda: VerificationInfo(status="none") + ) + effective_capabilities: Capabilities = Field(default_factory=Capabilities) + + +class AuthRef(BaseModel): + """Snapshot credential reference; never carries ``secret_value``.""" + + mode: AuthMode + credential_id: CredentialId | None = None + credential_revision: PositiveInt | None = None + + @model_validator(mode="after") + def _mode_none_has_no_credential(self) -> AuthRef: + if self.mode == "none" and ( + self.credential_id is not None or self.credential_revision is not None + ): + raise ValueError( + "auth_ref credential fields must be null when mode is 'none'." + ) + return self + + +class ClientOptions(BaseModel): + timeout_seconds: int = Field(ge=10, le=600) + max_retries: int = Field(ge=0, le=5) + + +class RequestOptions(BaseModel): + max_output_tokens: PositiveInt + temperature: float | None = Field(default=None, ge=0, le=2) + top_p: float | None = Field(default=None, gt=0, le=1) + reasoning_effort: ReasoningEffort = "auto" + + +class FixedReserves(BaseModel): + fixed_system_reserve_tokens: NonNegativeInt + fixed_tools_reserve_tokens: NonNegativeInt + fixed_attachments_reserve_tokens: NonNegativeInt + + +class InputBudget(BaseModel): + resolved_input_limit: PositiveInt + fixed_reserves: FixedReserves + message_budget: int + + +class ResolvedModelConfig(BaseModel): + """The frozen per-run configuration (section 6.4); contains no secrets.""" + + model_ref: ModelRef + role: ModelRole + adapter_id: AdapterId + adapter_spec_revision: PositiveInt + upstream_model_id: UpstreamModelId + base_url: ValidatedEndpoint + auth_ref: AuthRef + client_options: ClientOptions + request_options: RequestOptions + budget: InputBudget + effective_capabilities: Capabilities + + +# --- API helper models (sections 9.1, 9.2) --- + + +class CredentialStatus(BaseModel): + credential_id: CredentialId + configured: bool + hint: str | None = None + updated_at: str | None = None + + +class CredentialWrite(BaseModel): + credential_id: CredentialId + operation: Literal["replace"] = "replace" + secret_value: NonEmptyString diff --git a/EvoScientist/model_registry/store.py b/EvoScientist/model_registry/store.py new file mode 100644 index 0000000..c0b720f --- /dev/null +++ b/EvoScientist/model_registry/store.py @@ -0,0 +1,712 @@ +"""SQLite storage for the model registry, credentials, and run snapshots. + +The database lives at ``/model-runtime.sqlite3`` inside a ``0700`` +directory with ``0600`` file permissions, WAL journaling, foreign keys, and a +``busy_timeout`` (design doc section 4.2). Registry writes are serialized with +``BEGIN IMMEDIATE`` plus revision compare-and-swap; credential versions are +immutable; run snapshot uniqueness is enforced by a partial unique index +(section 8.2). Schema migrations follow the project's existing hand-written +``CREATE TABLE IF NOT EXISTS`` style — no alembic. +""" + +from __future__ import annotations + +import json +import re +import sqlite3 +import threading +import time +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from .errors import ( + CREDENTIAL_NOT_CONFIGURED, + MODEL_DISABLED, + REGISTRY_REVISION_CONFLICT, + RUN_CREDENTIAL_REVISION_UNAVAILABLE, + ModelRegistryError, +) +from .schemas import CredentialStatus, CredentialWrite, RegistryV4 + +DEFAULT_CONFIG_DIR = Path.home() / ".config" / "evoscientist" +DATABASE_FILENAME = "model-runtime.sqlite3" +_BUSY_TIMEOUT_MS = 30_000 + +_SNAPSHOT_STATUSES = ("prepared", "bound", "expired", "aborted") +_VERIFICATION_RESULTS = ("passed", "failed") +_ID_PATTERN = re.compile(r"[a-z0-9][a-z0-9._-]{0,63}") + +_DDL = """ +CREATE TABLE IF NOT EXISTS registry_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + revision INTEGER NOT NULL, + state TEXT NOT NULL CHECK (state IN ('bootstrap', 'active')), + registry_json TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS credential_pointers ( + credential_id TEXT PRIMARY KEY, + current_revision INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS credential_versions ( + credential_id TEXT NOT NULL, + revision INTEGER NOT NULL, + secret_value TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('active', 'retired')), + created_at INTEGER NOT NULL, + retired_at INTEGER, + PRIMARY KEY (credential_id, revision) +); + +CREATE TABLE IF NOT EXISTS model_verifications ( + provider_id TEXT NOT NULL, + model_key TEXT NOT NULL, + configuration_hash TEXT NOT NULL, + credential_revision INTEGER NOT NULL, + adapter_spec_revision INTEGER NOT NULL, + result TEXT NOT NULL CHECK (result IN ('passed', 'failed')), + verified_capabilities_json TEXT NOT NULL, + verified_at INTEGER NOT NULL, + error_code TEXT, + PRIMARY KEY ( + provider_id, model_key, configuration_hash, + credential_revision, adapter_spec_revision + ) +); + +CREATE TABLE IF NOT EXISTS run_runtime_snapshots ( + snapshot_id TEXT PRIMARY KEY, + deployment_id TEXT NOT NULL, + thread_id TEXT NOT NULL, + run_request_id TEXT NOT NULL, + selection_hash TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('prepared', 'bound', 'expired', 'aborted')), + langgraph_run_id TEXT, + payload_json TEXT NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL +); + +-- Only non-terminal snapshots occupy the {deployment_id, thread_id, +-- run_request_id} triplet; expired/aborted rows stay for audit and the same +-- triplet can be recreated (section 8.2). +CREATE UNIQUE INDEX IF NOT EXISTS run_runtime_snapshots_active_triplet + ON run_runtime_snapshots (deployment_id, thread_id, run_request_id) + WHERE status IN ('prepared', 'bound'); + +CREATE TABLE IF NOT EXISTS delegation_jtis ( + jti TEXT PRIMARY KEY, + expires_at INTEGER NOT NULL +); +""" + + +class SharedStorageError(RuntimeError): + """Raised at startup when the database cannot take a write lock.""" + + +def _rfc3339(epoch_seconds: int) -> str: + return datetime.fromtimestamp(epoch_seconds, UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _validate_credential_id(credential_id: str) -> str: + if not isinstance(credential_id, str) or not _ID_PATTERN.fullmatch(credential_id): + raise ValueError( + "credential_id must start with a lowercase letter or digit and contain " + "only lowercase letters, digits, dots, underscores, or hyphens." + ) + return credential_id + + +class ModelRuntimeStore: + """Storage facade over ``model-runtime.sqlite3``. + + Instances are cheap and stateless apart from the target directory, so + Workers can open one per process; all concurrency control happens in + SQLite itself. + """ + + def __init__(self, config_dir: str | Path | None = None) -> None: + self._config_dir = ( + Path(config_dir) if config_dir is not None else (DEFAULT_CONFIG_DIR) + ) + self._lock = threading.RLock() + self._config_dir.mkdir(parents=True, exist_ok=True) + try: + self._config_dir.chmod(0o700) + except OSError: + pass + with self._lock, self._connect() as connection: + connection.executescript(_DDL) + connection.commit() + try: + self.database_path.chmod(0o600) + except OSError: + pass + + @property + def config_dir(self) -> Path: + return self._config_dir + + @property + def database_path(self) -> Path: + return self._config_dir / DATABASE_FILENAME + + def _connect(self) -> sqlite3.Connection: + connection = sqlite3.connect( + self.database_path, timeout=_BUSY_TIMEOUT_MS / 1000 + ) + connection.execute("PRAGMA journal_mode = WAL") + connection.execute("PRAGMA foreign_keys = ON") + connection.execute(f"PRAGMA busy_timeout = {_BUSY_TIMEOUT_MS}") + return connection + + # --- startup checks ------------------------------------------------- + + def check_shared_storage(self) -> None: + """Probe that the database accepts a ``BEGIN IMMEDIATE`` write lock. + + Multi-node deployments that do not share the persistent volume must + fail at startup instead of each creating their own registry (4.2). + """ + with self._lock: + try: + connection = self._connect() + except sqlite3.Error as exc: + raise SharedStorageError( + "model-runtime.sqlite3 is not writable on this volume; " + "all Workers must share one persistent config directory." + ) from exc + try: + connection.execute("BEGIN IMMEDIATE") + connection.execute("SELECT revision FROM registry_state WHERE id = 1") + connection.rollback() + except sqlite3.Error as exc: + raise SharedStorageError( + "model-runtime.sqlite3 cannot take a BEGIN IMMEDIATE write " + "lock; all Workers must share one persistent config directory." + ) from exc + finally: + connection.close() + + # --- registry ------------------------------------------------------- + + def load_registry(self) -> RegistryV4: + """Load the stored registry, or a bootstrap document when absent.""" + with self._lock, self._connect() as connection: + row = connection.execute( + "SELECT revision, state, registry_json FROM registry_state WHERE id = 1" + ).fetchone() + if row is None: + return RegistryV4() + registry = RegistryV4.model_validate(json.loads(str(row[2]))) + return registry.model_copy( + update={"revision": int(row[0]), "state": str(row[1])} + ) + + def save_registry( + self, + *, + expected_revision: int, + registry: RegistryV4, + credential_writes: list[CredentialWrite] | None = None, + ) -> RegistryV4: + """Compare-and-swap the registry inside one ``BEGIN IMMEDIATE`` transaction. + + Credential versions are written first (immutable, with incrementing + revisions), then + the registry row is replaced with ``revision + 1``. A bootstrap + document atomically turns ``active`` the first time it carries an + enabled model, a valid primary, and a satisfied auth reference (4.3). + Any failure rolls the whole transaction back. + """ + now = int(time.time()) + with self._lock: + connection = self._connect() + try: + connection.execute("BEGIN IMMEDIATE") + row = connection.execute( + "SELECT revision, state FROM registry_state WHERE id = 1" + ).fetchone() + current_revision = int(row[0]) if row is not None else 1 + current_state = str(row[1]) if row is not None else "bootstrap" + if current_revision != expected_revision: + raise ModelRegistryError( + REGISTRY_REVISION_CONFLICT, + "registry revision conflict; reload and retry.", + details=[ + { + "path": "expected_revision", + "code": REGISTRY_REVISION_CONFLICT, + } + ], + ) + for write in credential_writes or []: + self._write_credential_version( + connection, write.credential_id, write.secret_value, now=now + ) + self._validate_defaults(registry) + new_state = self._resolve_state(connection, current_state, registry) + stored = registry.model_copy( + update={"revision": current_revision + 1, "state": new_state} + ) + connection.execute( + "INSERT INTO registry_state (id, revision, state, registry_json) " + "VALUES (1, ?, ?, ?) " + "ON CONFLICT (id) DO UPDATE SET " + "revision = excluded.revision, state = excluded.state, " + "registry_json = excluded.registry_json", + ( + stored.revision, + stored.state, + json.dumps( + stored.model_dump(mode="json"), + sort_keys=True, + separators=(",", ":"), + ), + ), + ) + connection.commit() + return stored + except BaseException: + connection.rollback() + raise + finally: + connection.close() + + def _validate_defaults(self, registry: RegistryV4) -> None: + """Enforce section 9.2 rule 7 on every save, regardless of state.""" + for role, ref in ( + ("primary", registry.defaults.primary), + ("auxiliary", registry.defaults.auxiliary), + ): + if ref is None: + continue + located = registry._locate(ref) + if located is None: + raise ModelRegistryError( + MODEL_DISABLED, + f"defaults.{role} references an unknown model.", + details=[{"path": f"defaults.{role}", "code": MODEL_DISABLED}], + ) + provider, model = located + if not (provider.enabled and model.enabled): + raise ModelRegistryError( + MODEL_DISABLED, + f"defaults.{role} must reference an enabled model.", + details=[{"path": f"defaults.{role}", "code": MODEL_DISABLED}], + ) + + def _resolve_state( + self, + connection: sqlite3.Connection, + current_state: str, + registry: RegistryV4, + ) -> str: + ready, failure = self._activation_readiness(connection, registry) + wants_active = current_state == "active" or registry.state == "active" + if wants_active and not ready: + assert failure is not None + raise failure + return "active" if ready else "bootstrap" + + def _activation_readiness( + self, connection: sqlite3.Connection, registry: RegistryV4 + ) -> tuple[bool, ModelRegistryError | None]: + """Check enabled model + valid primary + satisfied auth reference.""" + primary = registry.defaults.primary + located = registry._locate(primary) if primary is not None else None + if located is None: + return False, ModelRegistryError( + MODEL_DISABLED, + "an active registry requires defaults.primary to reference " + "an enabled model.", + details=[{"path": "defaults.primary", "code": MODEL_DISABLED}], + ) + provider, model = located + if not (provider.enabled and model.enabled): + return False, ModelRegistryError( + MODEL_DISABLED, + "an active registry requires defaults.primary to reference " + "an enabled model.", + details=[{"path": "defaults.primary", "code": MODEL_DISABLED}], + ) + if provider.auth.mode == "none": + return True, None + credential_id = provider.auth.credential_id + configured = False + if credential_id is not None: + row = connection.execute( + "SELECT current_revision FROM credential_pointers " + "WHERE credential_id = ?", + (credential_id,), + ).fetchone() + configured = row is not None + if not configured: + return False, ModelRegistryError( + CREDENTIAL_NOT_CONFIGURED, + "the primary provider's credential is not configured.", + details=[ + { + "path": f"providers[{provider.id}].auth", + "code": CREDENTIAL_NOT_CONFIGURED, + } + ], + ) + return True, None + + # --- credentials ----------------------------------------------------- + + def _write_credential_version( + self, + connection: sqlite3.Connection, + credential_id: str, + secret_value: str, + *, + now: int, + ) -> int: + """Append one immutable credential version; return its revision. + + Rewriting the exact current secret is idempotent and returns the + existing revision instead of creating a new one. + """ + _validate_credential_id(credential_id) + if not secret_value: + raise ValueError("secret_value must not be empty.") + pointer = connection.execute( + "SELECT current_revision FROM credential_pointers WHERE credential_id = ?", + (credential_id,), + ).fetchone() + if pointer is not None: + current_revision = int(pointer[0]) + latest = connection.execute( + "SELECT secret_value, status FROM credential_versions " + "WHERE credential_id = ? AND revision = ?", + (credential_id, current_revision), + ).fetchone() + if ( + latest is not None + and str(latest[1]) == "active" + and str(latest[0]) == secret_value + ): + return current_revision + new_revision = current_revision + 1 + else: + new_revision = 1 + connection.execute( + "INSERT INTO credential_versions " + "(credential_id, revision, secret_value, status, created_at) " + "VALUES (?, ?, ?, 'active', ?)", + (credential_id, new_revision, secret_value, now), + ) + connection.execute( + "INSERT INTO credential_pointers (credential_id, current_revision) " + "VALUES (?, ?) " + "ON CONFLICT (credential_id) DO UPDATE SET " + "current_revision = excluded.current_revision", + (credential_id, new_revision), + ) + return new_revision + + def write_credential_version(self, credential_id: str, secret_value: str) -> int: + """Idempotently prepare a new immutable credential version.""" + now = int(time.time()) + with self._lock: + connection = self._connect() + try: + connection.execute("BEGIN IMMEDIATE") + revision = self._write_credential_version( + connection, credential_id, secret_value, now=now + ) + connection.commit() + return revision + except BaseException: + connection.rollback() + raise + finally: + connection.close() + + def resolve_credential(self, credential_id: str, revision: int) -> str: + """Return the secret of one frozen credential version. + + Missing or destroyed versions raise + ``RUN_CREDENTIAL_REVISION_UNAVAILABLE``; callers must never silently + switch to a newer key (section 5.2). + """ + _validate_credential_id(credential_id) + with self._lock, self._connect() as connection: + row = connection.execute( + "SELECT secret_value, status FROM credential_versions " + "WHERE credential_id = ? AND revision = ?", + (credential_id, revision), + ).fetchone() + if row is None or str(row[1]) != "active": + raise ModelRegistryError( + RUN_CREDENTIAL_REVISION_UNAVAILABLE, + "the credential version frozen by this run is unavailable.", + details=[ + { + "path": "auth_ref.credential_revision", + "code": RUN_CREDENTIAL_REVISION_UNAVAILABLE, + } + ], + ) + return str(row[0]) + + def retire_credential_version(self, credential_id: str, revision: int) -> None: + """Destroy one credential version so it can no longer be resolved.""" + _validate_credential_id(credential_id) + now = int(time.time()) + with self._lock: + connection = self._connect() + try: + connection.execute("BEGIN IMMEDIATE") + cursor = connection.execute( + "UPDATE credential_versions " + "SET status = 'retired', retired_at = ? " + "WHERE credential_id = ? AND revision = ? " + "AND status = 'active'", + (now, credential_id, revision), + ) + if cursor.rowcount == 0: + raise ModelRegistryError( + RUN_CREDENTIAL_REVISION_UNAVAILABLE, + "the credential version is unavailable.", + ) + connection.commit() + except BaseException: + connection.rollback() + raise + finally: + connection.close() + + def credential_status(self, credential_id: str) -> CredentialStatus: + """Return the redacted browser-safe status; never the plaintext.""" + _validate_credential_id(credential_id) + with self._lock, self._connect() as connection: + pointer = connection.execute( + "SELECT current_revision FROM credential_pointers " + "WHERE credential_id = ?", + (credential_id,), + ).fetchone() + row = None + if pointer is not None: + row = connection.execute( + "SELECT secret_value, status, created_at " + "FROM credential_versions " + "WHERE credential_id = ? AND revision = ?", + (credential_id, int(pointer[0])), + ).fetchone() + if row is None or str(row[1]) != "active": + return CredentialStatus(credential_id=credential_id, configured=False) + secret = str(row[0]) + # Short secrets never leak through the hint at all. + hint = f"...{secret[-4:]}" if len(secret) >= 4 else None + return CredentialStatus( + credential_id=credential_id, + configured=True, + hint=hint, + updated_at=_rfc3339(int(row[2])), + ) + + # --- model verifications --------------------------------------------- + + def record_model_verification( + self, + *, + provider_id: str, + model_key: str, + configuration_hash: str, + credential_revision: int, + adapter_spec_revision: int, + result: str, + verified_capabilities: dict[str, bool], + error_code: str | None = None, + ) -> None: + """Upsert the latest test result for one verification five-tuple.""" + if result not in _VERIFICATION_RESULTS: + raise ValueError(f"result must be one of {_VERIFICATION_RESULTS}.") + now = int(time.time()) + with self._lock: + connection = self._connect() + try: + connection.execute("BEGIN IMMEDIATE") + connection.execute( + "INSERT INTO model_verifications " + "(provider_id, model_key, configuration_hash, " + " credential_revision, adapter_spec_revision, result, " + " verified_capabilities_json, verified_at, error_code) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) " + "ON CONFLICT (provider_id, model_key, configuration_hash, " + " credential_revision, adapter_spec_revision) DO UPDATE SET " + "result = excluded.result, " + "verified_capabilities_json = " + "excluded.verified_capabilities_json, " + "verified_at = excluded.verified_at, " + "error_code = excluded.error_code", + ( + provider_id, + model_key, + configuration_hash, + credential_revision, + adapter_spec_revision, + result, + json.dumps( + verified_capabilities, + sort_keys=True, + separators=(",", ":"), + ), + now, + error_code, + ), + ) + connection.commit() + except BaseException: + connection.rollback() + raise + finally: + connection.close() + + def get_model_verification( + self, + *, + provider_id: str, + model_key: str, + configuration_hash: str, + credential_revision: int, + adapter_spec_revision: int, + ) -> dict[str, Any] | None: + """Return the latest verification record for one five-tuple.""" + with self._lock, self._connect() as connection: + row = connection.execute( + "SELECT result, verified_capabilities_json, verified_at, " + "error_code FROM model_verifications " + "WHERE provider_id = ? AND model_key = ? " + "AND configuration_hash = ? AND credential_revision = ? " + "AND adapter_spec_revision = ?", + ( + provider_id, + model_key, + configuration_hash, + credential_revision, + adapter_spec_revision, + ), + ).fetchone() + if row is None: + return None + return { + "result": str(row[0]), + "verified_capabilities": json.loads(str(row[1])), + "verified_at": int(row[2]), + "error_code": row[3] if row[3] is None else str(row[3]), + } + + # --- run runtime snapshots --------------------------------------------- + + def insert_run_snapshot( + self, + *, + snapshot_id: str, + deployment_id: str, + thread_id: str, + run_request_id: str, + selection_hash: str, + payload: dict[str, Any], + expires_at: int, + status: str = "prepared", + langgraph_run_id: str | None = None, + ) -> None: + """Insert one snapshot row; the partial unique index guards the triplet.""" + if status not in _SNAPSHOT_STATUSES: + raise ValueError(f"status must be one of {_SNAPSHOT_STATUSES}.") + now = int(time.time()) + with self._lock: + connection = self._connect() + try: + connection.execute("BEGIN IMMEDIATE") + connection.execute( + "INSERT INTO run_runtime_snapshots " + "(snapshot_id, deployment_id, thread_id, run_request_id, " + " selection_hash, status, langgraph_run_id, payload_json, " + " created_at, expires_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + ( + snapshot_id, + deployment_id, + thread_id, + run_request_id, + selection_hash, + status, + langgraph_run_id, + json.dumps(payload, sort_keys=True, separators=(",", ":")), + now, + expires_at, + ), + ) + connection.commit() + except BaseException: + connection.rollback() + raise + finally: + connection.close() + + def set_run_snapshot_status( + self, + snapshot_id: str, + status: str, + *, + langgraph_run_id: str | None = None, + ) -> None: + """Update a snapshot's status (and optionally bind a LangGraph run).""" + if status not in _SNAPSHOT_STATUSES: + raise ValueError(f"status must be one of {_SNAPSHOT_STATUSES}.") + with self._lock: + connection = self._connect() + try: + connection.execute("BEGIN IMMEDIATE") + if langgraph_run_id is None: + connection.execute( + "UPDATE run_runtime_snapshots SET status = ? " + "WHERE snapshot_id = ?", + (status, snapshot_id), + ) + else: + connection.execute( + "UPDATE run_runtime_snapshots " + "SET status = ?, langgraph_run_id = ? " + "WHERE snapshot_id = ?", + (status, langgraph_run_id, snapshot_id), + ) + connection.commit() + except BaseException: + connection.rollback() + raise + finally: + connection.close() + + def get_run_snapshot(self, snapshot_id: str) -> dict[str, Any] | None: + """Return one snapshot row as a dict, or ``None`` when absent.""" + with self._lock, self._connect() as connection: + row = connection.execute( + "SELECT snapshot_id, deployment_id, thread_id, run_request_id, " + "selection_hash, status, langgraph_run_id, payload_json, " + "created_at, expires_at FROM run_runtime_snapshots " + "WHERE snapshot_id = ?", + (snapshot_id,), + ).fetchone() + if row is None: + return None + return { + "snapshot_id": str(row[0]), + "deployment_id": str(row[1]), + "thread_id": str(row[2]), + "run_request_id": str(row[3]), + "selection_hash": str(row[4]), + "status": str(row[5]), + "langgraph_run_id": row[6] if row[6] is None else str(row[6]), + "payload": json.loads(str(row[7])), + "created_at": int(row[8]), + "expires_at": int(row[9]), + } diff --git a/tests/test_model_registry_schemas.py b/tests/test_model_registry_schemas.py new file mode 100644 index 0000000..a86a5db --- /dev/null +++ b/tests/test_model_registry_schemas.py @@ -0,0 +1,625 @@ +"""Tests for the RegistryV4 schema, adapter contracts, and error taxonomy. + +These cover design doc sections 4.3 (configuration objects), 6.2 (adapter +parameter contracts), 6.4 (resolved config), 9.1 (read models), and 9.5 +(unified error structure). +""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from EvoScientist.model_registry.errors import ( + CREDENTIAL_NOT_CONFIGURED, + DELEGATION_REPLAYED, + ERROR_HTTP_STATUS, + MODEL_NOT_FOUND, + PROVIDER_UNREACHABLE, + REGISTRY_REVISION_CONFLICT, + ErrorDetail, + ErrorPayload, + ModelRegistryError, +) +from EvoScientist.model_registry.hashing import configuration_hash +from EvoScientist.model_registry.schemas import ( + AdapterParameterSpec, + AuthConfig, + AuthRef, + AuthSpec, + CredentialStatus, + CredentialWrite, + ModelAvailability, + ModelConfig, + ModelRef, + ModelRuntimeConfig, + ParameterRule, + ProviderConfig, + ProviderRuntimeConfig, + RegistryV4, + ResolvedModelConfig, + VerificationInfo, +) + +ALL_ERROR_CODES = { + "REGISTRY_REVISION_CONFLICT": 409, + "THREAD_MODEL_SELECTION_CONFLICT": 409, + "RUN_REQUEST_CONFLICT": 409, + "SNAPSHOT_ALREADY_BOUND": 409, + "SNAPSHOT_EXPIRED": 409, + "MODEL_CONFIGURATION_CHANGED": 409, + "DELEGATION_REPLAYED": 401, + "MODEL_NOT_FOUND": 404, + "MODEL_REGISTRY_NOT_READY": 422, + "MODEL_DISABLED": 422, + "MODEL_NOT_AVAILABLE": 422, + "MODEL_CONFIG_OUTSIDE_SNAPSHOT": 422, + "CREDENTIAL_NOT_CONFIGURED": 422, + "CREDENTIAL_REJECTED": 422, + "RUN_CREDENTIAL_REVISION_UNAVAILABLE": 422, + "AUTH_MODE_UNSUPPORTED": 422, + "ADAPTER_NOT_SUPPORTED": 422, + "CAPABILITY_UNSUPPORTED_BY_ADAPTER": 422, + "MODEL_CAPABILITY_UNAVAILABLE": 422, + "UNSUPPORTED_RUNTIME_PARAMETER": 422, + "MODEL_LIMITS_UNCONFIRMED": 422, + "CONTEXT_BUDGET_UNSATISFIABLE": 422, + "PROVIDER_UNREACHABLE": 422, +} + + +def _model_runtime(**overrides): + payload = { + "limit_mode": "combined", + "context_window_tokens": 1048576, + "max_input_tokens": None, + "max_output_tokens": 32768, + "min_effective_input_tokens": 8192, + "fixed_system_reserve_tokens": 4096, + "fixed_tools_reserve_tokens": 8192, + "fixed_attachments_reserve_tokens": 4096, + "limits_status": "confirmed", + "limits_source": "provider", + "temperature": None, + "top_p": None, + "reasoning_effort": "auto", + "declared_capabilities": { + "tools": True, + "vision": False, + "structured_output": True, + }, + } + payload.update(overrides) + return payload + + +def _provider(**overrides): + payload = { + "id": "zhipu-glm", + "name": "Zhipu GLM", + "adapter": "openai-compatible", + "base_url": "https://open.bigmodel.cn/api/paas/v4", + "auth": {"mode": "api_key", "credential_id": "zhipu-primary"}, + "enabled": True, + "runtime": { + "timeout_seconds": 120, + "max_retries": 2, + "default_temperature": 0.7, + "default_top_p": 0.95, + "default_reasoning_effort": "auto", + }, + "models": [ + { + "key": "glm-5.2", + "name": "GLM-5.2", + "upstream_model_id": "glm-5.2", + "enabled": True, + "runtime": _model_runtime(), + } + ], + } + payload.update(overrides) + return payload + + +def _registry(**overrides): + payload = { + "version": 4, + "revision": 1, + "state": "bootstrap", + "defaults": { + "primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"}, + "auxiliary": None, + }, + "providers": [_provider()], + } + payload.update(overrides) + return payload + + +class TestRegistryV4: + def test_design_doc_example_validates(self): + registry = RegistryV4.model_validate(_registry(state="active", revision=17)) + assert registry.version == 4 + assert registry.revision == 17 + assert registry.state == "active" + assert registry.defaults.primary == ModelRef( + provider_id="zhipu-glm", model_key="glm-5.2" + ) + assert registry.defaults.auxiliary is None + provider = registry.providers[0] + assert provider.runtime.timeout_seconds == 120 + model = provider.models[0] + assert model.runtime.min_effective_input_tokens == 8192 + assert model.runtime.declared_capabilities.tools is True + + def test_version_must_be_four(self): + with pytest.raises(ValidationError): + RegistryV4.model_validate(_registry(version=3)) + + def test_revision_must_be_positive(self): + with pytest.raises(ValidationError): + RegistryV4.model_validate(_registry(revision=0)) + + def test_bootstrap_allows_empty_providers_and_null_primary(self): + registry = RegistryV4.model_validate( + _registry( + state="bootstrap", + defaults={"primary": None, "auxiliary": None}, + providers=[], + ) + ) + assert registry.state == "bootstrap" + assert registry.providers == [] + + def test_active_requires_primary(self): + with pytest.raises(ValidationError): + RegistryV4.model_validate( + _registry( + state="active", + defaults={"primary": None, "auxiliary": None}, + ) + ) + + def test_active_requires_primary_referencing_enabled_model(self): + provider = _provider() + provider["models"][0]["enabled"] = False + with pytest.raises(ValidationError): + RegistryV4.model_validate(_registry(state="active", providers=[provider])) + + def test_active_rejects_primary_referencing_disabled_provider(self): + provider = _provider(enabled=False) + with pytest.raises(ValidationError): + RegistryV4.model_validate(_registry(state="active", providers=[provider])) + + def test_defaults_must_reference_existing_models(self): + with pytest.raises(ValidationError): + RegistryV4.model_validate( + _registry( + defaults={ + "primary": {"provider_id": "zhipu-glm", "model_key": "missing"}, + "auxiliary": None, + } + ) + ) + + def test_provider_ids_must_be_unique(self): + with pytest.raises(ValidationError): + RegistryV4.model_validate( + _registry(providers=[_provider(), _provider(name="Duplicate")]) + ) + + def test_model_keys_must_be_unique_per_provider(self): + provider = _provider() + provider["models"].append(dict(provider["models"][0])) + with pytest.raises(ValidationError): + ProviderConfig.model_validate(provider) + + +class TestIdPatterns: + @pytest.mark.parametrize( + "value", + ["Zhipu", "-abc", ".abc", "_abc", "", "a" * 65, "abc$", "abc def", "ABC"], + ) + def test_provider_id_rejects_invalid_patterns(self, value): + with pytest.raises(ValidationError): + ModelRef(provider_id=value, model_key="glm-5.2") + + @pytest.mark.parametrize( + "value", ["zhipu-glm", "a", "a.b_c-d", "glm-5.2", "0abc", "x" * 64] + ) + def test_provider_id_accepts_valid_patterns(self, value): + ref = ModelRef(provider_id=value, model_key="glm-5.2") + assert ref.provider_id == value + + def test_credential_id_uses_same_pattern(self): + with pytest.raises(ValidationError): + AuthConfig(mode="api_key", credential_id="Invalid Id") + + def test_upstream_model_id_preserves_case_and_bounds_length(self): + model = ModelConfig.model_validate( + { + "key": "glm-5.2", + "name": "GLM-5.2", + "upstream_model_id": "GLM-5.2-Air.X", + "enabled": True, + "runtime": _model_runtime(), + } + ) + assert model.upstream_model_id == "GLM-5.2-Air.X" + with pytest.raises(ValidationError): + ModelConfig.model_validate( + { + "key": "glm-5.2", + "name": "GLM-5.2", + "upstream_model_id": "m" * 301, + "enabled": True, + "runtime": _model_runtime(), + } + ) + with pytest.raises(ValidationError): + ModelConfig.model_validate( + { + "key": "glm-5.2", + "name": "GLM-5.2", + "upstream_model_id": "", + "enabled": True, + "runtime": _model_runtime(), + } + ) + + +class TestRuntimeRanges: + @pytest.mark.parametrize("value", [9, 601, 0, -10]) + def test_timeout_seconds_range(self, value): + with pytest.raises(ValidationError): + ProviderRuntimeConfig(timeout_seconds=value) + + @pytest.mark.parametrize("value", [10, 120, 600]) + def test_timeout_seconds_accepts_bounds(self, value): + assert ProviderRuntimeConfig(timeout_seconds=value).timeout_seconds == value + + @pytest.mark.parametrize("value", [-1, 6]) + def test_max_retries_range(self, value): + with pytest.raises(ValidationError): + ProviderRuntimeConfig(max_retries=value) + + def test_temperature_upper_bound_is_two(self): + assert ProviderRuntimeConfig(default_temperature=2).default_temperature == 2 + with pytest.raises(ValidationError): + ProviderRuntimeConfig(default_temperature=2.5) + with pytest.raises(ValidationError): + ModelRuntimeConfig.model_validate(_model_runtime(temperature=2.5)) + with pytest.raises(ValidationError): + ModelRuntimeConfig.model_validate(_model_runtime(temperature=-0.1)) + + def test_top_p_is_open_closed_interval(self): + with pytest.raises(ValidationError): + ProviderRuntimeConfig(default_top_p=0) + with pytest.raises(ValidationError): + ProviderRuntimeConfig(default_top_p=1.5) + assert ProviderRuntimeConfig(default_top_p=1).default_top_p == 1 + + def test_reasoning_effort_defaults_to_auto(self): + assert ProviderRuntimeConfig().default_reasoning_effort == "auto" + runtime = ModelRuntimeConfig.model_validate(_model_runtime()) + assert runtime.reasoning_effort == "auto" + with pytest.raises(ValidationError): + ModelRuntimeConfig.model_validate(_model_runtime(reasoning_effort="max")) + + def test_min_effective_input_tokens_default_and_floor(self): + runtime = ModelRuntimeConfig.model_validate( + _model_runtime(min_effective_input_tokens=4096) + ) + assert runtime.min_effective_input_tokens == 4096 + with pytest.raises(ValidationError): + ModelRuntimeConfig.model_validate( + _model_runtime(min_effective_input_tokens=1023) + ) + + def test_fixed_reserves_are_non_negative(self): + with pytest.raises(ValidationError): + ModelRuntimeConfig.model_validate( + _model_runtime(fixed_system_reserve_tokens=-1) + ) + + +class TestLimitModes: + def test_combined_requires_context_window(self): + with pytest.raises(ValidationError): + ModelRuntimeConfig.model_validate( + _model_runtime(limit_mode="combined", context_window_tokens=None) + ) + + def test_input_only_requires_max_input_tokens(self): + with pytest.raises(ValidationError): + ModelRuntimeConfig.model_validate( + _model_runtime(limit_mode="input_only", max_input_tokens=None) + ) + + def test_input_only_valid(self): + runtime = ModelRuntimeConfig.model_validate( + _model_runtime( + limit_mode="input_only", + context_window_tokens=None, + max_input_tokens=131072, + ) + ) + assert runtime.max_input_tokens == 131072 + + +class TestAuthConfig: + def test_mode_none_forbids_credential_id(self): + with pytest.raises(ValidationError): + AuthConfig(mode="none", credential_id="zhipu-primary") + + def test_mode_none_allows_null_credential(self): + auth = AuthConfig(mode="none", credential_id=None) + assert auth.credential_id is None + + def test_api_key_allows_credential_reference(self): + auth = AuthConfig(mode="api_key", credential_id="zhipu-primary") + assert auth.mode == "api_key" + + def test_unknown_mode_rejected(self): + with pytest.raises(ValidationError): + AuthConfig(mode="oauth") + + +class TestAdapterContracts: + def _glm_spec_payload(self): + return { + "adapter_id": "openai-compatible", + "spec_revision": 1, + "model_selector": "glm-5.2", + "connection": { + "chat_model": "ChatOpenAI", + "model_field": "model", + "base_url_field": "base_url", + }, + "auth_specs": { + "api_key": { + "credential_required": True, + "credential_kind": "api_key", + "target": "client_option", + "target_name": "api_key", + } + }, + "parameters": { + "timeout_seconds": { + "supported": True, + "value_type": "integer", + "minimum": 10, + "maximum": 600, + "nullable": "forbidden", + "target": "client_option", + "target_name": "timeout", + "normalizer": "identity", + }, + "temperature": { + "supported": True, + "value_type": "number", + "minimum": 0, + "maximum": 1, + "nullable": "omit", + "target": "request_option", + "target_name": "temperature", + "normalizer": "omit_when_none", + }, + "reasoning_effort": { + "supported": False, + "value_type": "enum", + "nullable": "omit", + "target": "request_option", + "target_name": "", + "normalizer": "reject_non_auto", + }, + }, + "protocol_capabilities": { + "tools": True, + "vision": False, + "structured_output": True, + }, + } + + def test_adapter_parameter_spec_from_design_doc(self): + spec = AdapterParameterSpec.model_validate(self._glm_spec_payload()) + assert spec.adapter_id == "openai-compatible" + assert spec.spec_revision == 1 + assert spec.model_selector == "glm-5.2" + assert spec.auth_specs["api_key"].credential_kind == "api_key" + assert spec.parameters["temperature"].maximum == 1 + assert spec.parameters["reasoning_effort"].supported is False + assert spec.protocol_capabilities.structured_output is True + + def test_auth_spec_rejects_unknown_credential_kind(self): + with pytest.raises(ValidationError): + AuthSpec( + credential_required=True, + credential_kind="oauth_token", + target="client_option", + target_name="api_key", + ) + + def test_parameter_rule_conflicts_default_empty(self): + rule = ParameterRule( + supported=True, + value_type="integer", + nullable="forbidden", + target="client_option", + target_name="max_retries", + normalizer="identity", + ) + assert rule.conflicts_with == [] + + +class TestAvailabilityAndResolvedConfig: + def test_model_availability_shape(self): + availability = ModelAvailability( + model_ref=ModelRef(provider_id="zhipu-glm", model_key="glm-5.2"), + state="enabled", + selectable=True, + reason_code=None, + verification=VerificationInfo(status="passed"), + effective_capabilities={ + "tools": True, + "vision": False, + "structured_output": True, + }, + ) + assert availability.selectable is True + assert availability.verification.status == "passed" + + def test_resolved_model_config_has_no_secret_value(self): + resolved = ResolvedModelConfig( + model_ref=ModelRef(provider_id="zhipu-glm", model_key="glm-5.2"), + role="primary", + adapter_id="openai-compatible", + adapter_spec_revision=1, + upstream_model_id="glm-5.2", + base_url="https://open.bigmodel.cn/api/paas/v4", + auth_ref=AuthRef( + mode="api_key", credential_id="zhipu-primary", credential_revision=3 + ), + client_options={"timeout_seconds": 120, "max_retries": 2}, + request_options={ + "max_output_tokens": 32768, + "temperature": 0.7, + "top_p": 0.95, + "reasoning_effort": "auto", + }, + budget={ + "resolved_input_limit": 1048576 - 32768, + "fixed_reserves": { + "fixed_system_reserve_tokens": 4096, + "fixed_tools_reserve_tokens": 8192, + "fixed_attachments_reserve_tokens": 4096, + }, + "message_budget": 1048576 - 32768 - 16384, + }, + effective_capabilities={ + "tools": True, + "vision": False, + "structured_output": True, + }, + ) + assert "secret" not in ResolvedModelConfig.model_fields + assert "secret_value" not in AuthRef.model_fields + assert resolved.auth_ref.credential_revision == 3 + + def test_auth_ref_mode_none_forbids_credential(self): + with pytest.raises(ValidationError): + AuthRef(mode="none", credential_id="zhipu-primary") + + +class TestCredentialModels: + def test_credential_status_shape(self): + status = CredentialStatus( + credential_id="zhipu-primary", + configured=True, + hint="...abcd", + updated_at="2026-07-20T00:00:00Z", + ) + assert status.hint == "...abcd" + + def test_credential_write_requires_replace_operation(self): + write = CredentialWrite( + credential_id="zhipu-primary", secret_value="sk-test-1234" + ) + assert write.operation == "replace" + with pytest.raises(ValidationError): + CredentialWrite( + credential_id="zhipu-primary", + operation="append", + secret_value="sk-test-1234", + ) + + +class TestErrorTaxonomy: + def test_error_table_covers_all_documented_codes(self): + assert ERROR_HTTP_STATUS == ALL_ERROR_CODES + + def test_spot_check_http_mappings(self): + assert ERROR_HTTP_STATUS[REGISTRY_REVISION_CONFLICT] == 409 + assert ERROR_HTTP_STATUS[DELEGATION_REPLAYED] == 401 + assert ERROR_HTTP_STATUS[MODEL_NOT_FOUND] == 404 + assert ERROR_HTTP_STATUS[PROVIDER_UNREACHABLE] == 422 + + def test_error_payload_matches_section_9_5(self): + payload = ErrorPayload( + code="STABLE_ERROR_CODE", + message="safe human-readable message", + details=[ + ErrorDetail( + path="providers[0].runtime.temperature", + code="UNSUPPORTED_RUNTIME_PARAMETER", + ) + ], + request_id="req-1", + ) + assert payload.model_dump() == { + "code": "STABLE_ERROR_CODE", + "message": "safe human-readable message", + "details": [ + { + "path": "providers[0].runtime.temperature", + "code": "UNSUPPORTED_RUNTIME_PARAMETER", + } + ], + "request_id": "req-1", + } + + def test_registry_error_carries_code_status_and_payload(self): + error = ModelRegistryError( + CREDENTIAL_NOT_CONFIGURED, + "credential is not configured", + details=[{"path": "providers[0].auth", "code": CREDENTIAL_NOT_CONFIGURED}], + ) + assert error.code == CREDENTIAL_NOT_CONFIGURED + assert error.http_status == 422 + payload = error.payload(request_id="req-9") + assert payload["code"] == CREDENTIAL_NOT_CONFIGURED + assert payload["request_id"] == "req-9" + assert payload["details"] == [ + {"path": "providers[0].auth", "code": CREDENTIAL_NOT_CONFIGURED} + ] + + def test_unknown_error_code_rejected(self): + with pytest.raises(ValueError, match="Unknown model registry error code"): + ModelRegistryError("NOT_A_REAL_CODE", "bad") + + +class TestConfigurationHash: + def test_hash_is_deterministic(self): + provider = ProviderConfig.model_validate(_provider()) + first = configuration_hash(provider, provider.models[0]) + second = configuration_hash(provider, provider.models[0]) + assert first == second + assert len(first) == 64 + + def test_hash_changes_with_runtime_parameters(self): + provider = ProviderConfig.model_validate(_provider()) + changed = ProviderConfig.model_validate( + _provider(runtime={"timeout_seconds": 300}) + ) + assert configuration_hash(provider, provider.models[0]) != configuration_hash( + changed, changed.models[0] + ) + + def test_hash_changes_with_base_url_and_upstream_model(self): + provider = ProviderConfig.model_validate(_provider()) + other_url = ProviderConfig.model_validate( + _provider(base_url="https://example.com/v1") + ) + assert configuration_hash(provider, provider.models[0]) != configuration_hash( + other_url, other_url.models[0] + ) + + def test_hash_changes_with_limits(self): + provider = ProviderConfig.model_validate(_provider()) + changed_limits = _provider() + changed_limits["models"][0]["runtime"] = _model_runtime(max_output_tokens=16384) + other = ProviderConfig.model_validate(changed_limits) + assert configuration_hash(provider, provider.models[0]) != configuration_hash( + other, other.models[0] + ) diff --git a/tests/test_model_registry_store.py b/tests/test_model_registry_store.py new file mode 100644 index 0000000..cedacc5 --- /dev/null +++ b/tests/test_model_registry_store.py @@ -0,0 +1,406 @@ +"""Tests for the SQLite-backed ModelRuntimeStore (design doc 4.2, 5.1, 8.2).""" + +from __future__ import annotations + +import sqlite3 +import stat +import threading + +import pytest + +from EvoScientist.model_registry.errors import ( + CREDENTIAL_NOT_CONFIGURED, + MODEL_DISABLED, + REGISTRY_REVISION_CONFLICT, + RUN_CREDENTIAL_REVISION_UNAVAILABLE, + ModelRegistryError, +) +from EvoScientist.model_registry.schemas import CredentialWrite, RegistryV4 +from EvoScientist.model_registry.store import ModelRuntimeStore + + +def _model_runtime(**overrides): + payload = { + "limit_mode": "combined", + "context_window_tokens": 1048576, + "max_input_tokens": None, + "max_output_tokens": 32768, + "min_effective_input_tokens": 8192, + "fixed_system_reserve_tokens": 4096, + "fixed_tools_reserve_tokens": 8192, + "fixed_attachments_reserve_tokens": 4096, + "limits_status": "confirmed", + "limits_source": "provider", + "temperature": None, + "top_p": None, + "reasoning_effort": "auto", + "declared_capabilities": { + "tools": True, + "vision": False, + "structured_output": True, + }, + } + payload.update(overrides) + return payload + + +def _registry_payload(**overrides): + payload = { + "version": 4, + "revision": 1, + "state": "bootstrap", + "defaults": { + "primary": {"provider_id": "zhipu-glm", "model_key": "glm-5.2"}, + "auxiliary": None, + }, + "providers": [ + { + "id": "zhipu-glm", + "name": "Zhipu GLM", + "adapter": "openai-compatible", + "base_url": "https://open.bigmodel.cn/api/paas/v4", + "auth": {"mode": "api_key", "credential_id": "zhipu-primary"}, + "enabled": True, + "runtime": {"timeout_seconds": 120, "max_retries": 2}, + "models": [ + { + "key": "glm-5.2", + "name": "GLM-5.2", + "upstream_model_id": "glm-5.2", + "enabled": True, + "runtime": _model_runtime(), + } + ], + } + ], + } + payload.update(overrides) + return payload + + +def _registry(**overrides) -> RegistryV4: + return RegistryV4.model_validate(_registry_payload(**overrides)) + + +def _local_registry() -> RegistryV4: + """A registry whose provider needs no credential (mode=none).""" + payload = _registry_payload() + provider = payload["providers"][0] + provider["id"] = "local-ollama" + provider["adapter"] = "ollama" + provider["base_url"] = "http://localhost:11434" + provider["auth"] = {"mode": "none", "credential_id": None} + payload["defaults"]["primary"] = { + "provider_id": "local-ollama", + "model_key": "glm-5.2", + } + return RegistryV4.model_validate(payload) + + +class TestBootstrap: + def test_empty_database_loads_bootstrap_registry(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + registry = store.load_registry() + assert registry.version == 4 + assert registry.revision == 1 + assert registry.state == "bootstrap" + assert registry.providers == [] + assert registry.defaults.primary is None + assert registry.defaults.auxiliary is None + + def test_database_path_and_permissions(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + assert store.database_path == tmp_path / "model-runtime.sqlite3" + file_mode = stat.S_IMODE(store.database_path.stat().st_mode) + dir_mode = stat.S_IMODE(tmp_path.stat().st_mode) + assert file_mode == 0o600 + assert dir_mode == 0o700 + + def test_default_config_dir(self, tmp_path, monkeypatch): + from EvoScientist.model_registry import store as store_module + + fake_home = tmp_path / "home" + monkeypatch.setattr( + store_module, "DEFAULT_CONFIG_DIR", fake_home / ".config" / "evoscientist" + ) + store = ModelRuntimeStore() + assert store.database_path.parent.name == "evoscientist" + assert store.database_path.parent.parent.name == ".config" + assert store.database_path.exists() + + def test_check_shared_storage_succeeds_on_local_volume(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + store.check_shared_storage() + + +class TestRegistrySave: + def test_first_ready_save_transitions_to_active(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + saved = store.save_registry( + expected_revision=1, + registry=_local_registry(), + ) + assert saved.state == "active" + assert saved.revision == 2 + loaded = store.load_registry() + assert loaded.state == "active" + assert loaded.revision == 2 + + def test_bootstrap_save_without_ready_primary_stays_bootstrap(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + registry = _registry( + defaults={"primary": None, "auxiliary": None}, + ) + saved = store.save_registry(expected_revision=1, registry=registry) + assert saved.state == "bootstrap" + assert saved.revision == 2 + + def test_activation_requires_configured_credential(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + # auth.mode=api_key with credential_id, but no credential written. + saved = store.save_registry(expected_revision=1, registry=_registry()) + assert saved.state == "bootstrap" + + def test_credential_writes_in_same_transaction_activate(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + saved = store.save_registry( + expected_revision=1, + registry=_registry(), + credential_writes=[ + CredentialWrite( + credential_id="zhipu-primary", secret_value="sk-live-9876abcd" + ) + ], + ) + assert saved.state == "active" + assert store.resolve_credential("zhipu-primary", 1) == "sk-live-9876abcd" + + def test_explicit_active_state_without_credential_fails(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + with pytest.raises(ModelRegistryError) as excinfo: + store.save_registry( + expected_revision=1, + registry=_registry(state="active"), + ) + assert excinfo.value.code == CREDENTIAL_NOT_CONFIGURED + # The failed save must not leave partial state behind. + assert store.load_registry().revision == 1 + + def test_stale_expected_revision_conflicts(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + store.save_registry(expected_revision=1, registry=_local_registry()) + with pytest.raises(ModelRegistryError) as excinfo: + store.save_registry(expected_revision=1, registry=_local_registry()) + assert excinfo.value.code == REGISTRY_REVISION_CONFLICT + assert store.load_registry().revision == 2 + + def test_concurrent_saves_single_winner(self, tmp_path): + first = ModelRuntimeStore(config_dir=tmp_path) + first.save_registry(expected_revision=1, registry=_local_registry()) + + barrier = threading.Barrier(2) + outcomes: list[str] = [] + + def attempt(config_dir): + store = ModelRuntimeStore(config_dir=config_dir) + barrier.wait(timeout=10) + try: + store.save_registry(expected_revision=2, registry=_local_registry()) + outcomes.append("ok") + except ModelRegistryError as exc: + outcomes.append(exc.code) + + threads = [threading.Thread(target=attempt, args=(tmp_path,)) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=30) + + assert sorted(outcomes, key=str.lower) == ["ok", REGISTRY_REVISION_CONFLICT] + assert first.load_registry().revision == 3 + + def test_defaults_must_reference_enabled_models(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + payload = _registry_payload() + payload["providers"].append( + { + "id": "local-ollama", + "name": "Local Ollama", + "adapter": "ollama", + "base_url": "http://localhost:11434", + "auth": {"mode": "none", "credential_id": None}, + "enabled": True, + "runtime": {"timeout_seconds": 120, "max_retries": 2}, + "models": [ + { + "key": "qwen3", + "name": "Qwen3", + "upstream_model_id": "qwen3", + "enabled": False, + "runtime": _model_runtime(), + } + ], + } + ) + payload["defaults"]["auxiliary"] = { + "provider_id": "local-ollama", + "model_key": "qwen3", + } + with pytest.raises(ModelRegistryError) as excinfo: + store.save_registry( + expected_revision=1, + registry=RegistryV4.model_validate(payload), + ) + assert excinfo.value.code == MODEL_DISABLED + + +class TestCredentials: + def test_write_creates_incrementing_immutable_versions(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + first = store.write_credential_version("zhipu-primary", "sk-first-1111") + second = store.write_credential_version("zhipu-primary", "sk-second-2222") + assert (first, second) == (1, 2) + # Old revisions stay intact and resolvable. + assert store.resolve_credential("zhipu-primary", 1) == "sk-first-1111" + assert store.resolve_credential("zhipu-primary", 2) == "sk-second-2222" + + def test_rewriting_same_secret_is_idempotent(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + first = store.write_credential_version("zhipu-primary", "sk-same-3333") + again = store.write_credential_version("zhipu-primary", "sk-same-3333") + assert again == first + + def test_resolve_missing_or_retired_version_fails(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + store.write_credential_version("zhipu-primary", "sk-live-4444") + with pytest.raises(ModelRegistryError) as excinfo: + store.resolve_credential("zhipu-primary", 99) + assert excinfo.value.code == RUN_CREDENTIAL_REVISION_UNAVAILABLE + + store.retire_credential_version("zhipu-primary", 1) + with pytest.raises(ModelRegistryError) as excinfo: + store.resolve_credential("zhipu-primary", 1) + assert excinfo.value.code == RUN_CREDENTIAL_REVISION_UNAVAILABLE + + def test_retire_unknown_version_fails(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + with pytest.raises(ModelRegistryError) as excinfo: + store.retire_credential_version("zhipu-primary", 1) + assert excinfo.value.code == RUN_CREDENTIAL_REVISION_UNAVAILABLE + + def test_credential_status_is_masked(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + store.write_credential_version("zhipu-primary", "sk-live-9876abcd") + status = store.credential_status("zhipu-primary") + assert status.credential_id == "zhipu-primary" + assert status.configured is True + assert status.hint == "...abcd" + assert status.updated_at is not None + payload = status.model_dump_json() + assert "sk-live-9876abcd" not in payload + + def test_short_secret_never_leaks_through_hint(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + store.write_credential_version("zhipu-primary", "abc") + status = store.credential_status("zhipu-primary") + assert status.configured is True + assert status.hint is None + + def test_unknown_credential_status(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + status = store.credential_status("missing") + assert status.configured is False + assert status.hint is None + assert status.updated_at is None + + +class TestModelVerifications: + def test_upsert_keeps_only_latest_result(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + key = { + "provider_id": "zhipu-glm", + "model_key": "glm-5.2", + "configuration_hash": "a" * 64, + "credential_revision": 1, + "adapter_spec_revision": 1, + } + store.record_model_verification( + **key, + result="failed", + verified_capabilities={ + "tools": False, + "vision": False, + "structured_output": False, + }, + error_code="PROVIDER_UNREACHABLE", + ) + store.record_model_verification( + **key, + result="passed", + verified_capabilities={ + "tools": True, + "vision": False, + "structured_output": True, + }, + ) + record = store.get_model_verification(**key) + assert record is not None + assert record["result"] == "passed" + assert record["verified_capabilities"]["tools"] is True + assert record["error_code"] is None + with sqlite3.connect(store.database_path) as connection: + count = connection.execute( + "SELECT COUNT(*) FROM model_verifications" + ).fetchone()[0] + assert count == 1 + + +class TestRunRuntimeSnapshots: + def _insert(self, store, **overrides): + payload = { + "snapshot_id": "snap-1", + "deployment_id": "local", + "thread_id": "thread-1", + "run_request_id": "req-1", + "selection_hash": "b" * 64, + "payload": {"primary": None}, + "expires_at": 4_000_000_000, + } + payload.update(overrides) + return store.insert_run_snapshot(**payload) + + def test_partial_unique_index_blocks_duplicate_active_triplet(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + self._insert(store, snapshot_id="snap-1") + with pytest.raises(sqlite3.IntegrityError): + self._insert(store, snapshot_id="snap-2") + + def test_triplet_reusable_after_terminal_status(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + self._insert(store, snapshot_id="snap-1") + store.set_run_snapshot_status("snap-1", "expired") + self._insert(store, snapshot_id="snap-2") + + store.set_run_snapshot_status("snap-2", "aborted") + self._insert(store, snapshot_id="snap-3") + + snapshot = store.get_run_snapshot("snap-3") + assert snapshot is not None + assert snapshot["status"] == "prepared" + assert snapshot["langgraph_run_id"] is None + + def test_bound_snapshot_keeps_langgraph_run_id(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + self._insert(store, snapshot_id="snap-1") + store.set_run_snapshot_status("snap-1", "bound", langgraph_run_id="lg-run-1") + snapshot = store.get_run_snapshot("snap-1") + assert snapshot["status"] == "bound" + assert snapshot["langgraph_run_id"] == "lg-run-1" + # A bound snapshot still occupies the triplet. + with pytest.raises(sqlite3.IntegrityError): + self._insert(store, snapshot_id="snap-2") + + def test_invalid_snapshot_status_rejected(self, tmp_path): + store = ModelRuntimeStore(config_dir=tmp_path) + with pytest.raises(ValueError, match="status must be one of"): + self._insert(store, status="running")