Files
EvoScientist-Multi/EvoScientist/sessions.py
T
m4 5a581c78a2
Build / build (push) Has been cancelled
Docker / build (push) Has been cancelled
Lint / ruff (push) Has been cancelled
Test / pytest (ubuntu-latest, 3.11) (push) Has been cancelled
Test / pytest (ubuntu-latest, 3.12) (push) Has been cancelled
Test / pytest (windows-latest, 3.11) (push) Has been cancelled
Test / pytest (windows-latest, 3.12) (push) Has been cancelled
feat: add scoped model runtime configuration
Introduce provider, model, and invocation contracts with encrypted configuration persistence. Add web runtime fencing, route fallback, recovery middleware, workspace scoping, and comprehensive tests.
2026-08-14 22:03:04 +08:00

2026 lines
81 KiB
Python

"""Session persistence using LangGraph's SQLite checkpoint storage.
Provides thread CRUD operations, prefix-matched resume, and an async
context manager for the shared ``AsyncSqliteSaver`` checkpointer.
Adapted from upstream ``deepagents_cli/sessions.py``.
Per-step pruning:
LangGraph's checkpointer writes a full state snapshot per super-step,
causing unbounded growth (multi-GB sessions.db). EvoScientist never
reads historical checkpoints — resume always reads the latest, HITL
interrupts attach pending writes to the just-written row. So
``get_checkpointer()`` yields a ``PruningCheckpointer`` that prunes
older rows for the same ``(thread_id, checkpoint_ns)`` after every
``aput()``. The first-run migration sweep cleans up legacy bloat.
WebUI / langgraph-dev checkpointer:
``create_checkpointer_for_langgraph_api()`` — the ``checkpointer.path``
target in ``langgraph_dev/langgraph.json`` — backs every ``langgraph
dev`` subprocess (deploy / WebUI / CLI-spawned) with this same SQLite
file instead of the default pickle-based ``InMemorySaver``, whose flush
window and pickle-compatibility failures lose session history on
restart (issue #277). On startup it purges leftover evomemory-worker
rows and rebuilds the in-memory thread registry from SQLite. See
``_restore_webui_threads_to_global_store`` for the restore scope and
``_ApiPruningCheckpointer`` for the metadata stamping that makes WebUI
threads first-class CLI sessions.
"""
import asyncio
import atexit
import hashlib
import logging
import math
import time
import uuid
from collections.abc import AsyncIterator, Awaitable, Callable
from contextlib import asynccontextmanager
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, cast
import aiosqlite
from langchain_core.messages import (
AnyMessage,
BaseMessage,
RemoveMessage,
convert_to_messages,
)
from langchain_core.runnables import RunnableConfig
from langgraph.checkpoint.serde.jsonplus import JsonPlusSerializer
from langgraph.checkpoint.sqlite.aio import AsyncSqliteSaver
from langgraph.graph.message import REMOVE_ALL_MESSAGES
from langgraph.types import Overwrite
_logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Monkey-patch aiosqlite for langgraph-checkpoint >= 2.1.0 compatibility
# ---------------------------------------------------------------------------
if not hasattr(aiosqlite.Connection, "is_alive"):
def _is_alive(self: aiosqlite.Connection) -> bool:
return self._connection is not None
aiosqlite.Connection.is_alive = _is_alive # type: ignore[attr-defined]
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
AGENT_NAME = "EvoScientist"
MAIN_THREAD_FILTER_SQL = (
"json_extract(metadata, '$.agent_name') = ? "
"AND (json_extract(metadata, '$.graph_id') IS NULL "
" OR json_extract(metadata, '$.graph_id') = ?)"
)
MAIN_THREAD_FILTER_PARAMS = (AGENT_NAME, AGENT_NAME)
_CHECKPOINT_MSGPACK_MODULES = frozenset(
{
("EvoScientist.llm.errors", "AgentControlError"),
("EvoScientist.llm.errors", "ModelToolProtocolError"),
("EvoScientist.llm.errors", "ProviderStreamError"),
}
)
def _checkpoint_serde() -> JsonPlusSerializer:
"""Return the checkpoint serializer with app-owned types allowlisted."""
return JsonPlusSerializer(allowed_msgpack_modules=_CHECKPOINT_MSGPACK_MODULES)
# ---------------------------------------------------------------------------
# Paths & ID generation
# ---------------------------------------------------------------------------
def _to_short_path(path: str) -> str:
"""Try to convert a Windows path to its 8.3 short form.
On Windows, sqlite3 may fail to open databases at paths containing
non-ASCII characters (e.g., Chinese usernames). Short paths are
ASCII-safe when available, but conversion is best-effort: it fails
when 8.3 name generation is disabled, on non-NTFS volumes, or for
nonexistent targets. Returns the original path on non-Windows or
on failure.
"""
import sys
if sys.platform != "win32":
return path
import ctypes
buf = ctypes.create_unicode_buffer(32767)
if ctypes.windll.kernel32.GetShortPathNameW(path, buf, len(buf)):
return buf.value
return path
def get_db_path() -> Path:
"""Return the sessions database path, creating parents.
Uses ``paths.DATA_DIR`` (~/.evoscientist/ by default), then applies
a best-effort Windows 8.3 short-path conversion on the *directory*
(which exists after ``mkdir``) so sqlite3 can handle non-ASCII paths.
"""
from .paths import DATA_DIR
db_dir = DATA_DIR
db_dir.mkdir(parents=True, exist_ok=True)
return Path(_to_short_path(str(db_dir))) / "sessions.db"
def short_thread_id(thread_id: str) -> str:
"""First 8 chars for display (git-style); legacy 8-hex ids pass through.
All lookup commands (``/resume``, ``/delete``) accept prefixes, so the
shortened form is always a usable handle.
"""
return thread_id[:8]
def generate_thread_id() -> str:
"""Generate a full-UUID thread ID.
UUID format (not the legacy 8-char hex) so CLI threads are addressable
by langgraph-api — its thread endpoints reject non-UUID ids — which is
what lets the WebUI list and resume CLI sessions. UIs display the
first 8 chars; ``/resume`` prefix matching is unaffected. Pre-existing
8-char hex threads keep working in the CLI but stay CLI-only.
"""
return str(uuid.uuid4())
# ---------------------------------------------------------------------------
# Checkpoint pruning
# ---------------------------------------------------------------------------
# Default kept when the caller cannot resolve config (tests, unit-init paths).
# Production callers use ``EvoScientistConfig.checkpoint_keep_per_thread``.
# Kept in sync with the dataclass default so config-failure fallbacks
# don't silently regress to the pre-DeltaChannel aggressive value (which
# could prune away ``_DeltaSnapshot`` seeds and break message replay).
_DEFAULT_KEEP_PER_NS = 1000
class PruningCheckpointer(AsyncSqliteSaver):
"""``AsyncSqliteSaver`` that prunes stale checkpoints after every ``aput()``.
After a successful ``aput()``, deletes rows in ``checkpoints`` and
``writes`` whose ``(thread_id, checkpoint_ns)`` matches the just-written
row but whose ``checkpoint_id`` is not among the ``keep_per_ns`` most
recent ids. The just-written row is always kept (it is the head of the
descending order and ``keep_per_ns >= 1`` is enforced).
Inherits from ``AsyncSqliteSaver`` (rather than wrapping it) so
LangGraph's ``compile()`` ``isinstance(x, BaseCheckpointSaver)`` check
succeeds. All other behavior — ``aget_tuple``, ``alist``,
``aput_writes``, ``adelete_thread``, ``setup``, the async context
manager protocol, the connection lock — is inherited unchanged.
HITL safety: pregel records pending writes (e.g. ``interrupt``) against
the checkpoint id returned by the most recent ``aput()``, which is
exactly the row we keep. Older rows can never receive new writes after
a newer ``aput()`` lands, so deleting them is provably safe.
Setting ``keep_per_ns <= 0`` disables pruning (escape hatch for debug).
"""
def __init__(
self,
conn: aiosqlite.Connection,
*,
keep_per_ns: int = _DEFAULT_KEEP_PER_NS,
serde: Any = None,
) -> None:
super().__init__(
conn, serde=serde if serde is not None else _checkpoint_serde()
)
self._keep_per_ns = max(0, int(keep_per_ns))
# Outer lock guarantees ``super().aput()`` and ``_prune_after_put()``
# are atomic *as a pair*. Without this, a concurrent ``aput()`` on a
# different ``(thread_id, checkpoint_ns)`` could land between the
# two phases and squeeze the earlier caller's just-written row out
# of the top-N retention window (only matters when ``keep_per_ns``
# is small or N parallel writers race; harmless otherwise but the
# invariant "the just-written row is always kept" must hold).
self._aput_lock = asyncio.Lock()
@classmethod
@asynccontextmanager
async def from_conn_string_with_keep(
cls, conn_string: str, keep_per_ns: int = _DEFAULT_KEEP_PER_NS
) -> AsyncIterator["PruningCheckpointer"]:
"""Build a ``PruningCheckpointer`` from a SQLite connection string.
Mirrors ``AsyncSqliteSaver.from_conn_string`` but threads
``keep_per_ns`` into ``__init__``. The native ``from_conn_string``
classmethod cannot accept extra kwargs, so callers that need
retention control should use this method instead.
"""
async with aiosqlite.connect(conn_string) as conn:
yield cls(conn, keep_per_ns=keep_per_ns)
async def aput(
self,
config: Any,
checkpoint: Any,
metadata: Any,
new_versions: Any,
) -> Any:
"""Delegate to ``super().aput``, then prune older rows atomically.
Wraps both the inner write and the prune in ``self._aput_lock`` so
a concurrent ``aput()`` cannot squeeze this caller's just-written
row out of the top-N retention window. The inner ``self.lock``
(held by ``super().aput`` and by ``_prune_after_put``) is a
separate, finer-grained lock that protects the SQLite connection;
the outer lock here is about the put+prune pair invariant.
Pruning is best-effort: any exception is logged at WARNING and
swallowed so a transient SQLite error never fails the agent step.
"""
async with self._aput_lock:
result = await super().aput(config, checkpoint, metadata, new_versions)
if self._keep_per_ns <= 0:
return result
try:
thread_id = config["configurable"]["thread_id"]
checkpoint_ns = config["configurable"].get("checkpoint_ns", "") or ""
await self._prune_after_put(str(thread_id), str(checkpoint_ns))
except Exception as exc: # pragma: no cover - defensive
_logger.warning("checkpoint pruning failed: %s", exc, exc_info=True)
return result
# Safety cap on the snapshot walk. Upstream default
# ``snapshot_frequency`` is 1000; ``DELTA_MAX_SUPERSTEPS_SINCE_SNAPSHOT``
# is 5000. 10000 is a generous ceiling that catches pathological data
# (cycles, malformed parents) without raising a hard limit on normal
# operation.
_MAX_SNAPSHOT_WALK_STEPS = 10000
async def _prune_after_put(self, thread_id: str, checkpoint_ns: str) -> None:
"""Prune old checkpoints with DeltaChannel awareness.
Naively keeping the N most-recent rows can sever the
``_DeltaSnapshot`` chain that ``messages`` reconstruction
depends on — the surviving "latest" checkpoint is rarely a
snapshot point itself, so delta channels silently reconstruct
as empty (upstream ``BaseCheckpointSaver.prune`` spells out the
same failure mode).
After selecting the N most-recent anchor ids, walk back from
the OLDEST anchor's parent via ``parent_checkpoint_id`` until
hitting an ancestor whose ``channel_values["messages"]`` is a
seed (``_DeltaSnapshot`` blob or plain list — both detected via
``_unwrap_messages_seed``). All visited ancestors are preserved
alongside the anchor set. Anchors form a contiguous head, so a
single walk from the oldest one covers all of them.
Restricted to rows whose ``metadata.agent_name == AGENT_NAME``.
``json_extract(metadata, '$.agent_name') = ?`` evaluates to NULL
(and so fails the predicate) for any row whose metadata lacks an
``agent_name`` key — by design, those rows belong to third-party
LangGraph users and must never be pruned by us.
Walk + DELETEs held under ``self.lock`` for atomicity with
concurrent ``aput()`` on the same thread.
"""
keep = self._keep_per_ns
agent = AGENT_NAME
async with self.lock:
# ``writes`` table is checked inside ``_delete_outside`` so a
# legacy DB that only has ``checkpoints`` still gets pruned
# (writes DELETE silently skipped; checkpoints DELETE runs).
# The migration sweep depends on this — it walks legacy DBs
# that often pre-date the ``writes`` table entirely.
anchor_ids = await self._fetch_recent_checkpoint_ids(
thread_id, checkpoint_ns, agent, keep
)
if len(anchor_ids) < keep:
return # nothing to prune yet
extra_preserve = await self._walk_to_snapshot_ancestor(
thread_id, checkpoint_ns, anchor_ids[-1]
)
kept = set(anchor_ids) | extra_preserve
await self._delete_outside(thread_id, checkpoint_ns, agent, kept)
await self.conn.commit()
async def _fetch_recent_checkpoint_ids(
self,
thread_id: str,
checkpoint_ns: str,
agent: str,
limit: int,
) -> list[str]:
"""Return the ``limit`` most-recent checkpoint ids (newest first)."""
query = (
"SELECT checkpoint_id FROM checkpoints "
"WHERE thread_id = ? AND checkpoint_ns = ? "
" AND json_extract(metadata, '$.agent_name') = ? "
"ORDER BY checkpoint_id DESC LIMIT ?"
)
async with self.conn.execute(
query, (thread_id, checkpoint_ns, agent, limit)
) as cur:
rows = await cur.fetchall()
return [r[0] for r in rows]
async def _walk_to_snapshot_ancestor(
self,
thread_id: str,
checkpoint_ns: str,
oldest_anchor_id: str,
) -> set[str]:
"""Walk the parent chain until a messages seed is found."""
extra: set[str] = set()
cursor = await self._fetch_parent_checkpoint_id(
thread_id, checkpoint_ns, oldest_anchor_id
)
steps = 0
while cursor is not None and steps < self._MAX_SNAPSHOT_WALK_STEPS:
steps += 1
blob = await self._fetch_checkpoint_blob(thread_id, checkpoint_ns, cursor)
if blob is None:
break
extra.add(cursor)
try:
ck = self.serde.loads_typed(blob)
except Exception as exc:
_logger.warning(
"Failed to deserialize checkpoint %s while walking to "
"snapshot for thread %s: %s",
cursor,
thread_id,
exc,
)
break
cv = ck.get("channel_values") or {}
if _unwrap_messages_seed(cv.get("messages")) is not None:
break
cursor = await self._fetch_parent_checkpoint_id(
thread_id, checkpoint_ns, cursor
)
return extra
async def _fetch_parent_checkpoint_id(
self, thread_id: str, checkpoint_ns: str, checkpoint_id: str
) -> str | None:
query = (
"SELECT parent_checkpoint_id FROM checkpoints "
"WHERE thread_id = ? AND checkpoint_ns = ? AND checkpoint_id = ?"
)
async with self.conn.execute(
query, (thread_id, checkpoint_ns, checkpoint_id)
) as cur:
row = await cur.fetchone()
return row[0] if row and row[0] else None
async def _fetch_checkpoint_blob(
self, thread_id: str, checkpoint_ns: str, checkpoint_id: str
) -> tuple[str, bytes] | None:
query = (
"SELECT type, checkpoint FROM checkpoints "
"WHERE thread_id = ? AND checkpoint_ns = ? AND checkpoint_id = ?"
)
async with self.conn.execute(
query, (thread_id, checkpoint_ns, checkpoint_id)
) as cur:
row = await cur.fetchone()
if not row or not row[0] or not row[1]:
return None
return (row[0], row[1])
async def _delete_outside(
self,
thread_id: str,
checkpoint_ns: str,
agent: str,
kept_ids: set[str],
) -> None:
"""Delete checkpoints outside the retained head and seed chain."""
if not kept_ids:
return
kept_list = list(kept_ids)
placeholders = ",".join("?" * len(kept_list))
if await _table_exists(self.conn, "writes"):
del_writes = (
"DELETE FROM writes "
"WHERE thread_id = ? AND checkpoint_ns = ? "
" AND checkpoint_id IN ("
" SELECT checkpoint_id FROM checkpoints "
" WHERE thread_id = ? AND checkpoint_ns = ? "
" AND json_extract(metadata, '$.agent_name') = ? "
f" AND checkpoint_id NOT IN ({placeholders})"
" )"
)
await self.conn.execute(
del_writes,
(
thread_id,
checkpoint_ns,
thread_id,
checkpoint_ns,
agent,
*kept_list,
),
)
del_checkpoints = (
"DELETE FROM checkpoints "
"WHERE thread_id = ? AND checkpoint_ns = ? "
" AND json_extract(metadata, '$.agent_name') = ? "
f" AND checkpoint_id NOT IN ({placeholders})"
)
await self.conn.execute(
del_checkpoints,
(thread_id, checkpoint_ns, agent, *kept_list),
)
@dataclass(frozen=True, slots=True)
class TurnLease:
thread_id: str
owner_id: str
fencing_token: int
expires_at_ms: int
checkpoint_snapshot_id: str
class FencedPruningCheckpointer(PruningCheckpointer):
"""Single-worker linearizable turn lease around LangGraph checkpoint writes."""
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self._turn_fence_lock = asyncio.Lock()
@classmethod
@asynccontextmanager
async def from_conn_string_with_keep(
cls, conn_string: str, keep_per_ns: int = _DEFAULT_KEEP_PER_NS
) -> AsyncIterator["FencedPruningCheckpointer"]:
async with aiosqlite.connect(conn_string) as conn:
saver = cls(conn, keep_per_ns=keep_per_ns)
await saver.setup_fencing()
yield saver
async def setup_fencing(self) -> None:
await super().setup()
async with self.lock:
await self.conn.executescript(
"""
CREATE TABLE IF NOT EXISTS thread_turn_fences (
thread_id TEXT PRIMARY KEY,
generation INTEGER NOT NULL CHECK (generation > 0),
owner_id TEXT,
expires_at_ms INTEGER,
updated_at_ms INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS thread_checkpoint_versions (
thread_id TEXT PRIMARY KEY,
sequence INTEGER NOT NULL DEFAULT 0 CHECK (sequence >= 0),
checkpoint_id TEXT,
updated_at_ms INTEGER NOT NULL
);
"""
)
await self.conn.commit()
async def acquire_turn_lease(
self,
thread_id: str,
owner_id: str,
*,
ttl_seconds: int,
) -> TurnLease:
clean_thread = str(thread_id or "").strip()
clean_owner = str(owner_id or "").strip()
if not clean_thread or not clean_owner or ttl_seconds < 1:
raise ValueError("thread, owner and positive TTL are required")
async with self._turn_fence_lock, self.lock:
now = time.time_ns() // 1_000_000
await self.conn.execute("BEGIN IMMEDIATE")
try:
row = await (
await self.conn.execute(
"SELECT generation, owner_id, expires_at_ms FROM thread_turn_fences WHERE thread_id=?",
(clean_thread,),
)
).fetchone()
if (
row is not None
and row[1]
and row[1] != clean_owner
and int(row[2] or 0) >= now
):
raise RuntimeError("TURN_LEASE_BUSY")
generation = int(row[0]) + 1 if row is not None else 1
expires_at = now + ttl_seconds * 1000
await self.conn.execute(
"""INSERT INTO thread_turn_fences
(thread_id, generation, owner_id, expires_at_ms, updated_at_ms)
VALUES (?, ?, ?, ?, ?)
ON CONFLICT(thread_id) DO UPDATE SET
generation=excluded.generation,
owner_id=excluded.owner_id,
expires_at_ms=excluded.expires_at_ms,
updated_at_ms=excluded.updated_at_ms""",
(clean_thread, generation, clean_owner, expires_at, now),
)
version = await (
await self.conn.execute(
"SELECT sequence, checkpoint_id FROM thread_checkpoint_versions WHERE thread_id=?",
(clean_thread,),
)
).fetchone()
sequence = int(version[0]) if version is not None else 0
checkpoint_id = (
str(version[1] or "root") if version is not None else "root"
)
if version is None:
await self.conn.execute(
"""INSERT INTO thread_checkpoint_versions
(thread_id, sequence, checkpoint_id, updated_at_ms)
VALUES (?, 0, NULL, ?)""",
(clean_thread, now),
)
await self.conn.commit()
except Exception:
await self.conn.rollback()
raise
snapshot = hashlib.sha256(
f"{clean_thread}\0{sequence}\0{checkpoint_id}".encode()
).hexdigest()
return TurnLease(
clean_thread, clean_owner, generation, expires_at, f"sha256:{snapshot}"
)
async def renew_turn_lease(
self, lease: TurnLease, *, ttl_seconds: int
) -> TurnLease:
async with self._turn_fence_lock, self.lock:
now = time.time_ns() // 1_000_000
expires_at = now + ttl_seconds * 1000
cursor = await self.conn.execute(
"""UPDATE thread_turn_fences
SET expires_at_ms=?, updated_at_ms=?
WHERE thread_id=? AND generation=? AND owner_id=?
AND expires_at_ms>=?""",
(
expires_at,
now,
lease.thread_id,
lease.fencing_token,
lease.owner_id,
now,
),
)
await self.conn.commit()
if cursor.rowcount != 1:
raise RuntimeError("TURN_LEASE_LOST")
return TurnLease(
lease.thread_id,
lease.owner_id,
lease.fencing_token,
expires_at,
lease.checkpoint_snapshot_id,
)
async def release_turn_lease(self, lease: TurnLease) -> bool:
async with self._turn_fence_lock, self.lock:
now = time.time_ns() // 1_000_000
cursor = await self.conn.execute(
"""UPDATE thread_turn_fences
SET owner_id=NULL, expires_at_ms=NULL, updated_at_ms=?
WHERE thread_id=? AND generation=? AND owner_id=?""",
(now, lease.thread_id, lease.fencing_token, lease.owner_id),
)
await self.conn.commit()
return cursor.rowcount == 1
async def _require_write_lease(self, config: Any) -> tuple[str, int]:
configurable = dict(config.get("configurable") or {})
thread_id = str(configurable.get("thread_id") or "")
if not thread_id.startswith("web:"):
return thread_id, 0
owner_id = str(configurable.get("turn_lease_owner") or "")
token = int(configurable.get("turn_fencing_token") or 0)
now = time.time_ns() // 1_000_000
row = await (
await self.conn.execute(
"""SELECT 1 FROM thread_turn_fences
WHERE thread_id=? AND generation=? AND owner_id=?
AND expires_at_ms>=?""",
(thread_id, token, owner_id, now),
)
).fetchone()
if row is None:
raise RuntimeError("TURN_FENCED")
return thread_id, token
async def aput(
self, config: Any, checkpoint: Any, metadata: Any, new_versions: Any
) -> Any:
async with self._turn_fence_lock:
async with self.lock:
thread_id, token = await self._require_write_lease(config)
result = await super().aput(config, checkpoint, metadata, new_versions)
if token == 0:
return result
checkpoint_id = str(
result.get("configurable", {}).get("checkpoint_id") or ""
)
async with self.lock:
now = time.time_ns() // 1_000_000
await self.conn.execute(
"""INSERT INTO thread_checkpoint_versions
(thread_id, sequence, checkpoint_id, updated_at_ms)
VALUES (?, 1, ?, ?)
ON CONFLICT(thread_id) DO UPDATE SET
sequence=thread_checkpoint_versions.sequence+1,
checkpoint_id=excluded.checkpoint_id,
updated_at_ms=excluded.updated_at_ms""",
(thread_id, checkpoint_id, now),
)
await self.conn.commit()
return result
async def aput_writes(
self,
config: Any,
writes: Any,
task_id: str,
task_path: str = "",
) -> None:
async with self._turn_fence_lock:
async with self.lock:
await self._require_write_lease(config)
await super().aput_writes(config, writes, task_id, task_path)
# ---------------------------------------------------------------------------
# Checkpointer context manager
# ---------------------------------------------------------------------------
def _resolve_keep_per_ns() -> int:
"""Resolve the retention count from EvoScientistConfig, with safe fallback."""
try:
from .config import get_effective_config
return max(0, int(get_effective_config().checkpoint_keep_per_thread))
except Exception: # pragma: no cover - defensive (config import errors)
return _DEFAULT_KEEP_PER_NS
@asynccontextmanager
async def get_checkpointer() -> AsyncIterator[FencedPruningCheckpointer]:
"""Yield a pruning-enabled checkpointer connected to the sessions DB.
Wraps ``AsyncSqliteSaver`` with ``PruningCheckpointer`` so every
super-step trims the per-(thread, ns) history to ``keep_per_ns``. The
retention count is read from ``EvoScientistConfig`` at context entry;
setting it to 0 disables pruning entirely.
Also runs the legacy-bloat migration sweep synchronously *before*
yielding the saver when needed. Sequencing sweep ahead of any agent
``aput()`` eliminates the SQLite file-lock contention that produced
"database is locked" when channel inbound raced the sweep mid-DELETE.
The sweep is gated by ``PRAGMA user_version`` so it runs at most
once across all future launches; subsequent invocations cost nothing.
On failure ``user_version`` is NOT bumped, so the next launch retries.
"""
keep = _resolve_keep_per_ns()
async with FencedPruningCheckpointer.from_conn_string_with_keep(
str(get_db_path()), keep_per_ns=keep
) as saver:
# The whole gate is wrapped in a broad try/except: any unexpected
# failure (incl. preview / status print) must degrade to "yield
# the saver, log a warning" — never to "hang startup".
if keep > 0:
try:
if await _needs_migration():
import time
from rich.console import Console
# stderr-bound so non-interactive callers redirecting
# stdout don't capture migration progress noise.
console = Console(stderr=True)
size_str, pair_count, size_bytes = await _migration_preview()
eta_str = _format_duration(_estimate_sweep_seconds(size_bytes))
console.print(
f"[dim]·[/dim] Compacting sessions DB "
f"([cyan]{size_str}[/cyan], "
f"[yellow]{pair_count} thread-namespace pairs[/yellow]"
f"[dim], est. ~{eta_str}[/dim])"
)
t0 = time.time()
with console.status(
"[dim]Compacting...[/dim]", spinner="dots"
) as status:
async def _on_progress(done: int, total: int) -> None:
elapsed = time.time() - t0
pct = (done * 100 // total) if total else 0
eta = (elapsed / done) * (total - done) if done > 0 else 0
status.update(
f"[dim]Compacting[/dim] "
f"[yellow]{pct}%[/yellow] "
f"[dim]({done}/{total} pairs · "
f"{_format_duration(elapsed)} elapsed · "
f"~{_format_duration(eta)} remaining)[/dim]"
)
await _run_migration_sweep(keep, progress_cb=_on_progress)
console.print(
f"[dim]·[/dim] [green]✓[/green] Compaction done in "
f"[green]{_format_duration(time.time() - t0)}[/green]"
)
except Exception as exc:
_logger.warning("migration sweep failed: %s", exc, exc_info=True)
yield saver
# ---------------------------------------------------------------------------
# Internal helpers
# ---------------------------------------------------------------------------
async def _table_exists(conn: aiosqlite.Connection, table: str) -> bool:
query = "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?"
async with conn.execute(query, (table,)) as cur:
return await cur.fetchone() is not None
def _reduce_messages_delta(
state: list[AnyMessage] | None, writes: list[Any]
) -> list[AnyMessage]:
"""Inline copy of deepagents' ``_messages_delta_reducer``.
The upstream reducer lives in ``deepagents._messages_reducer`` (a
private module) and itself adapts langgraph's experimental
``_messages_delta_reducer`` (PR #7729). Both surfaces are
pre-stable — langgraph marks DeltaChannel as Beta, and the deepagents
file's leading underscore signals it's not part of the public API.
We copy the implementation here so a future upstream rename or
semantic shift doesn't silently break thread reconstruction.
Behavior MUST stay equivalent: dedups by message ``id``, tombstones
via ``RemoveMessage``, resets on ``REMOVE_ALL_MESSAGES``. ID-less
messages are appended without ID assignment — checkpointers
serialize pending writes before ``update()`` runs, so IDs assigned
inside the reducer never reach stored writes and would differ on
replay, defeating deduplication.
Raw dict / string / tuple inputs are coerced to typed ``BaseMessage``
so HTTP-driven graphs (and persisted blobs that round-tripped
through JSON) reconstruct correctly without a separate coercion
step.
``state`` may be None on ``DeltaChannel.replay_writes`` for threads
whose earliest checkpoint did not seed ``messages: []``, and is
treated as the empty list.
"""
flat: list[Any] = []
for w in writes:
if isinstance(w, list):
flat.extend(w)
else:
flat.append(w)
# Steady-state writes from this module already typed; only raw input
# (deserialized blobs, dict shorthands) needs ``convert_to_messages``.
state_msgs: list[AnyMessage] = (
state
if state and isinstance(state[0], BaseMessage)
else cast("list[AnyMessage]", convert_to_messages(state or []))
)
msgs: list[AnyMessage] = cast("list[AnyMessage]", convert_to_messages(flat))
# ``REMOVE_ALL_MESSAGES`` resets everything; honor the last sentinel
# in the batch — discard prior state plus every write before it.
remove_all_idx: int | None = None
for idx, m in enumerate(msgs):
if isinstance(m, RemoveMessage) and m.id == REMOVE_ALL_MESSAGES:
remove_all_idx = idx
if remove_all_idx is not None:
state_msgs = []
msgs = msgs[remove_all_idx + 1 :]
index: dict[str, int] = {
m.id: i for i, m in enumerate(state_msgs) if m.id is not None
}
result: list[AnyMessage | None] = list(state_msgs)
for msg in msgs:
mid = msg.id
if mid is None:
result.append(msg)
elif isinstance(msg, RemoveMessage):
if mid in index:
result[index[mid]] = None
del index[mid]
elif mid in index:
result[index[mid]] = msg
else:
index[mid] = len(result)
result.append(msg)
return [m for m in result if m is not None]
async def _load_checkpoint_messages(
saver: AsyncSqliteSaver,
thread_id: str,
) -> list:
"""Load messages from the most recent checkpoint for *thread_id*.
Delegates the ``messages`` DeltaChannel walk to upstream
``BaseCheckpointSaver.aget_delta_channel_history`` — it finds the
nearest ancestor whose ``channel_values["messages"]`` carries a seed
(``_DeltaSnapshot`` blob or plain list) and returns that plus every
on-path pending write oldest→newest. The local
``_reduce_messages_delta`` (inline copy of deepagents' reducer; see
that function's docstring) is then applied in a single batched call,
preserving dedup-by-id, ``RemoveMessage`` tombstones, and
``REMOVE_ALL_MESSAGES`` reset semantics.
``Overwrite`` (``langgraph.types.Overwrite``) is not a message-like
and the reducer doesn't recognize it — split the batch at each
occurrence, replacing accumulated state with the wrapped value before
resuming reducer application.
``_summarization_event`` doesn't ride on the ``messages`` channel,
so it's fetched separately from the latest checkpoint's
``channel_values``.
Returns a list of LangChain message objects, or an empty list on failure.
"""
# Pre-resolve the latest main EvoScientist checkpoint_id, then pin it
# into the config so
# ``aget_tuple`` fetches THAT specific row. Without the pin,
# ``aget_tuple`` returns the latest by ``checkpoint_id`` alone — in
# a multi-agent DB where a third-party tool shares the same
# ``(thread_id, checkpoint_ns)`` and happens to have a higher id,
# we'd leak that agent's transcript into our /resume. The ancestor
# walk via ``parent_checkpoint_id`` chain is unambiguous (specific
# ids), so pinning the head is sufficient — the rest of the chain
# follows EvoScientist's parent links.
head_query = (
"SELECT checkpoint_id FROM checkpoints "
"WHERE thread_id = ? AND checkpoint_ns = '' "
f" AND {MAIN_THREAD_FILTER_SQL} "
"ORDER BY checkpoint_id DESC LIMIT 1"
)
async with saver.conn.execute(
head_query, (thread_id, *MAIN_THREAD_FILTER_PARAMS)
) as cur:
head_row = await cur.fetchone()
if head_row is None:
return []
config: RunnableConfig = {
"configurable": {
"thread_id": thread_id,
"checkpoint_ns": "",
"checkpoint_id": head_row[0],
}
}
target = await saver.aget_tuple(config)
if target is None:
return []
# ``aget_delta_channel_history`` walks from ``target.parent_config`` —
# it deliberately excludes the target itself (its caller is the
# runtime preparing to apply a NEW delta on top). For /resume we want
# the state AT the latest checkpoint, so check the target's own
# ``channel_values`` first, falling back to the ancestor walk only
# when no seed is materialized locally.
target_cv = target.checkpoint.get("channel_values") or {}
target_seed = _unwrap_messages_seed(target_cv.get("messages"))
if target_seed is not None:
accumulated: list = target_seed
writes: list = [w for w in (target.pending_writes or []) if w[1] == "messages"]
else:
history = await saver.aget_delta_channel_history(
config=config, channels=["messages"]
)
entry = history.get("messages", {})
accumulated = _unwrap_messages_seed(entry.get("seed")) or []
writes = list(entry.get("writes", []))
# ``target.pending_writes`` are the deltas recorded at this step;
# apply them on top of whatever the ancestor walk reconstructed.
writes.extend(w for w in (target.pending_writes or []) if w[1] == "messages")
# Batched reducer: collect contiguous message-like writes and flush
# in one call. Overwrite splits the batch because it resets state
# rather than appending.
batch: list = []
def _flush() -> None:
nonlocal accumulated
if not batch:
return
try:
accumulated = _reduce_messages_delta(accumulated, batch)
except Exception as exc:
_logger.warning(
"Failed to apply %d messages deltas for thread %s: %s",
len(batch),
thread_id,
exc,
)
batch.clear()
for _task_id, _channel, delta in writes:
# ``Overwrite`` wraps a value with "replace this channel"
# semantics — flush any pending writes first, then reset state
# to the wrapped value.
if isinstance(delta, Overwrite):
_flush()
inner = getattr(delta, "value", None)
accumulated = (
list(inner)
if isinstance(inner, list)
else ([inner] if inner is not None else [])
)
else:
batch.append(delta)
_flush()
# ``_summarization_event`` rides on its own channel; pick it off the
# target's ``channel_values`` we already deserialized above.
event = target_cv.get("_summarization_event")
summarization_event = event if isinstance(event, dict) else None
if not accumulated:
await _log_orphan_warning_if_pruned(saver.conn, thread_id)
if not isinstance(accumulated, list):
return []
return _apply_summarization_event(accumulated, summarization_event)
async def _log_orphan_warning_if_pruned(
conn: aiosqlite.Connection, thread_id: str
) -> None:
"""Emit a WARNING if *thread_id* has orphan ``parent_checkpoint_id`` refs.
Empty reconstructed history + a broken parent chain is the signature
of a pre-fix DB where the old ``keep_per_ns=10`` default pruned early
writes before the snapshot frequency materialized a ``_DeltaSnapshot``
seed. The DB has no path to recover those messages — log so /resume
showing a stub history isn't silent.
"""
query = """
SELECT 1 FROM checkpoints c1
WHERE c1.thread_id = ? AND c1.checkpoint_ns = ''
AND c1.parent_checkpoint_id IS NOT NULL
AND NOT EXISTS (
SELECT 1 FROM checkpoints c2
WHERE c2.thread_id = c1.thread_id
AND c2.checkpoint_ns = ''
AND c2.checkpoint_id = c1.parent_checkpoint_id
)
LIMIT 1
"""
async with conn.execute(query, (thread_id,)) as cur:
if await cur.fetchone():
_logger.warning(
"Thread %s has orphan checkpoints (pre-fix pruning) "
"and no surviving DeltaChannel snapshot; reconstructed "
"history is empty. Early messages cannot be recovered.",
thread_id,
)
def _unwrap_messages_seed(value: object) -> list | None:
"""Coerce a ``channel_values["messages"]`` snapshot seed into a plain list.
LangGraph 1.2 stores snapshot blobs as ``_DeltaSnapshot(value=[...])``
(a ``NamedTuple``, NOT a list subclass), so a bare ``isinstance(v, list)``
check silently ignores the seed and reconstruction starts from whatever
writes survived pruning. Pre-migration / non-DeltaChannel checkpoints
still store a plain list. Returns ``None`` when no usable seed is
present (caller leaves accumulated state untouched).
"""
if value is None:
return None
if isinstance(value, list):
return list(value)
inner = getattr(value, "value", None)
if isinstance(inner, list):
return list(inner)
return None
def _apply_summarization_event(messages: list, event: dict | None) -> list:
"""Return the effective message list after applying a summarization event."""
if not event:
return list(messages)
try:
summary_message = event["summary_message"]
cutoff_index = int(event["cutoff_index"])
except (KeyError, TypeError, ValueError):
return list(messages)
if summary_message is None:
return list(messages)
if cutoff_index < 0 or cutoff_index > len(messages):
return list(messages)
return [summary_message, *messages[cutoff_index:]]
def _extract_preview(messages: list, max_len: int = 50) -> str:
"""Extract the first human message as a preview string."""
for msg in messages:
if getattr(msg, "type", None) != "human":
continue
content = getattr(msg, "content", "") or ""
if isinstance(content, list):
parts = [
b.get("text", "")
for b in content
if isinstance(b, dict) and b.get("type") == "text"
]
content = " ".join(parts)
content = content.strip()
if content:
return content[:max_len] + "..." if len(content) > max_len else content
return ""
def _format_relative_time(iso_ts: str | None) -> str:
"""Convert ISO timestamp to a human-readable relative string."""
if not iso_ts:
return ""
try:
dt = datetime.fromisoformat(iso_ts)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=UTC)
now = datetime.now(UTC)
delta = now - dt
seconds = int(delta.total_seconds())
if seconds < 60:
return "just now"
minutes = seconds // 60
if minutes < 60:
return f"{minutes} min ago"
hours = minutes // 60
if hours < 24:
return f"{hours} hour{'s' if hours != 1 else ''} ago"
days = hours // 24
if days < 30:
return f"{days} day{'s' if days != 1 else ''} ago"
months = days // 30
return f"{months} month{'s' if months != 1 else ''} ago"
except (ValueError, TypeError):
return ""
# ---------------------------------------------------------------------------
# Thread CRUD
# ---------------------------------------------------------------------------
async def list_threads(
limit: int = 20,
include_message_count: bool = False,
include_preview: bool = False,
) -> list[dict]:
"""List EvoScientist threads, most-recent first.
Returns list of dicts with keys: ``thread_id``, ``updated_at``,
``workspace_dir``, ``model``, and optionally ``message_count``
and ``preview``.
"""
db_path = str(get_db_path())
async with aiosqlite.connect(db_path, timeout=30.0) as conn:
if not await _table_exists(conn, "checkpoints"):
return []
query = f"""
SELECT thread_id,
MAX(json_extract(metadata, '$.updated_at')) as updated_at,
json_extract(metadata, '$.workspace_dir') as workspace_dir,
json_extract(metadata, '$.model') as model
FROM checkpoints
WHERE {MAIN_THREAD_FILTER_SQL}
GROUP BY thread_id
ORDER BY updated_at DESC
"""
params: tuple = MAIN_THREAD_FILTER_PARAMS
if limit > 0:
query += " LIMIT ?\n"
params = (*MAIN_THREAD_FILTER_PARAMS, limit)
async with conn.execute(query, params) as cur:
rows = await cur.fetchall()
threads = [
{
"thread_id": r[0],
"updated_at": r[1],
"workspace_dir": r[2],
"model": r[3],
}
for r in rows
]
if (include_message_count or include_preview) and threads:
# Share one saver across all threads so ``setup()`` runs once.
serde = _checkpoint_serde()
saver = AsyncSqliteSaver(conn, serde=serde)
for t in threads:
msgs = await _load_checkpoint_messages(saver, t["thread_id"])
if include_message_count:
t["message_count"] = len(msgs)
if include_preview:
t["preview"] = _extract_preview(msgs)
return threads
async def get_most_recent() -> str | None:
"""Return the most recent EvoScientist thread ID, or ``None``."""
db_path = str(get_db_path())
async with aiosqlite.connect(db_path, timeout=30.0) as conn:
if not await _table_exists(conn, "checkpoints"):
return None
query = f"""
SELECT thread_id FROM checkpoints
WHERE {MAIN_THREAD_FILTER_SQL}
ORDER BY checkpoint_id DESC
LIMIT 1
"""
async with conn.execute(query, MAIN_THREAD_FILTER_PARAMS) as cur:
row = await cur.fetchone()
return row[0] if row else None
async def thread_exists(thread_id: str) -> bool:
"""Return ``True`` if *thread_id* has at least one EvoScientist checkpoint."""
db_path = str(get_db_path())
async with aiosqlite.connect(db_path, timeout=30.0) as conn:
if not await _table_exists(conn, "checkpoints"):
return False
query = f"""
SELECT 1 FROM checkpoints
WHERE thread_id = ? AND {MAIN_THREAD_FILTER_SQL}
LIMIT 1
"""
async with conn.execute(query, (thread_id, *MAIN_THREAD_FILTER_PARAMS)) as cur:
return (await cur.fetchone()) is not None
async def find_similar_threads(thread_id: str, limit: int = 5) -> list[str]:
"""Find EvoScientist thread IDs that start with *thread_id* (prefix match)."""
db_path = str(get_db_path())
async with aiosqlite.connect(db_path, timeout=30.0) as conn:
if not await _table_exists(conn, "checkpoints"):
return []
# Escape SQL LIKE wildcards so user-supplied prefixes are matched
# literally (e.g. `--resume %` must not match every thread).
escaped = (
thread_id.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
)
query = f"""
SELECT DISTINCT thread_id
FROM checkpoints
WHERE thread_id LIKE ? ESCAPE '\\'
AND {MAIN_THREAD_FILTER_SQL}
ORDER BY thread_id
LIMIT ?
"""
async with conn.execute(
query, (escaped + "%", *MAIN_THREAD_FILTER_PARAMS, limit)
) as cur:
rows = await cur.fetchall()
return [r[0] for r in rows]
async def resolve_thread_id_prefix(tid: str) -> tuple[str | None, list[str]]:
"""Resolve a (possibly partial) thread ID.
Returns ``(resolved_id, matches)``:
- ``(full_id, [])`` when *tid* is an exact hit or a unique prefix.
- ``(None, [a, b, ...])`` when the prefix is ambiguous (multiple matches).
- ``(None, [])`` when no thread matches.
"""
if await thread_exists(tid):
return tid, []
similar = await find_similar_threads(tid)
if len(similar) == 1:
return similar[0], []
return None, similar
async def delete_thread(thread_id: str) -> bool:
"""Delete all EvoScientist checkpoints (and writes) for *thread_id*."""
db_path = str(get_db_path())
async with aiosqlite.connect(db_path, timeout=30.0) as conn:
if not await _table_exists(conn, "checkpoints"):
return False
# Delete writes FIRST — the subquery needs checkpoints to still exist
if await _table_exists(conn, "writes"):
await conn.execute(
f"""DELETE FROM writes
WHERE thread_id = ?
AND checkpoint_id IN (
SELECT checkpoint_id FROM checkpoints
WHERE thread_id = ?
AND {MAIN_THREAD_FILTER_SQL}
)""",
(thread_id, thread_id, *MAIN_THREAD_FILTER_PARAMS),
)
cur = await conn.execute(
f"DELETE FROM checkpoints WHERE thread_id = ? AND {MAIN_THREAD_FILTER_SQL}",
(thread_id, *MAIN_THREAD_FILTER_PARAMS),
)
deleted = cur.rowcount > 0
await conn.commit()
return deleted
async def get_thread_metadata(thread_id: str) -> dict | None:
"""Return metadata dict for *thread_id*, or ``None`` if not found.
Keys: ``workspace_dir``, ``model``, ``updated_at``.
"""
db_path = str(get_db_path())
async with aiosqlite.connect(db_path, timeout=30.0) as conn:
if not await _table_exists(conn, "checkpoints"):
return None
query = f"""
SELECT json_extract(metadata, '$.workspace_dir') as workspace_dir,
json_extract(metadata, '$.model') as model,
json_extract(metadata, '$.updated_at') as updated_at
FROM checkpoints
WHERE thread_id = ?
AND {MAIN_THREAD_FILTER_SQL}
ORDER BY checkpoint_id DESC
LIMIT 1
"""
async with conn.execute(query, (thread_id, *MAIN_THREAD_FILTER_PARAMS)) as cur:
row = await cur.fetchone()
if not row:
return None
return {
"workspace_dir": row[0],
"model": row[1],
"updated_at": row[2],
}
async def get_thread_messages(thread_id: str) -> list:
"""Return the list of LangChain message objects for *thread_id*.
Only returns messages for EvoScientist threads.
Returns an empty list if the thread has no checkpoints.
Reconstructs the full message history by walking the checkpoint chain
and applying pending writes — required under deepagents 0.6
``DeltaChannel`` where messages live in the ``writes`` table rather
than the latest checkpoint's ``channel_values``.
"""
db_path = str(get_db_path())
async with aiosqlite.connect(db_path, timeout=30.0) as conn:
if not await _table_exists(conn, "checkpoints"):
return []
# Verify this thread belongs to EvoScientist before loading messages
check = f"""
SELECT 1 FROM checkpoints
WHERE thread_id = ? AND {MAIN_THREAD_FILTER_SQL}
LIMIT 1
"""
async with conn.execute(check, (thread_id, *MAIN_THREAD_FILTER_PARAMS)) as cur:
if not await cur.fetchone():
return []
serde = _checkpoint_serde()
saver = AsyncSqliteSaver(conn, serde=serde)
return await _load_checkpoint_messages(saver, thread_id)
# ---------------------------------------------------------------------------
# Migration sweep & VACUUM (one-time legacy cleanup)
# ---------------------------------------------------------------------------
# PRAGMA user_version is a 32-bit int slot in the SQLite file header. We
# bump this to 1 once the legacy-bloat sweep has run successfully so it
# never runs again. Future structural migrations can use 2, 3, ...
_MIGRATION_VERSION = 1
# Threshold below which the sweep is skipped (DB is already small enough
# that legacy bloat is not the user's problem). 100 MB is chosen so a
# normally-pruned DB after a few months of use never triggers the sweep,
# while the 2.6 GB pathology is comfortably above the line.
_MIGRATION_THRESHOLD_BYTES = 100 * 1024 * 1024
# Inter-pair sleep so the sweep yields to the agent loop and never spikes
# CPU on a large DB. Tunable for tests via monkeypatch.
_SWEEP_YIELD_SECONDS = 0.0
async def _get_user_version(conn: aiosqlite.Connection) -> int:
async with conn.execute("PRAGMA user_version") as cur:
row = await cur.fetchone()
return int(row[0]) if row else 0
async def _set_user_version(conn: aiosqlite.Connection, version: int) -> None:
# PRAGMAs cannot be parameter-bound; the integer is interpolated safely
# because we control the value (constant int).
await conn.execute(f"PRAGMA user_version = {int(version)}")
await conn.commit()
def _format_duration(seconds: float) -> str:
"""Render a number of seconds as ``45s`` / ``2m 18s`` / ``1h 5m``.
Guards against ``inf`` / ``nan`` so a stray non-finite value (e.g. an
ETA computed before any pair completes) cannot raise ``OverflowError``
via ``int(float('inf'))``.
"""
if not math.isfinite(seconds):
return "?"
seconds = max(0, int(seconds))
if seconds < 60:
return f"{seconds}s"
if seconds < 3600:
m, s = divmod(seconds, 60)
return f"{m}m {s}s" if s else f"{m}m"
h, rem = divmod(seconds, 3600)
m = rem // 60
return f"{h}h {m}m" if m else f"{h}h"
def _estimate_sweep_seconds(size_bytes: int) -> int:
"""Rough ETA for a full migration sweep on the given DB size.
Empirical baseline: 14.53 GB ≈ 138s on a typical SSD, i.e. ~10 s/GB.
Used only for a user-facing pre-sweep hint — real time is reported
after completion so the estimate doesn't need to be precise.
"""
return max(2, round(size_bytes / (1024 * 1024 * 1024) * 9.5))
async def _migration_preview() -> tuple[str, int, int]:
"""Return ``(size_str, pair_count, size_bytes)`` for the pre-sweep status line.
``size_str`` is a human-readable file size (``"2.62 GB"`` / ``"364.3 MB"``).
``pair_count`` is the number of distinct ``(thread_id, checkpoint_ns)``
pairs the sweep will iterate. ``size_bytes`` is the raw byte count
(used by ``_estimate_sweep_seconds`` for the ETA). Best-effort:
returns zeros on any error.
"""
db_path = get_db_path()
try:
size_bytes = db_path.stat().st_size
except OSError:
size_bytes = 0
if size_bytes < 1024 * 1024:
size_str = f"{size_bytes / 1024:.1f} KB"
elif size_bytes < 1024 * 1024 * 1024:
size_str = f"{size_bytes / (1024 * 1024):.1f} MB"
else:
size_str = f"{size_bytes / (1024 * 1024 * 1024):.2f} GB"
pair_count = 0
try:
async with aiosqlite.connect(str(db_path), timeout=30.0) as conn:
if await _table_exists(conn, "checkpoints"):
async with conn.execute(
"SELECT COUNT(*) FROM ("
" SELECT DISTINCT thread_id, checkpoint_ns FROM checkpoints "
" WHERE json_extract(metadata, '$.agent_name') = ?"
")",
(AGENT_NAME,),
) as cur:
row = await cur.fetchone()
pair_count = int(row[0]) if row else 0
except aiosqlite.Error:
pass
return size_str, pair_count, size_bytes
async def _needs_migration() -> bool:
"""Return True if the legacy-bloat sweep should run now.
True iff the DB exists, is larger than ``_MIGRATION_THRESHOLD_BYTES``,
and ``PRAGMA user_version`` is below ``_MIGRATION_VERSION``.
"""
db_path = get_db_path()
if not db_path.exists():
return False
try:
size = db_path.stat().st_size
except OSError:
return False
if size < _MIGRATION_THRESHOLD_BYTES:
return False
try:
async with aiosqlite.connect(str(db_path), timeout=30.0) as conn:
if not await _table_exists(conn, "checkpoints"):
return False
return await _get_user_version(conn) < _MIGRATION_VERSION
except aiosqlite.Error:
return False
async def _run_migration_sweep(
keep: int,
progress_cb: Callable[[int, int], Awaitable[None]] | None = None,
) -> int:
"""Prune all ``(thread_id, checkpoint_ns)`` pairs to ``keep`` rows each.
Iterates pairs in deterministic order, applies the same DELETE pattern
the per-step pruner uses, and yields to the event loop between pairs
so the agent stays responsive. On success bumps ``PRAGMA user_version``
so the sweep never reruns.
``progress_cb``: optional ``async (done: int, total: int) -> None``
fired after each pair is committed; used by callers to drive a live
progress indicator.
Returns the number of pairs pruned.
"""
if keep <= 0:
return 0
db_path = str(get_db_path())
pairs_pruned = 0
async with aiosqlite.connect(db_path, timeout=60.0) as conn:
if not await _table_exists(conn, "checkpoints"):
return 0
if await _get_user_version(conn) >= _MIGRATION_VERSION:
return 0
async with conn.execute(
"SELECT DISTINCT thread_id, checkpoint_ns FROM checkpoints "
"WHERE json_extract(metadata, '$.agent_name') = ?",
(AGENT_NAME,),
) as cur:
pairs = list(await cur.fetchall())
# Reuse the DeltaChannel-aware prune logic from PruningCheckpointer
# instead of running naive keep_latest SQL: legacy DBs almost always
# have threads where the latest N checkpoints sit ABOVE a
# ``_DeltaSnapshot`` ancestor, and the naive form would sever the
# snapshot chain — exactly the failure mode the steady-state Fix
# already prevents. Sharing one saver across all pairs means
# ``setup()`` and the in-class lock are constructed once.
#
# We invoke ``_prune_after_put`` directly (not ``aput``) — the
# sweep is a bulk cleanup, not a checkpoint write. As a result
# ``saver._aput_lock`` (the outer put+prune pair lock) is
# intentionally unused here; only the inner ``self.lock`` that
# ``_prune_after_put`` itself acquires runs.
saver = PruningCheckpointer(conn, keep_per_ns=keep)
for thread_id, checkpoint_ns in pairs:
ns = checkpoint_ns or ""
await saver._prune_after_put(str(thread_id), ns)
pairs_pruned += 1
if progress_cb is not None:
try:
await progress_cb(pairs_pruned, len(pairs))
except Exception: # pragma: no cover - never let UI break sweep
pass
if _SWEEP_YIELD_SECONDS >= 0:
await asyncio.sleep(_SWEEP_YIELD_SECONDS)
await _set_user_version(conn, _MIGRATION_VERSION)
# Schedule VACUUM at process exit (must run after the long-lived saver
# connection closes to acquire the exclusive lock VACUUM requires).
# Pass ``db_path`` explicitly so test-time monkeypatches of
# ``get_db_path`` don't leak into atexit and hit the real DB.
_schedule_vacuum_atexit(db_path)
return pairs_pruned
_vacuum_scheduled = False
def _schedule_vacuum_atexit(db_path: str) -> None:
"""Register the atexit VACUUM hook exactly once per process.
Captures ``db_path`` at registration time so the hook always operates
on the path that was current when the sweep ran. This matters in
tests, where ``get_db_path`` is monkey-patched to a temp file but the
patch has unwound by the time atexit fires — re-resolving at exit
would point at the user's real ``sessions.db`` and trigger an
unwanted VACUUM on production data.
"""
global _vacuum_scheduled
if _vacuum_scheduled:
return
_vacuum_scheduled = True
atexit.register(_atexit_vacuum, db_path)
def _atexit_vacuum(db_path: str) -> None:
"""Run ``VACUUM`` synchronously at process exit on the captured path.
Uses stdlib ``sqlite3`` (atexit can't await aiosqlite). Best-effort:
swallow any error since this runs during shutdown when stderr may be
closed.
"""
import os
import sqlite3
if not os.path.exists(db_path):
return
try:
with sqlite3.connect(db_path, timeout=60.0) as conn:
# VACUUM cannot run inside a transaction; sqlite3 starts one
# implicitly on the first execute, so isolation_level=None ensures
# we are in autocommit mode for the VACUUM statement.
conn.isolation_level = None
conn.execute("VACUUM")
except sqlite3.Error as exc:
# Best-effort during shutdown: stderr may already be closed, but
# try to log so a persistent VACUUM failure is at least diagnosable
# from the next session. Swallow any logging error in turn.
try:
_logger.warning("VACUUM at exit failed: %s", exc)
except Exception:
pass
# ---------------------------------------------------------------------------
# DB stats (read-only diagnostic for `EvoSci sessions stats`)
# ---------------------------------------------------------------------------
async def db_stats(top_n: int = 5) -> dict[str, Any]:
"""Return read-only diagnostics about the sessions DB.
Keys: ``db_path`` (str), ``size_bytes`` (int, file size or 0 if absent),
``thread_count`` (int, EvoScientist threads), ``checkpoint_count``
(int, EvoScientist rows), ``write_count`` (int, EvoScientist writes
only — scoped via JOIN to ``checkpoints.metadata.agent_name`` so
co-located non-EvoSci agents are excluded), ``top_threads`` (list of
dicts with ``thread_id`` and ``count``, sorted desc).
"""
db_path = get_db_path()
size = db_path.stat().st_size if db_path.exists() else 0
out: dict[str, Any] = {
"db_path": str(db_path),
"size_bytes": size,
"thread_count": 0,
"checkpoint_count": 0,
"write_count": 0,
"top_threads": [],
}
if not db_path.exists():
return out
try:
async with aiosqlite.connect(str(db_path), timeout=30.0) as conn:
if not await _table_exists(conn, "checkpoints"):
return out
async with conn.execute(
"SELECT COUNT(DISTINCT thread_id), COUNT(*) FROM checkpoints "
"WHERE json_extract(metadata, '$.agent_name') = ?",
(AGENT_NAME,),
) as cur:
row = await cur.fetchone()
if row:
out["thread_count"] = int(row[0] or 0)
out["checkpoint_count"] = int(row[1] or 0)
if await _table_exists(conn, "writes"):
# Scope writes to EvoScientist rows by joining against
# checkpoints — the ``writes`` table itself has no
# ``agent_name`` column, so a bare ``COUNT(*)`` would
# over-report when other LangGraph apps share this DB.
async with conn.execute(
"SELECT COUNT(*) FROM writes w "
"JOIN checkpoints c "
" ON c.thread_id = w.thread_id "
" AND c.checkpoint_ns = w.checkpoint_ns "
" AND c.checkpoint_id = w.checkpoint_id "
"WHERE json_extract(c.metadata, '$.agent_name') = ?",
(AGENT_NAME,),
) as cur:
row = await cur.fetchone()
if row:
out["write_count"] = int(row[0] or 0)
async with conn.execute(
"SELECT thread_id, COUNT(*) AS n FROM checkpoints "
"WHERE json_extract(metadata, '$.agent_name') = ? "
"GROUP BY thread_id ORDER BY n DESC LIMIT ?",
(AGENT_NAME, int(top_n)),
) as cur:
rows = await cur.fetchall()
out["top_threads"] = [
{"thread_id": r[0], "count": int(r[1])} for r in rows
]
except aiosqlite.Error:
# Read-only — corrupt/locked DB → return zeroed stats rather than crash.
return out
return out
# ---------------------------------------------------------------------------
# langgraph-api / WebUI checkpointer factory
# ---------------------------------------------------------------------------
def _api_workspace_dir() -> str:
"""Resolve the langgraph dev subprocess's workspace directory.
``start_langgraph_dev`` injects ``EVOSCIENTIST_WORKSPACE_DIR`` and sets
the subprocess cwd to the workspace, so either source identifies the
workspace this server instance is serving.
NOTE: ``Path.resolve()`` / ``Path.cwd()`` call ``os.getcwd()``, a blocking
syscall. Call this from a sync context or via ``asyncio.to_thread`` — never
directly from an async function on the dev runtime (blockbuster will flag it).
"""
import os
ws = os.environ.get("EVOSCIENTIST_WORKSPACE_DIR", "").strip()
if ws:
return str(Path(ws).expanduser().resolve())
return str(Path.cwd().resolve())
async def _api_workspace_dir_async() -> str:
"""Async wrapper for :func:`_api_workspace_dir`.
Offloads the blocking ``os.getcwd()`` (via ``Path.cwd()/resolve()``) to a
thread so it never runs on the event loop — the dev runtime's blockbuster
guard flags it otherwise. Use this from any async call site.
"""
return await asyncio.to_thread(_api_workspace_dir)
class _ApiPruningCheckpointer(PruningCheckpointer):
"""``PruningCheckpointer`` that stamps CLI-compatible ownership metadata.
langgraph-api run metadata carries ``graph_id``/``assistant_id`` but not
always the ``workspace_dir`` / ``updated_at`` keys needed to safely
rebuild the in-memory thread registry after server restart. Stamping graph
rows with the current workspace keeps main and async-subagent threads
restorable without exposing other workspaces. Memory-worker rows still get
workspace metadata, but remain disposable until worker cloning lands.
Only the main graph receives ``agent_name``. The local CLI session
surface still uses that ownership key, so worker/subagent graph rows must
remain outside ordinary ``/threads``, ``/resume``, and ``/delete``.
"""
async def aput(
self,
config: Any,
checkpoint: Any,
metadata: Any,
new_versions: Any,
) -> Any:
if isinstance(metadata, dict) and isinstance(metadata.get("graph_id"), str):
metadata = dict(metadata)
# _api_workspace_dir() calls Path.resolve()/Path.cwd() -> os.getcwd(),
# a blocking syscall flagged by the dev runtime's blockbuster guard.
# Run it in a thread, and only when actually needed — ``setdefault``
# would evaluate the argument eagerly on every write even when the
# key is already present.
if "workspace_dir" not in metadata:
metadata["workspace_dir"] = await _api_workspace_dir_async()
metadata["updated_at"] = datetime.now(UTC).isoformat()
if metadata.get("graph_id") == AGENT_NAME:
metadata.setdefault("agent_name", AGENT_NAME)
return await super().aput(config, checkpoint, metadata, new_versions)
async def _purge_internal_worker_threads() -> None:
"""Best-effort removal of evomemory-worker checkpoint residue.
Finished workers delete their own thread (see
``middleware/memory_lifecycle.py``), but a crash between run completion
and deletion leaves rows behind — and rows written before that cleanup
existed are still in the DB. Idempotent, runs on every server start,
and never blocks startup on failure.
"""
try:
db_path = str(get_db_path())
async with aiosqlite.connect(db_path, timeout=30.0) as conn:
if not await _table_exists(conn, "checkpoints"):
return
if await _table_exists(conn, "writes"):
await conn.execute(
"""
DELETE FROM writes WHERE thread_id IN (
SELECT DISTINCT thread_id FROM checkpoints
WHERE json_extract(metadata, '$.graph_id') LIKE 'evomemory-%'
)
"""
)
cur = await conn.execute(
"""
DELETE FROM checkpoints
WHERE json_extract(metadata, '$.graph_id') LIKE 'evomemory-%'
"""
)
await conn.commit()
if cur.rowcount:
_logger.info(
"Purged %d leftover evomemory-worker checkpoint row(s).",
cur.rowcount,
)
except Exception:
_logger.warning(
"evomemory-worker residue purge failed (non-fatal).", exc_info=True
)
@dataclass(frozen=True, slots=True)
class _RestoredThreadInfo:
updated_at: str | None
assistant_id: str | None
graph_id: str
workspace_dir: str
model: str | None
async def _restore_webui_threads_to_global_store() -> None:
"""Re-populate ``GlobalStore["threads"]`` from SQLite on server startup.
The inmem runtime's thread registry lives in memory (pickled to
``.langgraph_ops.pckl``) and is cleared on every start — if the pickle
is absent or corrupt, the WebUI sidebar is empty even though all
checkpoint data sits safely in SQLite. This rebuilds it: ghost entries
whose threads have no checkpoint rows are dropped, surviving entries
are normalized in place, and missing threads are appended as stub
dicts that satisfy ``POST /threads/search``.
Restore scope — UUID-format graph threads owned by this server's
workspace (``metadata.workspace_dir`` matches). This includes the main
graph and async-subagent graphs. Memory-worker graphs are excluded for now:
they are still treated as disposable residue until worker cloning lands.
The workspace filter is required because sessions.db is machine-global,
and an unscoped restore would expose every workspace's history on the
unauthenticated API — worst case ``--tunnel``. CLI/TUI threads (8-char
hex IDs, managed by ``list_threads()``) and pre-stamping rows without
``workspace_dir`` are excluded.
Best-effort: any exception is logged and swallowed so a broken restore
never prevents the ``langgraph dev`` server from starting.
"""
try:
from langgraph_runtime_inmem.database import (
GLOBAL_STORE,
)
except ImportError:
# langgraph_runtime_inmem not available (unit tests, plain CLI mode).
return
def _to_uuid_safe(v: Any) -> uuid.UUID | None:
try:
return uuid.UUID(str(v))
except (ValueError, AttributeError):
return None
try:
rows: list[Any] = []
# Restore scope: graph threads belonging to THIS server's workspace.
# sessions.db is machine-global, so an unscoped restore would
# resurrect every workspace's history into this server's thread
# registry — and expose it over the unauthenticated API / --tunnel.
# Legacy WebUI/CLI interop rows without graph_id are restored as the
# main graph only when they carry agent_name == AGENT_NAME. Rows
# predating workspace stamping remain deliberately excluded.
current_workspace = await _api_workspace_dir_async()
sqlite_data: dict[uuid.UUID, _RestoredThreadInfo] = {}
titles: dict[uuid.UUID, str] = {}
db_path = str(get_db_path())
async with aiosqlite.connect(db_path, timeout=30.0) as conn:
# No checkpoints table (fresh DB) → rows stays empty, but the
# ghost-removal pass below must still run: every UUID entry the
# .pckl registry loaded is then stale by definition.
if await _table_exists(conn, "checkpoints"):
# UUID regex: 8-4-4-4-12 hex groups. assistant_id/graph_id
# are needed by the WebUI's POST /threads/search filters.
# Every metadata field is MAX-aggregated: an interop thread
# mixes CLI rows (no assistant_id/graph_id) with WebUI rows,
# and a bare column under GROUP BY would let SQLite pick an
# arbitrary row's NULL.
query = """
SELECT thread_id,
MAX(json_extract(metadata, '$.updated_at')) as updated_at,
MAX(json_extract(metadata, '$.assistant_id')) as assistant_id,
MAX(json_extract(metadata, '$.graph_id')) as graph_id,
MAX(json_extract(metadata, '$.workspace_dir')) as workspace_dir,
MAX(json_extract(metadata, '$.model')) as model,
MAX(json_extract(metadata, '$.agent_name')) as agent_name
FROM checkpoints
WHERE thread_id LIKE '________-____-____-____-____________'
AND (
json_extract(metadata, '$.graph_id') IS NULL
OR json_extract(metadata, '$.graph_id') NOT LIKE 'evomemory-%'
)
GROUP BY thread_id
ORDER BY updated_at DESC
"""
async with conn.execute(query) as cur:
rows = list(await cur.fetchall())
for row in rows:
(
thread_id_str,
updated_at,
assistant_id,
graph_id,
workspace_dir,
model,
agent_name,
) = row
thread_uuid = _to_uuid_safe(thread_id_str)
if thread_uuid is None:
continue
restored_graph_id = graph_id
if restored_graph_id is None and agent_name == AGENT_NAME:
restored_graph_id = AGENT_NAME
if restored_graph_id is None:
continue
if not workspace_dir or workspace_dir != current_workspace:
continue
sqlite_data[thread_uuid] = _RestoredThreadInfo(
updated_at=updated_at,
assistant_id=assistant_id,
graph_id=restored_graph_id,
workspace_dir=workspace_dir,
model=model,
)
# Derive a sidebar title from each scoped thread's first human
# message (stubs carry values=None, so the WebUI would otherwise
# render every restored thread as "Untitled Thread").
if sqlite_data:
saver = AsyncSqliteSaver(conn, serde=_checkpoint_serde())
for thread_uuid in sqlite_data:
try:
msgs = await _load_checkpoint_messages(saver, str(thread_uuid))
preview = _extract_preview(msgs)
if preview:
titles[thread_uuid] = preview
except Exception:
continue
def _parse_dt(s: str | None) -> datetime:
"""Parse an ISO timestamp string to datetime, falling back to now."""
if s:
try:
return datetime.fromisoformat(s)
except (ValueError, TypeError):
pass
return datetime.now(UTC)
# Drop stale registry entries: UUID entries outside the scoped restore
# set either point at missing state or another workspace's state.
# Slice assignment mutates the live registry list.
store_threads: list[dict[str, Any]] = GLOBAL_STORE.get("threads", [])
before = len(store_threads)
store_threads[:] = [
entry
for entry in store_threads
if (tid := _to_uuid_safe(entry.get("thread_id"))) is None
or tid in sqlite_data
]
removed = before - len(store_threads)
# Normalize surviving .pckl-loaded entries in place.
fixed = 0
existing_uuids: set[uuid.UUID] = set()
for entry in store_threads:
tid_uuid = _to_uuid_safe(entry.get("thread_id"))
if tid_uuid is None:
continue
existing_uuids.add(tid_uuid)
changed = False
# Threads.get() compares against _ensure_uuid() — str never matches.
if not isinstance(entry.get("thread_id"), uuid.UUID):
entry["thread_id"] = tid_uuid
changed = True
if tid_uuid in sqlite_data:
info = sqlite_data[tid_uuid]
meta: dict[str, Any] = entry.setdefault("metadata", {})
if info.assistant_id and "assistant_id" not in meta:
# str, not uuid.UUID: the runtime stores str and search
# filters compare with raw == against JSON strings.
meta["assistant_id"] = str(info.assistant_id)
changed = True
if info.graph_id and "graph_id" not in meta:
meta["graph_id"] = info.graph_id
changed = True
if meta.get("workspace_dir") != info.workspace_dir:
meta["workspace_dir"] = info.workspace_dir
changed = True
if info.model and meta.get("model") != info.model:
meta["model"] = info.model
changed = True
if "title" not in meta and tid_uuid in titles:
meta["title"] = titles[tid_uuid]
changed = True
# State.get() KeyErrors without "config".
if "config" not in entry:
entry["config"] = {}
changed = True
# Threads.search() sorted() raises on datetime-vs-str mixes.
for ts_key in ("created_at", "updated_at", "state_updated_at"):
if isinstance(entry.get(ts_key), str):
entry[ts_key] = _parse_dt(entry[ts_key])
changed = True
if changed:
fixed += 1
# Append threads present in SQLite but absent from the registry.
restored = 0
for thread_uuid, info in sqlite_data.items():
if thread_uuid in existing_uuids:
continue
stub_metadata: dict[str, Any] = {
"graph_id": info.graph_id,
"workspace_dir": info.workspace_dir,
}
if info.assistant_id:
# str, not uuid.UUID — same convention as above.
stub_metadata["assistant_id"] = str(info.assistant_id)
if info.model:
stub_metadata["model"] = info.model
if thread_uuid in titles:
stub_metadata["title"] = titles[thread_uuid]
ts = _parse_dt(info.updated_at)
stub: dict[str, Any] = {
"thread_id": thread_uuid,
"created_at": ts,
"updated_at": ts,
"state_updated_at": ts,
"metadata": stub_metadata,
"status": "idle",
"config": {},
"values": None,
}
GLOBAL_STORE["threads"].append(stub)
existing_uuids.add(thread_uuid)
restored += 1
if fixed or restored or removed:
_logger.info(
"WebUI thread restore: fixed %d existing + appended %d new + "
"removed %d ghost thread(s) in GlobalStore "
"(langgraph_runtime_inmem).",
fixed,
restored,
removed,
)
except Exception:
_logger.warning(
"WebUI thread restore failed (non-fatal); WebUI session list may be "
"empty until new threads are created.",
exc_info=True,
)
@asynccontextmanager
async def create_checkpointer_for_langgraph_api() -> AsyncIterator[PruningCheckpointer]:
"""SQLite-backed checkpointer for the ``langgraph dev`` subprocess.
``checkpointer.path`` target in ``langgraph_dev/langgraph.json``
(applies to every ``langgraph dev`` launch: deploy, WebUI, and the
CLI-spawned subprocess). Replaces the default pickle-based
``InMemorySaver``, whose 10s flush window drops recent checkpoints on
SIGKILL and whose pickle-incompatible upgrades wipe the whole store
(issue #277); here every ``aput()`` commits a WAL transaction and a
bad row only loses that row. The langgraph-api adapter detects async
context managers and enters them automatically.
The yielded ``_ApiPruningCheckpointer`` stamps graph rows with
``workspace_dir`` / ``updated_at`` so they can be restored into the
LangGraph server registry. Main-graph rows also get ``agent_name`` so
WebUI threads surface in the CLI session commands and participate in
``_prune_after_put`` retention.
Capability note: ``adelete_thread`` is real, but ``aprune`` /
``adelete_for_runs`` / ``acopy_thread`` remain ``BaseCheckpointSaver``
raising stubs — langgraph-api's probe reports them missing and
degrades (``multitask_strategy='rollback'`` cleanup raises; thread
copy uses the slow generic fallback).
"""
keep = _resolve_keep_per_ns()
async with _ApiPruningCheckpointer.from_conn_string_with_keep(
str(get_db_path()), keep_per_ns=keep
) as saver:
await saver.setup()
await _purge_internal_worker_threads()
await _restore_webui_threads_to_global_store()
yield saver