Merge branch 'simp/r3-01-H' into simp/r3-01

This commit is contained in:
Teknium
2026-09-02 22:42:20 -07:00
9 changed files with 483 additions and 861 deletions
+25 -54
View File
@@ -102,9 +102,6 @@ class SessionPortabilityMixin:
with self._lock:
return self._conn.execute(sql, params).fetchall()
def _rich_rows(self, sql: str, params=()) -> List[Dict[str, Any]]:
return [self._rich_row(row) for row in self._locked_rows(sql, params)]
def distinct_session_cwds(self, include_archived: bool = False) -> List[Dict[str, Any]]:
"""Distinct non-empty session cwds with usage stats, for repo discovery. Aggregates
across ALL history; children/branches count (a worktree session is a real
@@ -116,10 +113,8 @@ class SessionPortabilityMixin:
"SELECT cwd AS cwd, COUNT(*) AS sessions, MAX(COALESCE(ended_at, started_at, 0)) AS last_active "
f"FROM sessions WHERE {where} GROUP BY cwd"
)
return [
{"cwd": r["cwd"], "sessions": int(r["sessions"] or 0), "last_active": float(r["last_active"] or 0)}
for r in rows
]
return [{"cwd": r["cwd"], "sessions": int(r["sessions"] or 0), "last_active": float(r["last_active"] or 0)}
for r in rows]
def list_cron_job_runs(self, job_id: str, limit: int = 20, offset: int = 0) -> List[Dict[str, Any]]:
"""Run sessions of one cron job, newest first, in the ``list_sessions_rich`` row shape.
@@ -135,7 +130,7 @@ class SessionPortabilityMixin:
"\n ORDER BY s.started_at DESC, s.id DESC\n LIMIT ? OFFSET ?",
prompt_select=f",\n {_PROMPT_RESOLVED_SQL}",
)
return self._rich_rows(query, (prefix, prefix_hi, limit, offset))
return [self._rich_row(row) for row in self._locked_rows(query, (prefix, prefix_hi, limit, offset))]
def _get_session_rich_row(self, session_id: str, compact_rows: bool = False) -> Optional[Dict[str, Any]]:
"""One session with the ``list_sessions_rich`` enriched columns, or None.
@@ -165,14 +160,13 @@ class SessionPortabilityMixin:
self._compact_session_cols() if compact_rows else "s.*", f"s.id IN ({','.join('?' for _ in ids)})",
prompt_select=None if compact_rows else f", {_PROMPT_RESOLVED_SQL}",
)
return {s["id"]: s for s in self._rich_rows(query, ids)}
return {s["id"]: s for s in map(self._rich_row, self._locked_rows(query, ids))}
def list_skill_scaffolded_sessions(self, limit: int = 200) -> List[Dict[str, Any]]:
"""Titled sessions whose first user turn was a ``/skill`` invocation (their titles
describe the expanded skill body, not the request). Returns ``id``, ``title`` and
the first-turn ``content`` so callers can re-derive what was typed. Newest first."""
rows = self._locked_rows(
"""
rows = self._locked_rows("""
SELECT s.id, s.title, m.content
FROM sessions s
JOIN messages m ON m.id = (
@@ -184,9 +178,7 @@ class SessionPortabilityMixin:
WHERE s.title IS NOT NULL AND m.content LIKE ?
ORDER BY s.started_at DESC
LIMIT ?
""",
(SKILL_SCAFFOLD_SQL_LIKE, int(limit)),
)
""", (SKILL_SCAFFOLD_SQL_LIKE, int(limit)))
return [dict(row) for row in rows]
# ── Export ─────────────────────────────────────────────────────────────
@@ -230,10 +222,8 @@ class SessionPortabilityMixin:
``adopted`` and ``donor_retired`` (True only when EVERY segment retired)."""
payload = donor_db.export_session_lineage(session_id)
if not payload:
return {
"ok": False, "adopted": False, "donor_retired": False,
"error": f"session {session_id!r} not found in donor store",
}
return {"ok": False, "adopted": False, "donor_retired": False,
"error": f"session {session_id!r} not found in donor store"}
segments = payload.get("segments") or [payload]
# Divergence guard: a segment we will SKIP (already here) may have kept growing in
@@ -248,21 +238,16 @@ class SessionPortabilityMixin:
local_count = len(self.get_messages(seg_id))
if donor_count > local_count:
donor_ahead = True
logger.warning(
"adoption divergence: donor segment %s has %d messages, "
"local copy has %d — donor will NOT be retired",
seg_id, donor_count, local_count,
)
logger.warning("adoption divergence: donor segment %s has %d messages, "
"local copy has %d — donor will NOT be retired", seg_id, donor_count, local_count)
result = self.import_sessions([dict(seg) for seg in segments])
imported = int(result.get("imported") or 0)
skipped = int(result.get("skipped") or 0)
adopted = result.get("ok", False) and (imported + skipped) == len(segments)
if not adopted:
logger.warning(
"adoption of %s did not complete: imported=%s skipped=%s of %s segment(s); errors=%s",
session_id, imported, skipped, len(segments), result.get("errors"),
)
logger.warning("adoption of %s did not complete: imported=%s skipped=%s of %s segment(s); errors=%s",
session_id, imported, skipped, len(segments), result.get("errors"))
donor_retired = False
if adopted and retire_donor and not donor_ahead:
@@ -282,8 +267,7 @@ class SessionPortabilityMixin:
if donor_now > local_now:
logger.warning(
"adoption divergence at retire time: donor segment %s grew to %d messages (local %d) — "
"leaving donor unretired",
seg_id, donor_now, local_now,
"leaving donor unretired", seg_id, donor_now, local_now,
)
return False
# First end_reason wins in end_session(); reopen so the adoption boundary is
@@ -338,26 +322,12 @@ class SessionPortabilityMixin:
except (TypeError, ValueError):
return default
@staticmethod
def _reasoning_json_value(value: Any) -> Any:
return safe_json_loads(value, default=value) if isinstance(value, str) else value
@staticmethod
def _import_error(index: int, session_id: str, error: str) -> Dict[str, Any]:
item: Dict[str, Any] = {"index": index, "error": error}
if session_id:
item["session_id"] = session_id
return item
def _normalize_import_session(self, raw: Dict[str, Any], session_id: str, messages: list) -> Dict[str, Any]:
"""Type-check one payload session + its messages; raises ValueError."""
clean_session = dict(raw)
clean_session["id"] = session_id
clean_session["model_config"] = self._import_json_object_or_none(clean_session.get("model_config"), "model_config")
clean_session["parent_session_id"] = self._import_text_or_none(
clean_session.get("parent_session_id"), "parent_session_id"
)
for field in _IMPORT_SESSION_TEXT_FIELDS:
for field in ("parent_session_id", *_IMPORT_SESSION_TEXT_FIELDS):
clean_session[field] = self._import_text_or_none(clean_session.get(field), field)
clean_messages: List[Dict[str, Any]] = []
for message_index, message in enumerate(messages):
@@ -383,7 +353,10 @@ class SessionPortabilityMixin:
try:
item = self._validate_import_session(raw, session_id, seen_ids, totals)
except ValueError as exc:
errors.append(self._import_error(index, session_id, str(exc)))
item = {"index": index, "error": str(exc)}
if session_id:
item["session_id"] = session_id
errors.append(item)
continue
seen_ids.add(session_id)
normalized.append({"index": index, **item})
@@ -433,15 +406,14 @@ class SessionPortabilityMixin:
**{col: self._coerce_or(raw.get(col), int, 0) for col in _IMPORT_INT_COLS},
}
conn.execute(_IMPORT_SESSION_INSERT_SQL, params)
def _json_value(value: Any) -> Any:
return safe_json_loads(value, default=value) if isinstance(value, str) else value
sanitized_messages = [
{**msg, **{key: self._reasoning_json_value(msg.get(key)) for key in _IMPORT_MESSAGE_JSON_FIELDS}}
for msg in messages
{**msg, **{key: _json_value(msg.get(key)) for key in _IMPORT_MESSAGE_JSON_FIELDS}} for msg in messages
]
total_messages, total_tool_calls = self._insert_message_rows(conn, session_id, sanitized_messages)
conn.execute(
"UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?",
(total_messages, total_tool_calls, session_id),
)
conn.execute("UPDATE sessions SET message_count = ?, tool_call_count = ? WHERE id = ?",
(total_messages, total_tool_calls, session_id))
@staticmethod
def _attach_import_parents(conn, parent_updates: List[tuple]) -> int:
@@ -509,11 +481,10 @@ class SessionPortabilityMixin:
if parent_id:
parent_updates.append((session_id, parent_id))
imported_ids.append(session_id)
detached = self._attach_import_parents(conn, parent_updates)
return {
"ok": True, "imported": len(imported_ids), "skipped": len(skipped_ids),
"detached": detached, "imported_ids": imported_ids, "skipped_ids": skipped_ids,
"errors": [],
"detached": self._attach_import_parents(conn, parent_updates),
"imported_ids": imported_ids, "skipped_ids": skipped_ids, "errors": [],
}
return self._execute_write(_do)
+16 -30
View File
@@ -20,6 +20,7 @@ Lifecycle rules:
from __future__ import annotations
import contextlib
import logging
import threading
from pathlib import Path
@@ -57,7 +58,7 @@ _opening: Dict[Path, threading.Event] = {}
def _open_session_db(path: Path) -> "SessionDB":
"""Construct the SessionDB for *path* (call-time import avoids cycles)."""
"""Construct the SessionDB for *path* (call-time import avoids cycles; tests patch this)."""
from hermes_state import SessionDB
return SessionDB(db_path=path)
@@ -65,10 +66,8 @@ def _open_session_db(path: Path) -> "SessionDB":
def _teardown(db: "SessionDB") -> None:
"""Close a shared instance, clearing its registry-owned flag first."""
try:
with contextlib.suppress(Exception):
db._shared_registry_owned = False
except Exception:
pass
try:
db.close()
except Exception:
@@ -78,10 +77,8 @@ def _teardown(db: "SessionDB") -> None:
def _db_path_of(db: "SessionDB") -> Optional[Path]:
"""``Path(db.db_path)`` or None when absent/unconvertible."""
path = getattr(db, "db_path", None)
if path is None:
return None
try:
return Path(path)
return None if path is None else Path(path)
except (TypeError, ValueError):
return None
@@ -113,8 +110,12 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB":
if generation is not None:
current = _stat_db_file_identity(path)
if current is not None and generation.identity is not None and current != generation.identity:
# File replaced: retire, then elect one caller to open the replacement.
_retire_generation_locked(path, generation)
# File replaced: retire this generation so it is never lent again, then elect one
# caller to open the replacement. It stays alive for its holders, tracked in
# ``_retired`` by ``id(db)`` so their releases find it after the path remaps.
generation.retired = True
del _generations[path]
_retired[id(generation.db)] = generation
else:
generation.refcount += 1
return generation.db
@@ -139,8 +140,7 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB":
with _lock:
existing = _generations.get(path)
if existing is not None:
# Defensive: installed by explicit registry manipulation mid-open.
if existing is not None: # Defensive: installed by explicit registry manipulation mid-open.
existing.refcount += 1
winner = existing.db
else:
@@ -152,16 +152,6 @@ def acquire(db_path: Optional[Path] = None) -> "SessionDB":
return winner
def _retire_generation_locked(path: Path, generation: _Generation) -> None:
"""Retire *generation* so it is never lent again (caller holds _lock). It stays alive
for its holders, tracked in ``_retired`` by ``id(db)`` so their releases find it even
after the path maps to a new generation."""
generation.retired = True
if _generations.get(path) is generation:
del _generations[path]
_retired[id(generation.db)] = generation
def release(db: "SessionDB") -> bool:
"""Decrement the refcount of a shared SessionDB. ``True`` if *db* was shared; ``False``
if it is not registry-managed (caller owns close()). The final release tears the
@@ -182,13 +172,10 @@ def release(db: "SessionDB") -> bool:
return False
generation.refcount -= 1
needs_teardown = generation.refcount <= 0
if needs_teardown:
if generation.retired:
_retired.pop(key, None)
else:
path = _db_path_of(db)
if path is not None:
_generations.pop(path, None)
if needs_teardown and generation.retired:
_retired.pop(key, None)
elif needs_teardown and (path := _db_path_of(db)) is not None:
_generations.pop(path, None)
# Teardown OUTSIDE the lock: stopping the token writer, WAL checkpoint and read-pool
# drain must not block acquisition for every other state.db.
if needs_teardown:
@@ -222,8 +209,7 @@ def stats() -> Dict[str, int]:
"""Registry census for tests and diagnostics (no locks held long)."""
with _lock:
return {
"live_generations": len(_generations),
"retired_generations": len(_retired),
"live_generations": len(_generations), "retired_generations": len(_retired),
"total_refcounts": sum(g.refcount for g in _generations.values()),
}
+159 -236
View File
@@ -15,8 +15,8 @@ import logging
import os
import shutil
import sqlite3
import stat
import time
from contextlib import contextmanager
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
@@ -59,17 +59,6 @@ def _sidecars(db_path: Path):
return (db_path.with_name(db_path.name + suffix) for suffix in _DB_SIDECAR_SUFFIXES)
def _bundle_bytes(db_path: Path) -> int:
"""Size of the main file plus every PRESENT sidecar (OSError propagates)."""
return db_path.stat().st_size + sum(p.stat().st_size for p in _sidecars(db_path) if p.exists())
def _unlink_quiet(path: Path) -> None:
"""Best-effort unlink; a failure here is never the caller's error."""
with contextlib.suppress(OSError):
path.unlink(missing_ok=True)
def _read_offline(db_path: Path, what: str, reader) -> Optional[str]:
"""Run *reader()* under ``hermes_cli.sqlite_safe_read.offline_file_access``.
@@ -112,7 +101,6 @@ def _open_lock_file(db_path: Path, suffix: str, what: str, tail: str):
def _msvcrt_lock(handle, flag_name: str) -> None:
import msvcrt
handle.seek(0)
msvcrt.locking(handle.fileno(), getattr(msvcrt, flag_name), 1) # type: ignore[attr-defined]
@@ -124,26 +112,20 @@ def _try_lock_nonblocking(handle) -> None:
_msvcrt_lock(handle, "LK_NBLCK")
else:
import fcntl
fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
def _release_lock_handle(handle, *, clear_record: bool = False) -> None:
"""Drop the advisory lock on *handle* (best effort) and close it."""
from hermes_state import _IS_WINDOWS
try:
with contextlib.closing(handle), contextlib.suppress(OSError): # best-effort release; always close
if _IS_WINDOWS:
_msvcrt_lock(handle, "LK_UNLCK")
else:
import fcntl
if clear_record:
_clear_lock_holder_record(handle)
fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
except OSError: # pragma: no cover - best effort release
pass
finally:
handle.close()
def _acquire_repair_lock_windows(lock_path: Path, handle, timeout: float):
@@ -176,27 +158,24 @@ def _cross_process_repair_lock(db_path: Path):
before the disk filled may be inside surgery."""
from hermes_state import _IS_WINDOWS, _REPAIR_LOCK_TIMEOUT_SECONDS
lock_path, handle = _open_lock_file(
db_path, ".repair.lock", "repair", "skipping schema surgery rather than running it without cross-process authority.",
)
db_path, ".repair.lock", "repair", "skipping schema surgery rather than running it without cross-process authority.")
if handle is None:
yield False
return
acquired = False
try:
if _IS_WINDOWS:
acquired = _acquire_repair_lock_windows(lock_path, handle, _REPAIR_LOCK_TIMEOUT_SECONDS)
else:
acquired, handle = _acquire_db_flock(
str(lock_path), handle, _REPAIR_LOCK_TIMEOUT_SECONDS, _REPAIR_LOCK_POLL_SECONDS, "state.db repair lock",
)
acquired, handle = _acquire_db_flock(str(lock_path), handle, _REPAIR_LOCK_TIMEOUT_SECONDS,
_REPAIR_LOCK_POLL_SECONDS, "state.db repair lock")
if acquired is None:
acquired = False # non-contention failure already logged with its errno
elif not acquired:
record = None if _IS_WINDOWS else _read_lock_holder_record(handle)
logger.warning("state.db repair lock %s held by another process for more than %.0fs — skipping schema "
"surgery in this process to avoid racing the repairer. Recorded holder: %s.",
lock_path, _REPAIR_LOCK_TIMEOUT_SECONDS, _describe_lock_holder(record))
lock_path, _REPAIR_LOCK_TIMEOUT_SECONDS,
_describe_lock_holder(None if _IS_WINDOWS else _read_lock_holder_record(handle)))
yield acquired
finally:
if acquired:
@@ -250,26 +229,26 @@ def _repair_backup_headroom_bytes(total_bytes: int) -> int:
return max(_REPAIR_BACKUP_MIN_FREE_BYTES, int(total_bytes * _REPAIR_BACKUP_FREE_FRACTION))
def _disk_budget(db_path: Path, refusal: str):
"""``(bundle_bytes, free_bytes, headroom_bytes)`` for *db_path*'s volume, or an
error string. Fails CLOSED on stat()/disk_usage() errors: the nearly-full
def _disk_budget(db_path: Path, refusal: str) -> "Tuple[Optional[str], int, int, int]":
"""``(error, bundle_bytes, free_bytes, headroom_bytes)`` for *db_path*'s volume (main file plus every PRESENT
sidecar); *error* is set (and the sizes zero) on stat()/disk_usage() failure. Fails CLOSED: the nearly-full
volume these guards exist for is exactly where they are most likely to fail."""
try:
need = _bundle_bytes(db_path)
need = db_path.stat().st_size + sum(p.stat().st_size for p in _sidecars(db_path) if p.exists())
usage = shutil.disk_usage(db_path.parent)
except OSError as exc:
return f"could not determine free space on {db_path.parent} ({exc}); refusing the {refusal} rather than risk filling the volume"
return need, usage.free, _repair_backup_headroom_bytes(usage.total)
return (f"could not determine free space on {db_path.parent} ({exc}); refusing the {refusal} rather than risk "
"filling the volume"), 0, 0, 0
return None, need, usage.free, _repair_backup_headroom_bytes(usage.total)
def _repair_scratch_space_error(db_path: Path) -> Optional[str]:
"""Return an error unless snapshot, VACUUM and promotion can fit safely."""
budget = _disk_budget(db_path, "repair snapshot")
if isinstance(budget, str):
return budget
snapshot_bytes, free, headroom = budget
# VACUUM on the staged DB may need up to 2x the database size (SQLite docs);
# the same reserve then covers transactional promotion into the live DB.
error, snapshot_bytes, free, headroom = _disk_budget(db_path, "repair snapshot")
if error is not None:
return error
# VACUUM on the staged DB may need up to 2x the database size (SQLite docs); the same reserve then covers
# transactional promotion into the live DB.
if free >= snapshot_bytes + (2 * snapshot_bytes) + headroom:
return None
return (f"only {free / 1e9:.2f}GB free on {db_path.parent}; the repair snapshot needs up to "
@@ -281,10 +260,9 @@ def _backup_free_space_error(db_path: Path) -> Optional[str]:
"""Disk guard for the forensic copy: reason to refuse, or None. A full raw copy on a nearly-full volume (which a
preceding repair loop may itself have caused) can finish off the disk and every process on the machine."""
hint = _MANUAL_RECOVER_HINT.format(db_path=db_path)
budget = _disk_budget(db_path, "forensic copy")
if isinstance(budget, str):
return f"{budget}. {hint}"
need, free, headroom = budget
error, need, free, headroom = _disk_budget(db_path, "forensic copy")
if error is not None:
return f"{error}. {hint}"
if free - need >= headroom:
return None
return (f"only {free / 1e9:.2f}GB free on {db_path.parent}; copying the damaged DB needs {need / 1e9:.2f}GB and must "
@@ -309,14 +287,12 @@ def _repair_failure_consumes_attempt(exc: BaseException) -> bool:
must not burn the repair ledger. Only SQLite's corruption/image result codes prove deterministic damage."""
if not isinstance(exc, sqlite3.DatabaseError):
return False
error_code = getattr(exc, "sqlite_errorcode", None)
if isinstance(error_code, int):
if isinstance(error_code := getattr(exc, "sqlite_errorcode", None), int):
# Extended result codes keep the primary code in the low byte.
return (error_code & 0xFF) in (sqlite3.SQLITE_CORRUPT, sqlite3.SQLITE_NOTADB)
# Older sqlite3 without result-code attributes: narrow message match only,
# never turning generic "disk is full"/"readonly" into permanent failures.
message = str(exc).lower()
return "file is not a database" in message or "database disk image is malformed" in message
# Older sqlite3 without result-code attributes: narrow message match only, never turning generic
# "disk is full"/"readonly" into permanent failures.
return any(m in str(exc).lower() for m in ("file is not a database", "database disk image is malformed"))
def _repair_ledger_path(db_path: Path) -> Path:
@@ -338,12 +314,9 @@ def _db_fingerprint(db_path: Path) -> "Optional[str]":
st = db_path.stat()
with open(db_path, "rb") as fh:
head = fh.read(_FINGERPRINT_SAMPLE_BYTES)
tail = b""
if st.st_size > _FINGERPRINT_SAMPLE_BYTES:
fh.seek(max(0, st.st_size - _FINGERPRINT_SAMPLE_BYTES))
tail = fh.read(_FINGERPRINT_SAMPLE_BYTES)
fh.seek(max(0, st.st_size - _FINGERPRINT_SAMPLE_BYTES))
tail = fh.read(_FINGERPRINT_SAMPLE_BYTES) if st.st_size > _FINGERPRINT_SAMPLE_BYTES else b""
return f"{st.st_size}:{hashlib.sha256(_mask_volatile_header(head) + tail).hexdigest()[:32]}"
return _read_offline(db_path, "fingerprint", _sample)
@@ -367,7 +340,6 @@ def _backup_content_identity(db_path: Path) -> "Optional[str]":
for chunk in iter(lambda: fh.read(1024 * 1024), b""):
hasher.update(chunk)
return hasher.hexdigest()
return _read_offline(db_path, "backup-identity", _digest)
@@ -420,8 +392,8 @@ def _record_repair_outcome(db_path: Path, *, repaired: bool, fingerprint: "Optio
fp = fingerprint if fingerprint is not None else _db_fingerprint(db_path)
if fp is None:
if not isinstance(recorded, str):
# No prior key to extend and no safe way to mint one; the
# in-process claim and cross-process lock still bound this run.
# No prior key to extend and no safe way to mint one; the in-process claim and cross-process lock
# still bound this run.
return
fp = recorded
attempts = int(ledger.get("failed_attempts", 0)) + 1 if recorded == fp else 1
@@ -457,22 +429,20 @@ def _publish_backup_bundle(db_path: Path, staging: Path, backup_path: Path) -> N
ORDER MATTERS: the main DB name is the bundle's commit marker (what ``_existing_malformed_backups``
counts), so sidecars publish FIRST and the main DB LAST — a failure partway never leaves a countable
backup over a missing sidecar. On failure, staging files AND anything promoted are removed."""
# (source, staged, destination); main DB copied first, published LAST.
main = (db_path, staging, backup_path)
triples: "List[Tuple[Path, Path, Path]]" = [
(sidecar, staging.with_name(staging.name + suffix), backup_path.with_name(backup_path.name + suffix))
for suffix, sidecar in zip(_DB_SIDECAR_SUFFIXES, _sidecars(db_path)) if sidecar.exists()
] + [main]
sidecars = [(sidecar, staging.with_name(staging.name + suffix), backup_path.with_name(backup_path.name + suffix))
for suffix, sidecar in zip(_DB_SIDECAR_SUFFIXES, _sidecars(db_path)) if sidecar.exists()]
published: "List[Path]" = []
try:
for src, staged, _dst in (main, *triples[:-1]):
for src, staged, _dst in (main, *sidecars):
shutil.copy2(src, staged)
for _src, staged, dst in triples:
for _src, staged, dst in (*sidecars, main):
os.replace(staged, dst)
published.append(dst)
except Exception:
for victim in (*(staged for _s, staged, _d in triples), *published):
_unlink_quiet(victim)
for victim in (*(staged for _s, staged, _d in (*sidecars, main)), *published):
with contextlib.suppress(OSError): # best effort; a failure here is never the caller's error
victim.unlink(missing_ok=True)
raise
@@ -489,17 +459,13 @@ def _backup_db_file(db_path: Path) -> "Tuple[Optional[Path], Optional[str]]":
— NOT mtime, NOT ``_db_fingerprint``); a repair loop once re-copied the same bytes on every restart. Staging
names live OUTSIDE the ``.malformed-backup-`` prefix: inside it they count as a backup, sort NEWEST (prune kept
partials, deleted intact copies) and dedupe could return one with no real forensic copy on disk."""
try:
with contextlib.suppress(ImportError): # scaffold/embed installs without hermes_cli track no connections
from hermes_cli.sqlite_safe_read import has_live_connection
live = has_live_connection(db_path)
except ImportError:
live = False
if live:
reason = (f"a connection to {db_path} is still open in this process; raw-copying it would cancel that "
"connection's POSIX advisory locks. Close all SessionDB handles first.")
logger.error("Refusing to raw-copy %s for backup: %s", db_path, reason)
return None, reason
if has_live_connection(db_path):
reason = (f"a connection to {db_path} is still open in this process; raw-copying it would cancel that "
"connection's POSIX advisory locks. Close all SessionDB handles first.")
logger.error("Refusing to raw-copy %s for backup: %s", db_path, reason)
return None, reason
stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
backup_path = db_path.with_name(f"{db_path.name}.malformed-backup-{stamp}")
for seq in itertools.count(1): # same-second collision must not overwrite the earlier forensic copy
@@ -512,22 +478,19 @@ def _backup_db_file(db_path: Path) -> "Tuple[Optional[Path], Optional[str]]":
# prefix-matches as a backup, sorts NEWEST and would otherwise survive prune forever.
for pattern in (f"{db_path.name}.backup-staging-*", f"{db_path.name}.malformed-backup-*.incomplete*"):
for old in db_path.parent.glob(pattern):
_unlink_quiet(old)
with contextlib.suppress(OSError):
old.unlink(missing_ok=True)
with contextlib.suppress(OSError):
# Hash the source only when there is a candidate: hashing a multi-GB
# file right before copying it is waste on the first-corruption pass.
newest = _existing_malformed_backups(db_path)[:1]
if newest:
# Hash the source only when there is a candidate: hashing a multi-GB file before copying it is waste.
if newest := _existing_malformed_backups(db_path)[:1]:
src_id = _backup_content_identity(db_path)
if src_id is not None and _backup_content_identity(newest[0]) == src_id:
logger.info("Reusing existing forensic backup %s (identical to the damaged DB).", newest[0])
return newest[0], None
reason = _backup_free_space_error(db_path)
if reason is not None:
if (reason := _backup_free_space_error(db_path)) is not None:
logger.error("Refusing forensic backup of %s: %s", db_path, reason)
return None, reason
staging = db_path.with_name(f"{db_path.name}.backup-staging-{stamp}")
_publish_backup_bundle(db_path, staging, backup_path)
_publish_backup_bundle(db_path, db_path.with_name(f"{db_path.name}.backup-staging-{stamp}"), backup_path)
_prune_malformed_backups(db_path)
return backup_path, None
except Exception as exc: # pragma: no cover - best effort
@@ -544,19 +507,14 @@ def preflight_db_writability(db_path: Path, *, db_label: str = "state.db") -> No
tree (Hermes owns those files; ``chmod`` fails on files the user doesn't own, bounding the repair exactly);
otherwise fail fast naming the file and command. Never deletes/truncates a WAL sidecar — once writable, the
normal open checkpoints it. ``:memory:``/``file:`` skipped. Shared with ``kanban_db``."""
import stat as _stat
if str(db_path) == ":memory:" or str(db_path).startswith("file:"):
return
home: Optional[Path] = None
with contextlib.suppress(Exception): # pragma: no cover - defensive
home = Path(get_hermes_home()).resolve()
# SQLite needs a writable directory in every journal mode (WAL/SHM sidecars,
# or the rollback journal in DELETE mode).
# SQLite needs a writable directory in every journal mode (WAL/SHM sidecars, or the DELETE-mode journal).
sidecars = (db_path.with_name(db_path.name + "-wal"), db_path.with_name(db_path.name + "-shm"))
targets = [(db_path.parent, True)] + [(p, False) for p in (db_path, *sidecars) if p.is_file()]
for p, is_dir in targets:
for p, is_dir in [(db_path.parent, True), *((p, False) for p in (db_path, *sidecars) if p.is_file())]:
if (is_dir and not p.is_dir()) or os.access(p, os.R_OK | os.W_OK):
continue
x = "x" if is_dir else ""
@@ -564,7 +522,7 @@ def preflight_db_writability(db_path: Path, *, db_label: str = "state.db") -> No
with contextlib.suppress(OSError, ValueError):
in_scope = home is not None and p.resolve().is_relative_to(home)
if in_scope:
os.chmod(p, p.stat().st_mode | _stat.S_IRUSR | _stat.S_IWUSR | (_stat.S_IXUSR if is_dir else 0))
os.chmod(p, p.stat().st_mode | stat.S_IRUSR | stat.S_IWUSR | (stat.S_IXUSR if is_dir else 0))
if in_scope and os.access(p, os.R_OK | os.W_OK):
logger.info("%s preflight: repaired read-only %s (chmod u+rw%s)", db_label, p, x)
continue
@@ -573,8 +531,7 @@ def preflight_db_writability(db_path: Path, *, db_label: str = "state.db") -> No
raise sqlite3.OperationalError(
f"{db_label} is not writable: {'directory' if is_dir else 'file'} {p} is read-only for this user. Hermes "
f"needs read-write access to open the database. Fix with: chmod u+rw{x} '{p}' (files owned by another "
f"user may need sudo/chown).{wal_note}"
)
f"user may need sudo/chown).{wal_note}")
def _connect_repair_durable(db_path: Path, *, timeout: float = 5.0) -> sqlite3.Connection:
@@ -616,9 +573,7 @@ def apply_durability_barriers(conn: sqlite3.Connection) -> bool:
ok = _reapply_durability_barriers(conn)
with contextlib.suppress(Exception):
from hermes_cli.config import cfg_get, load_config_readonly # local: avoids an import cycle
raw_synchronous = cfg_get(load_config_readonly(), "database", "synchronous", default=None)
if raw_synchronous is not None:
if (raw_synchronous := cfg_get(load_config_readonly(), "database", "synchronous", default=None)) is not None:
_apply_synchronous_pragma(conn, raw_synchronous, db_label="state.db (guest)")
return ok
@@ -635,16 +590,15 @@ def _open_exclusive(db_path: Path, begin: str) -> sqlite3.Connection:
*begin*; closed (unpinned) and re-raised when exclusion cannot be taken."""
conn = _connect_repair_durable(db_path, timeout=0.0)
try:
conn.execute("PRAGMA locking_mode=EXCLUSIVE")
conn.execute(begin)
conn.execute("ROLLBACK")
for statement in ("PRAGMA locking_mode=EXCLUSIVE", begin, "ROLLBACK"):
conn.execute(statement)
except BaseException:
_close_unpinned(conn)
raise
return conn
@contextmanager
@contextlib.contextmanager
def _exclusive_repair_db_guard(db_path: Path):
"""Yield ``(conn, None)`` — one live connection that excludes writers for repair surgery — or ``(None,
exc)`` when exclusion could not be taken.
@@ -663,15 +617,14 @@ def _exclusive_repair_db_guard(db_path: Path):
try:
yield guard, None
finally:
# Releasing the exclusive locks before close keeps a close-time checkpoint
# from being mistaken for a repair write by callers that reopen immediately.
# Releasing the exclusive locks before close keeps a close-time checkpoint from being mistaken for a
# repair write by callers that reopen immediately.
_close_unpinned(guard)
def _copy_database_snapshot(
source_path: Path, destination_path: Path, *, source_connection: Optional[sqlite3.Connection] = None,
destination_connection: Optional[sqlite3.Connection] = None,
) -> None:
def _copy_database_snapshot(source_path: Path, destination_path: Path, *,
source_connection: Optional[sqlite3.Connection] = None,
destination_connection: Optional[sqlite3.Connection] = None) -> None:
"""Copy one complete SQLite snapshot without replacing either file inode: the online backup API folds
committed WAL frames into the source snapshot and writes the destination in one transaction (rolled
back if interrupted), so ``state.db`` is never swapped out from under handles that refer to it."""
@@ -694,8 +647,7 @@ def _copy_database_snapshot(
def _schema_not_built(exc: BaseException) -> bool:
"""``no such table/column``: FTS5 / core tables not created yet (brand new file mid-init)."""
msg = str(exc).lower()
return "no such table" in msg or "no such column" in msg
return any(m in str(exc).lower() for m in ("no such table", "no such column"))
def _db_opens_cleanly(db_path: Path) -> Optional[str]:
@@ -708,53 +660,51 @@ def _db_opens_cleanly(db_path: Path) -> Optional[str]:
from hermes_state import SessionDB, load_fts5_cjk_extension
conn = _connect_repair_durable(db_path)
try:
# Best-effort tokenizer load: messages_fts_cjk needs cjk_unicode61 before any
# statement can touch it; tokenizer absence must never classify as corruption.
load_fts5_cjk_extension(conn)
conn.execute("PRAGMA journal_mode").fetchone()
rows = conn.execute("PRAGMA integrity_check").fetchall()
problems = [str(r[0]) for r in rows if r and str(r[0]).lower() != "ok"]
if problems:
return "; ".join(problems[:3])
conn.execute("SELECT COUNT(*) FROM sessions").fetchone()
# FTS5 read probe: partial shadow-table corruption makes MATCH/snippet/rank
# raise while integrity_check reports healthy. MATCH '""' (empty phrase)
# parses, scans zero rows and exercises the shadow tables; FTS5 rejects MATCH ''.
for fts_table in _FTS_TABLES:
with contextlib.closing(conn):
# Best-effort tokenizer load: messages_fts_cjk needs cjk_unicode61 before any statement can touch it;
# tokenizer absence must never classify as corruption.
load_fts5_cjk_extension(conn)
conn.execute("PRAGMA journal_mode").fetchone()
rows = conn.execute("PRAGMA integrity_check").fetchall()
problems = [str(r[0]) for r in rows if r and str(r[0]).lower() != "ok"]
if problems:
return "; ".join(problems[:3])
conn.execute("SELECT COUNT(*) FROM sessions").fetchone()
# FTS5 read probe: partial shadow-table corruption makes MATCH/snippet/rank raise while integrity_check
# reports healthy. MATCH '""' (empty phrase) parses, scans zero rows and exercises the shadow tables;
# FTS5 rejects MATCH ''.
for fts_table in _FTS_TABLES:
try:
conn.execute(f"SELECT 1 FROM {fts_table} WHERE {fts_table} MATCH '\"\"' LIMIT 1").fetchone()
except sqlite3.DatabaseError as exc:
# Builds without fts5/trigram raise "no such module|tokenizer"; calling that corruption would
# send the DB into repair, whose final fallback deletes messages_fts%. "no such table/column" =
# not built yet.
benign = SessionDB._is_fts5_unavailable_error(exc) or _schema_not_built(exc)
if not (isinstance(exc, sqlite3.OperationalError) and benign):
return f"fts5 read probe failed on {fts_table}: {exc}"
# FTS write probe: drive a row through the messages_fts* triggers in a transaction that is always
# rolled back.
probe_session_id = f"_hermes_fts_health_probe_{time.time_ns()}"
try:
conn.execute(f"SELECT 1 FROM {fts_table} WHERE {fts_table} MATCH '\"\"' LIMIT 1").fetchone()
except sqlite3.DatabaseError as exc:
# Builds without fts5/trigram raise "no such module|tokenizer"; calling that corruption would send the
# DB into repair, whose final fallback deletes messages_fts%. "no such table/column" = not built yet.
benign = SessionDB._is_fts5_unavailable_error(exc) or _schema_not_built(exc)
if not (isinstance(exc, sqlite3.OperationalError) and benign):
return f"fts5 read probe failed on {fts_table}: {exc}"
# FTS write probe: drive a row through the messages_fts* triggers in a
# transaction that is always rolled back.
probe_session_id = f"_hermes_fts_health_probe_{time.time_ns()}"
try:
conn.execute("BEGIN IMMEDIATE")
conn.execute("INSERT INTO sessions (id, source, started_at) VALUES (?, ?, ?)",
(probe_session_id, "_health_probe", time.time()))
conn.execute("INSERT INTO messages (session_id, role, content, timestamp) VALUES (?, ?, ?, ?)",
(probe_session_id, "user", "_fts_health_probe", time.time()))
conn.execute("ROLLBACK")
except sqlite3.OperationalError as exc:
with contextlib.suppress(sqlite3.Error):
conn.execute("BEGIN IMMEDIATE")
conn.execute("INSERT INTO sessions (id, source, started_at) VALUES (?, ?, ?)",
(probe_session_id, "_health_probe", time.time()))
conn.execute("INSERT INTO messages (session_id, role, content, timestamp) VALUES (?, ?, ?, ?)",
(probe_session_id, "user", "_fts_health_probe", time.time()))
conn.execute("ROLLBACK")
# Missing messages/sessions tables = brand new file mid-init, not corruption.
# "no such tokenizer": this process lacks the cjk extension the DB's index
# needs — capability gap; a tokenizer-less SessionDB drops the triggers itself.
if _schema_not_built(exc) or "no such tokenizer: cjk_unicode61" in str(exc).lower():
return None
return str(exc)
return None
except sqlite3.OperationalError as exc:
with contextlib.suppress(sqlite3.Error):
conn.execute("ROLLBACK")
# Missing messages/sessions tables = brand new file mid-init, not corruption. "no such tokenizer":
# this process lacks the cjk extension the DB's index needs — capability gap; a tokenizer-less
# SessionDB drops the triggers itself.
if _schema_not_built(exc) or "no such tokenizer: cjk_unicode61" in str(exc).lower():
return None
return str(exc)
return None
except sqlite3.DatabaseError as exc:
return str(exc)
finally:
conn.close()
def _live_writer_holds_db(db_path: Path) -> bool:
@@ -768,7 +718,7 @@ def _live_writer_holds_db(db_path: Path) -> bool:
repair is then serialised only by the cross-process repairer lock."""
try:
probe = _open_exclusive(db_path, "BEGIN IMMEDIATE")
with contextlib.suppress(Exception):
with contextlib.suppress(Exception): # a close() error is not evidence of a holder
_close_unpinned(probe)
return False
except sqlite3.OperationalError as exc:
@@ -798,27 +748,26 @@ def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, A
unless ``backup=False``. Serialised across processes (gateway, Desktop backend and CLI open the same file;
concurrent ``writable_schema`` surgery is itself a corruption source). Returns ``{repaired, strategy,
backup_path, error}``."""
from hermes_state import _cross_process_repair_lock, _db_opens_cleanly, _live_writer_holds_db, _persistent_repair_attempts_exhausted, _probe_journal_mode_for_repair, _record_repair_outcome, _repair_state_db_schema_locked
from hermes_state import (_cross_process_repair_lock, _db_opens_cleanly, _live_writer_holds_db,
_persistent_repair_attempts_exhausted, _probe_journal_mode_for_repair,
_record_repair_outcome, _repair_state_db_schema_locked)
report: Dict[str, Any] = {"repaired": False, "strategy": None, "backup_path": None, "error": None}
# Startup-watchdog lease: repair is I/O-bound (near-zero CPU), which the watchdog's CPU fallback would
# misread as a parked deadlock. One lease (clamped to _MAX_LEASE_S=900) beats per-chunk renewal complexity.
report_startup_progress(900.0, phase="state_db_repair")
db_path = Path(db_path)
if not db_path.exists():
report["error"] = f"{db_path} does not exist"
return report
# Cross-restart cap: the in-memory claim bounds one process, but unhealable
# b-tree damage used to re-run surgery + a fresh backup on EVERY restart.
# Cross-restart cap: the in-memory claim bounds one process, but unhealable b-tree damage used to re-run
# surgery + a fresh backup on EVERY restart.
if _persistent_repair_attempts_exhausted(db_path):
return _repair_skip(report, "skipped", _persistent_repair_exhausted_error(db_path))
with _cross_process_repair_lock(db_path) as holding_lock:
if not holding_lock:
# Another process holds the lock (or the lock file was unopenable);
# it may have healed the file already, so re-probe before failing.
# Another process holds the lock (or the lock file was unopenable); it may have healed the file
# already, so re-probe before failing.
if _db_opens_cleanly(db_path) is None:
report["repaired"], report["strategy"] = True, "repaired_by_other_process"
else:
@@ -827,18 +776,15 @@ def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, A
return report
result = report
# Recheck exhaustion after acquisition: a queued repairer can have
# recorded the final failure while this process waited.
# Recheck exhaustion after acquisition: a queued repairer can have recorded the final failure while
# this process waited.
if _persistent_repair_attempts_exhausted(db_path):
_repair_skip(report, "skipped", _persistent_repair_exhausted_error(db_path))
# WAL-holder preflight: fail closed for active readers before a backup is taken. Not the race defence — the
# exclusive guard in the locked routine excludes writers through promotion and sees DELETE-mode readers too.
elif _live_writer_holds_db(db_path):
_repair_skip(
report, "skipped",
"a live writer still holds state.db; skipped schema surgery to avoid tearing b-tree pages under a "
"concurrent writer. Stop the gateway (hermes gateway stop) and retry.",
)
_repair_skip(report, "skipped", "a live writer still holds state.db; skipped schema surgery to avoid tearing "
"b-tree pages under a concurrent writer. Stop the gateway (hermes gateway stop) and retry.")
else:
# Probe journal mode BEFORE surgery: a rebuilt file comes back in the default (delete) mode and nothing
# else records the flip. Unprobeable (damaged file) -> database.journal_mode is the restore target.
@@ -850,8 +796,7 @@ def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, A
# Environmental aborts (before a strategy mutates the snapshot) are retriable, not proof of exhaustion;
# the private marker stays out of the public report. The ledger update stays under the cross-process
# lock so two repairers cannot lose each other's updates; a queued loser must not record at all.
attempted = bool(result.pop("_repair_attempted", False))
if attempted or result.get("repaired"):
if result.pop("_repair_attempted", False) or result.get("repaired"):
_record_repair_outcome(db_path, repaired=bool(result.get("repaired")))
return result
@@ -899,84 +844,69 @@ def _repair_state_db_schema_locked(db_path: Path, *, backup: bool, report: Dict[
file from the schema SQLite can still parse — when the damage IS in the schema b-tree (the ``malformed database
schema ()`` class) every table hanging off the unreadable part is silently dropped, the probe still reports
malformed, and repair returned ``repaired=False`` having destroyed what it was asked to save."""
from hermes_state import _backup_db_file, _copy_database_snapshot, _db_opens_cleanly, _repair_scratch_space_error, _run_repair_strategies, _unlink_db_triple
from hermes_state import (_backup_db_file, _copy_database_snapshot, _db_opens_cleanly,
_repair_scratch_space_error, _run_repair_strategies, _unlink_db_triple)
scratch = db_path.with_name(f"{db_path.name}.repair-scratch")
cleanup_error = _unlink_db_triple(scratch)
if cleanup_error is not None:
if (cleanup_error := _unlink_db_triple(scratch)) is not None:
return _repair_skip(report, "aborted", f"could not remove a stale repair snapshot before probing state.db: {cleanup_error}")
# Re-probe under the lock: a process we queued behind may have just repaired
# the file; redoing surgery would undo it (the repair/re-corrupt cascade).
# Re-probe under the lock: a process we queued behind may have just repaired the file; redoing surgery
# would undo it (the repair/re-corrupt cascade).
if _db_opens_cleanly(db_path) is None:
report["repaired"], report["strategy"] = True, "already_healthy"
return report
if backup:
bpath, backup_error = _backup_db_file(db_path)
report["backup_path"] = str(bpath) if bpath else None
if bpath is None: # HARD STOP: the forensic image is the recovery path when every strategy fails.
return _repair_skip(report, "aborted", "pre-repair backup refused; aborting schema repair to avoid "
f"mutating the only copy of the damaged DB: {backup_error}")
# The forensic copy precedes this guard on purpose: its live-holder checks
# would be poisoned by our own exclusive connection. Everything touching the
# repair image or live promotion happens only under writer exclusion.
# The forensic copy precedes this guard on purpose: its live-holder checks would be poisoned by our own
# exclusive connection. Everything touching the repair image or live promotion happens under writer exclusion.
with _exclusive_repair_db_guard(db_path) as (live_guard, guard_error):
if live_guard is None:
return _repair_skip(report, "skipped", "could not acquire exclusive state.db repair ownership; skipped "
f"schema surgery to avoid overwriting a concurrent writer. Stop the gateway and retry: "
f"{guard_error}", exc=guard_error)
space_error = _repair_scratch_space_error(db_path)
if space_error is not None:
if (space_error := _repair_scratch_space_error(db_path)) is not None:
return _repair_skip(report, "aborted", space_error)
try:
# Source = live_guard: it owns the exclusion, and a second connection
# could be blocked by our own EXCLUSIVE lock on some SQLite builds.
# Source = live_guard: it owns the exclusion, and a second connection could be blocked by our own
# EXCLUSIVE lock on some SQLite builds.
_copy_database_snapshot(db_path, scratch, source_connection=live_guard)
except (OSError, sqlite3.Error, TimeoutError) as exc:
_unlink_db_triple(scratch)
return _repair_skip(report, "aborted", f"could not stage a complete SQLite repair snapshot of {db_path}: {exc}", exc=exc)
try:
# Private marker for the outer wrapper: a strategy failure consumes the
# persistent budget; a promotion failure is classified separately.
# Private marker for the outer wrapper: a strategy failure consumes the persistent budget; a
# promotion failure is classified separately.
report["_repair_attempted"] = True
_run_repair_strategies(scratch, report)
if report.get("repaired"):
_promote_repaired_snapshot(scratch, db_path, live_guard, report)
# Never ``os.replace`` the live DB: Windows rejects replacement under open handles and POSIX would
# leave those handles on the old inode. The guard keeps writer exclusion throughout.
try:
_copy_database_snapshot(scratch, db_path, destination_connection=live_guard)
except (OSError, sqlite3.Error, TimeoutError) as exc:
report.update(repaired=False, strategy=None,
_repair_attempted=_repair_failure_consumes_attempt(exc))
report["error"] = f"repaired snapshot could not be promoted transactionally: {exc}"
logger.error("state.db repair promotion failed: %s", exc)
else:
logger.warning("state.db repaired via '%s' and promoted transactionally: %s",
report.get("strategy"), db_path)
if not report.get("repaired"):
# Logged HERE, not in the strategies: they see the scratch copy, and
# the message a human acts on must name a path that still exists.
# Logged HERE, not in the strategies: they see the scratch copy, and the message a human acts on
# must name a path that still exists.
logger.error("state.db schema repair could not recover %s automatically (no committed canonical data "
"was modified or lost; backup: %s); manual restore from backup may be required.",
db_path, report["backup_path"])
return report
finally:
# Never leave a half-repaired file beside the DB to be mistaken for the real thing.
cleanup_error = _unlink_db_triple(scratch)
if cleanup_error is not None:
if (cleanup_error := _unlink_db_triple(scratch)) is not None:
logger.warning("Could not remove state.db repair snapshot after repair: %s", cleanup_error)
def _promote_repaired_snapshot(scratch: Path, db_path: Path, live_guard: sqlite3.Connection, report: Dict[str, Any]) -> None:
"""Copy the repaired *scratch* back into the live DB through *live_guard*.
Never ``os.replace`` the live DB: Windows rejects replacement under open
handles and POSIX would leave those handles on the old inode. The guard
keeps writer exclusion throughout. On failure the report reverts to unrepaired."""
from hermes_state import _copy_database_snapshot
try:
_copy_database_snapshot(scratch, db_path, destination_connection=live_guard)
except (OSError, sqlite3.Error, TimeoutError) as exc:
report.update(repaired=False, strategy=None, _repair_attempted=_repair_failure_consumes_attempt(exc))
report["error"] = f"repaired snapshot could not be promoted transactionally: {exc}"
logger.error("state.db repair promotion failed: %s", exc)
else:
logger.warning("state.db repaired via '%s' and promoted transactionally: %s", report.get("strategy"), db_path)
def _unlink_db_triple(path: Path) -> Optional[str]:
"""Remove *path* and every SQLite sidecar; return any cleanup failure."""
from hermes_state import _IS_WINDOWS
@@ -1014,10 +944,8 @@ def _strategy_rebuild_fts(conn: sqlite3.Connection) -> None:
# The cjk index can only be rebuilt with its tokenizer loaded (best-effort).
load_fts5_cjk_extension(conn)
for table_name in _FTS_TABLES:
try:
with contextlib.suppress(sqlite3.OperationalError): # table absent (FTS disabled / trigram off / cjk absent)
conn.execute(f"INSERT INTO {table_name}({table_name}) VALUES('rebuild')")
except sqlite3.OperationalError:
continue # table absent (FTS disabled / trigram off / cjk not present)
def _strategy_reindex(conn: sqlite3.Connection) -> None:
@@ -1029,19 +957,18 @@ def _strategy_reindex(conn: sqlite3.Connection) -> None:
conn.commit()
def _dedup_sqlite_master(conn: sqlite3.Connection) -> bool:
"""Delete duplicate sqlite_master rows (lowest rowid per type/name wins)."""
dupes = conn.execute(
"SELECT type, name, COUNT(*) AS c, MIN(rowid) AS keep FROM sqlite_master GROUP BY type, name HAVING c > 1"
).fetchall()
for type_, name, _count, keep in dupes:
conn.execute("DELETE FROM sqlite_master WHERE type IS ? AND name IS ? AND rowid <> ?", (type_, name, keep))
return bool(dupes)
def _strategy_dedup_schema(conn: sqlite3.Connection) -> None:
"""De-duplicate sqlite_master, keeping FTS."""
_edit_sqlite_master(conn, lambda: _dedup_sqlite_master(conn))
"""De-duplicate sqlite_master (lowest rowid per type/name wins), keeping FTS."""
def _dedup() -> bool:
dupes = conn.execute(
"SELECT type, name, COUNT(*) AS c, MIN(rowid) AS keep FROM sqlite_master GROUP BY type, name HAVING c > 1"
).fetchall()
for type_, name, _count, keep in dupes:
conn.execute("DELETE FROM sqlite_master WHERE type IS ? AND name IS ? AND rowid <> ?",
(type_, name, keep))
return bool(dupes)
_edit_sqlite_master(conn, _dedup)
def _strategy_drop_fts_vacuum(conn: sqlite3.Connection) -> None:
@@ -1057,18 +984,15 @@ def _strategy_drop_fts_vacuum(conn: sqlite3.Connection) -> None:
# (name, body, success log, failure log) in escalation order. failure log None =
# final strategy: its failure lands in report["error"] instead of logged-and-skipped.
_REPAIR_STRATEGIES = (
("rebuild_fts", _strategy_rebuild_fts,
"state.db FTS indexes rebuilt in place (schema preserved): %s",
("rebuild_fts", _strategy_rebuild_fts, "state.db FTS indexes rebuilt in place (schema preserved): %s",
"state.db FTS in-place rebuild pass failed: %s"),
("reindex_btree", _strategy_reindex,
"state.db B-tree indexes rebuilt via REINDEX: %s",
("reindex_btree", _strategy_reindex, "state.db B-tree indexes rebuilt via REINDEX: %s",
"state.db REINDEX pass failed: %s"),
("dedup_schema", _strategy_dedup_schema,
"state.db schema repaired by de-duplicating sqlite_master (FTS index preserved): %s",
"state.db dedup repair pass failed: %s"),
("drop_fts_rebuild", _strategy_drop_fts_vacuum,
"state.db schema repaired by dropping FTS schema; indexes will rebuild from messages on next open: %s",
None),
"state.db schema repaired by dropping FTS schema; indexes will rebuild from messages on next open: %s", None),
)
@@ -1076,7 +1000,6 @@ def _run_repair_strategies(db_path: Path, report: Dict[str, Any]) -> Dict[str, A
"""Escalating repair attempts, applied to *db_path* IN PLACE — only ever a scratch copy nothing else holds open,
never the user's database. The "could not recover" log lives in the caller so it names the user's database."""
from hermes_state import _db_opens_cleanly
for name, body, success_msg, failure_msg in _REPAIR_STRATEGIES:
try:
with _repair_conn(db_path) as conn:
+65 -126
View File
@@ -4,6 +4,7 @@ Plain mixin for ``hermes_state.SessionDB`` (no ``__init__``/state of its own).
Must never import hermes_state (cycle); shared constants live in hermes_state_common.
"""
import contextlib
import datetime
import hashlib
import logging
@@ -196,10 +197,8 @@ class SessionSchemaMixin:
def _drop_all_fts_triggers(self, cursor: sqlite3.Cursor) -> None:
self._drop_fts_triggers(cursor)
for trigger in _FTS_CJK_TRIGGERS:
try:
with contextlib.suppress(sqlite3.OperationalError):
cursor.execute(f"DROP TRIGGER IF EXISTS {trigger}")
except sqlite3.OperationalError:
pass
@staticmethod
def _fts_triggers_missing(cursor: sqlite3.Cursor, names: Sequence[str]) -> bool:
@@ -207,10 +206,8 @@ class SessionSchemaMixin:
if not names:
return False # "name IN ()" is a SQLite syntax error
placeholders = ",".join("?" for _ in names)
row = cursor.execute(
f"SELECT COUNT(*) FROM sqlite_master WHERE type = 'trigger' AND name IN ({placeholders})", tuple(names),
).fetchone()
return int(row[0]) < len(names)
sql = f"SELECT COUNT(*) FROM sqlite_master WHERE type = 'trigger' AND name IN ({placeholders})"
return int(cursor.execute(sql, tuple(names)).fetchone()[0]) < len(names)
@staticmethod
def _fts_update_trigger_needs_narrowing(sql: Optional[str]) -> bool:
@@ -227,13 +224,12 @@ class SessionSchemaMixin:
# CJK is v23-only. Decide the layout before selecting destructive candidates so the
# legacy branch never drops a trigger it won't recreate.
legacy_layout = self._db_has_legacy_inline_fts(cursor)
update_names = ("messages_fts_update", "messages_fts_trigram_update")
if not legacy_layout:
update_names += ("messages_fts_cjk_update",)
update_names = ("messages_fts_update", "messages_fts_trigram_update") + (
() if legacy_layout else ("messages_fts_cjk_update",)
)
placeholders = ", ".join("?" for _ in update_names)
rows = cursor.execute(
f"SELECT name, sql FROM sqlite_master WHERE type = 'trigger' AND name IN ({placeholders})", update_names,
).fetchall()
sql = f"SELECT name, sql FROM sqlite_master WHERE type = 'trigger' AND name IN ({placeholders})"
rows = cursor.execute(sql, update_names).fetchall()
to_drop = [name for name, sql in rows if self._fts_update_trigger_needs_narrowing(sql)]
if not to_drop:
return 0
@@ -253,7 +249,10 @@ class SessionSchemaMixin:
self._quarantine_cjk_after_update_of_migration(cursor)
logger.exception("CJK FTS re-ensure after UPDATE OF migration failed")
raise
if not self._cjk_update_trigger_is_narrowed(cursor):
row = cursor.execute(
"SELECT sql FROM sqlite_master WHERE type = 'trigger' AND name = ?", ("messages_fts_cjk_update",),
).fetchone()
if not row or self._fts_update_trigger_needs_narrowing(row[0]):
self._quarantine_cjk_after_update_of_migration(cursor)
logger.warning(
"CJK FTS UPDATE trigger missing or still broad after "
@@ -262,13 +261,6 @@ class SessionSchemaMixin:
logger.info("Migrated %d broad FTS UPDATE trigger(s) to AFTER UPDATE OF (no rebuild required)", len(to_drop))
return len(to_drop)
def _cjk_update_trigger_is_narrowed(self, cursor: sqlite3.Cursor) -> bool:
"""True when messages_fts_cjk_update exists with AFTER UPDATE OF."""
row = cursor.execute(
"SELECT sql FROM sqlite_master WHERE type = 'trigger' AND name = ?", ("messages_fts_cjk_update",),
).fetchone()
return bool(row) and not self._fts_update_trigger_needs_narrowing(row[0])
def _quarantine_cjk_after_update_of_migration(self, cursor: sqlite3.Cursor) -> None:
"""Fail closed after dropping the CJK UPDATE trigger mid-migration: clear availability,
persist ``fts_cjk_stale``, drop any residual trigger so a later open cannot
@@ -284,22 +276,20 @@ class SessionSchemaMixin:
logger.debug("Could not drop residual CJK UPDATE trigger after quarantine", exc_info=True)
@staticmethod
def _rebuild_fts_indexes(cursor: sqlite3.Cursor, *, include_trigram: bool = True) -> None:
def _rebuild_fts_indexes(cursor: sqlite3.Cursor, *, legacy: bool = False, include_trigram: bool = True) -> None:
"""v23+ external-content 'rebuild'. It indexes EVERY row, so the deferred-backfill
markers are cleared or the worker would re-insert covered rows (duplicates)."""
cursor.execute("INSERT INTO messages_fts(messages_fts) VALUES('rebuild')")
if include_trigram:
cursor.execute("INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('rebuild')")
cursor.execute(_CLEAR_REBUILD_MARKERS_SQL)
@staticmethod
def _rebuild_legacy_fts_indexes(cursor: sqlite3.Cursor, *, include_trigram: bool = True) -> None:
"""Rebuild the LEGACY inline (pre-v23) FTS indexes: no external-content 'rebuild' source,
so DELETE + reinsert the concatenated content the legacy triggers produced."""
markers are cleared or the worker would re-insert covered rows (duplicates).
``legacy`` (pre-v23 inline layout) has no external-content 'rebuild' source, so it
DELETEs + reinserts the concatenated content the legacy triggers produced."""
tables = ("messages_fts", "messages_fts_trigram") if include_trigram else ("messages_fts",)
for tbl in tables:
cursor.execute(f"DELETE FROM {tbl}")
cursor.execute(f"INSERT INTO {tbl}(rowid, content) SELECT id, {_LEGACY_INLINE_CONCAT_SQL}FROM messages")
if legacy:
cursor.execute(f"DELETE FROM {tbl}")
cursor.execute(f"INSERT INTO {tbl}(rowid, content) SELECT id, {_LEGACY_INLINE_CONCAT_SQL}FROM messages")
else:
cursor.execute(f"INSERT INTO {tbl}({tbl}) VALUES('rebuild')")
if not legacy:
cursor.execute(_CLEAR_REBUILD_MARKERS_SQL)
def _fts_table_probe(self, cursor: sqlite3.Cursor, table_name: str) -> Optional[bool]:
"""True = queryable, False = absent, None = FTS module/tokenizer missing or content
@@ -328,9 +318,7 @@ class SessionSchemaMixin:
decode_exc = exc
logger.warning(
"%s probe encountered invalid UTF-8 in FTS content; "
"search may return incomplete results until FTS is rebuilt: %s",
table_name,
decode_exc,
"search may return incomplete results until FTS is rebuilt: %s", table_name, decode_exc,
)
return None
@@ -342,17 +330,14 @@ class SessionSchemaMixin:
``_FTS_HOLDER_ESCALATE_SECONDS``, provably inactive orphan Desktop backends are
reaped and the holders re-checked."""
now = time.time()
record = {}
try:
row = cursor.execute(
"SELECT value FROM state_meta WHERE key = ? LIMIT 1", (FTS_REBUILD_DEFERRAL_KEY,),
).fetchone()
except sqlite3.Error:
row = None
if row:
parsed = safe_json_loads(row[0])
if isinstance(parsed, dict):
record = parsed
parsed = safe_json_loads(row[0]) if row else None
record = parsed if isinstance(parsed, dict) else {}
try:
first_seen = float(record.get("first_seen", now))
attempts = int(record.get("attempts", 0)) + 1
@@ -375,9 +360,7 @@ class SessionSchemaMixin:
if reaped:
logger.error(
"Reaped inactive orphan Desktop backend(s) %s after %d "
"state.db FTS rebuild deferrals; checking holders again.",
reaped,
attempts,
"state.db FTS rebuild deferrals; checking holders again.", reaped, attempts,
)
foreign_holders = self._foreign_state_db_holders()
if foreign_holders:
@@ -385,17 +368,14 @@ class SessionSchemaMixin:
"state.db FTS repair remains blocked after %d deferrals "
"by holder(s) %s. Stop the listed processes, then run "
"`hermes sessions optimize-storage` with the gateway stopped. "
"`hermes doctor` reports this degraded state.",
attempts,
foreign_holders,
"`hermes doctor` reports this degraded state.", attempts, foreign_holders,
)
if not foreign_holders:
return False
logger.warning(
"Deferred stale state.db FTS rebuild while foreign processes "
"hold the database or WAL sidecars (%s); canonical writes and LIKE search remain available (deferral %d).",
foreign_holders,
attempts,
foreign_holders, attempts,
)
return True
@@ -409,8 +389,8 @@ class SessionSchemaMixin:
with fts_rebuild_admission(self.db_path, timeout_seconds=timeout_seconds) as admitted:
if not admitted:
logger.warning(
"Deferred stale state.db FTS rebuild: another process "
"holds the rebuild authority; canonical writes and LIKE search remain available."
"Deferred stale state.db FTS rebuild: another process holds the rebuild authority; "
"canonical writes and LIKE search remain available."
)
return False
return self._recover_stale_fts_locked(cursor, legacy=legacy)
@@ -445,10 +425,8 @@ class SessionSchemaMixin:
# decides when it comes back online.
self._ensure_fts_cjk_schema(cursor)
self._fts_stale_retry_interval = 0.0
try:
with contextlib.suppress(sqlite3.Error):
self._conn.commit()
except sqlite3.Error:
pass
return recovered
except Exception: # noqa: BLE001 - background retry must never raise
logger.warning(
@@ -460,59 +438,43 @@ class SessionSchemaMixin:
"""Body of :meth:`_recover_stale_fts`; caller holds rebuild authority. One write
transaction, so no canonical writer slips between rebuild and trigger restoration."""
try:
trigram_status = self._fts_table_probe(cursor, "messages_fts_trigram")
include_trigram = self._fts_table_probe(cursor, "messages_fts_trigram") is True
except (sqlite3.DatabaseError, UnicodeDecodeError):
# A corrupt vtable may fail even a LIMIT 0 probe; still include it in the drop-and-recreate.
trigram_status = True
include_trigram = trigram_status is True
include_trigram = True
drop_sql = "".join(f"DROP TRIGGER IF EXISTS {trigger};" for trigger in _FTS_TRIGGERS)
if include_trigram:
drop_sql += "DROP TABLE IF EXISTS messages_fts_trigram;"
drop_sql += "DROP VIEW IF EXISTS messages_fts_trigram_src;"
drop_sql += "DROP TABLE IF EXISTS messages_fts;"
drop_sql += "DROP VIEW IF EXISTS messages_fts_trigram_src;DROP TABLE IF EXISTS messages_fts;"
if legacy:
schema_sql = LEGACY_FTS_SQL
if include_trigram:
schema_sql += LEGACY_FTS_TRIGRAM_SQL
rebuild_sql = schema_sql + _legacy_inline_reinsert_sql("messages_fts", 16)
rebuild_sql = LEGACY_FTS_SQL + (LEGACY_FTS_TRIGRAM_SQL if include_trigram else "")
rebuild_sql += _legacy_inline_reinsert_sql("messages_fts", 16)
if include_trigram:
rebuild_sql += _legacy_inline_reinsert_sql("messages_fts_trigram", 20, delete_first=True)
else:
schema_sql = FTS_SQL
if include_trigram:
schema_sql += FTS_TRIGRAM_SQL
rebuild_sql = schema_sql + "INSERT INTO messages_fts(messages_fts) VALUES('rebuild');"
rebuild_sql = FTS_SQL + (FTS_TRIGRAM_SQL if include_trigram else "")
rebuild_sql += "INSERT INTO messages_fts(messages_fts) VALUES('rebuild');"
if include_trigram:
rebuild_sql += "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('rebuild');"
rebuild_sql += _CLEAR_REBUILD_MARKERS_SQL + ";"
recovery_sql = (
"BEGIN IMMEDIATE;"
+ drop_sql
+ rebuild_sql
+ "DELETE FROM state_meta WHERE key IN "
+ f"('{FTS_STALE_KEY}', '{FTS_REBUILD_DEFERRAL_KEY}');"
+ "COMMIT;"
"BEGIN IMMEDIATE;" + drop_sql + rebuild_sql
+ f"DELETE FROM state_meta WHERE key IN ('{FTS_STALE_KEY}', '{FTS_REBUILD_DEFERRAL_KEY}');COMMIT;"
)
try:
cursor.executescript(recovery_sql)
except sqlite3.DatabaseError as exc:
try:
with contextlib.suppress(sqlite3.Error):
self._conn.rollback()
except sqlite3.Error:
pass
# Stale indexes must stay detached even on builds whose DDL transaction behavior differs.
self._drop_all_fts_triggers(cursor)
self._conn.commit()
logger.error(
"Automatic rebuild of stale FTS indexes failed (%s); "
"canonical writes remain enabled with FTS detached.",
exc,
"canonical writes remain enabled with FTS detached.", exc,
)
return False
self._fts_stale = False
self._fts_enabled = True
self._trigram_available = include_trigram
@@ -529,7 +491,7 @@ class SessionSchemaMixin:
database still runs every startup. A corrupt/stale cache degrades to recomputation."""
cache_path = None
schema_hash = hashlib.sha256(schema_sql.encode("utf-8")).hexdigest()
try:
with contextlib.suppress(Exception): # missing/corrupt cache → recompute below
# Late import: resolves a test-patched hermes_constants.get_hermes_home.
from hermes_constants import get_hermes_home as _home
cache_path = _home() / "cache" / "schema_columns.json"
@@ -539,8 +501,6 @@ class SessionSchemaMixin:
isinstance(cols, dict) and all(isinstance(v, str) for v in cols.values()) for cols in tables.values()
):
return tables
except Exception:
pass # missing/corrupt cache → recompute below
ref = sqlite3.connect(":memory:")
try:
@@ -550,9 +510,8 @@ class SessionSchemaMixin:
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'"
).fetchall():
cols: Dict[str, str] = {}
for _cid, col_name, col_type, notnull, default, pk in ref.execute(
f'PRAGMA table_info("{tbl}")'
).fetchall():
info = ref.execute(f'PRAGMA table_info("{tbl}")').fetchall()
for _cid, col_name, col_type, notnull, default, pk in info:
# Reconstruct the type expression for ALTER TABLE ADD COLUMN
parts = [col_type] if col_type else []
if notnull and not pk:
@@ -565,14 +524,12 @@ class SessionSchemaMixin:
ref.close()
if cache_path is not None:
try:
with contextlib.suppress(Exception): # cache write is best-effort
cache_path.parent.mkdir(parents=True, exist_ok=True)
fd, tmp = tempfile.mkstemp(dir=str(cache_path.parent), prefix=".schema_columns.")
with os.fdopen(fd, "w", encoding="utf-8") as fh:
json.dump({"schema_hash": schema_hash, "tables": table_columns}, fh)
os.replace(tmp, cache_path)
except Exception:
pass # cache write is best-effort
return table_columns
def _reconcile_columns(self, cursor: sqlite3.Cursor) -> None:
@@ -613,7 +570,7 @@ class SessionSchemaMixin:
try:
rows = cursor.execute(f'PRAGMA table_info("{table}")').fetchall()
except sqlite3.OperationalError:
return None
rows = None
if not rows:
return None
# row: (cid, name, type, notnull, dflt_value, pk)
@@ -730,15 +687,12 @@ class SessionSchemaMixin:
# Heal NULL ``active`` rows on every startup: older reconciler builds added ``active``
# without NOT NULL DEFAULT 1, so ``WHERE active = 1`` loaders hid whole histories. A
# ``current_version < 12`` gate never re-ran for already-v12+ databases.
try:
with contextlib.suppress(sqlite3.OperationalError):
cursor.execute("UPDATE messages SET active = 1 WHERE active IS NULL")
except sqlite3.OperationalError:
pass
fts5_available = self._sqlite_supports_fts5(cursor)
self._fts_stale = cursor.execute(
"SELECT 1 FROM state_meta WHERE key = ? LIMIT 1", (FTS_STALE_KEY,)
).fetchone() is not None
stale_row = cursor.execute("SELECT 1 FROM state_meta WHERE key = ? LIMIT 1", (FTS_STALE_KEY,)).fetchone()
self._fts_stale = stale_row is not None
if self._fts_stale:
# A prior process detached FTS after corruption; stay detached until a full rebuild.
self._drop_all_fts_triggers(cursor)
@@ -752,10 +706,9 @@ class SessionSchemaMixin:
cursor.execute("INSERT INTO schema_version (version) VALUES (?)", (SCHEMA_VERSION,))
# Store provenance so fresh vs wiped stores are distinguishable.
now_iso = datetime.datetime.now(datetime.timezone.utc).isoformat()
instance_id = str(uuid.uuid4())
cursor.executemany(
"INSERT OR IGNORE INTO state_meta (key, value) VALUES (?, ?)",
[("store_instance_id", instance_id), ("store_created_at_utc", now_iso)],
[("store_instance_id", str(uuid.uuid4())), ("store_created_at_utc", now_iso)],
)
else:
self._run_data_migrations(cursor, row[0], fts5_available)
@@ -773,7 +726,7 @@ class SessionSchemaMixin:
# (v10 trigram backfill and v11 inline FTS re-index were superseded by v23 and removed.)
if current_version < 16:
# v16: tag delegate subagent rows so pickers stay clean after parent deletes orphan them.
try:
with contextlib.suppress(sqlite3.OperationalError):
cursor.execute(
"UPDATE sessions SET model_config = json_set("
"COALESCE(model_config, '{}'), '$._delegate_from', parent_session_id) "
@@ -791,8 +744,6 @@ class SessionSchemaMixin:
"AND NOT EXISTS (SELECT 1 FROM sessions ch "
" WHERE ch.parent_session_id = sessions.id)"
)
except sqlite3.OperationalError:
pass
if current_version < 18:
# v18: best-effort gateway metadata backfill from sessions.json.
try:
@@ -801,10 +752,8 @@ class SessionSchemaMixin:
logger.debug("v18 gateway metadata backfill skipped: %s", exc)
if current_version < 20:
# v20: seed session_model_usage from sessions aggregates (OR IGNORE: newer rows win).
try:
with contextlib.suppress(sqlite3.OperationalError):
cursor.execute(_SESSION_MODEL_USAGE_V20_SEED_SQL)
except sqlite3.OperationalError:
pass
if current_version < 22:
self._migrate_v22_session_model_usage(cursor)
# v23: FTS storage redesign (external-content tables). OPT-IN, NOT AUTOMATIC: the
@@ -875,16 +824,14 @@ class SessionSchemaMixin:
cursor.execute(_TITLE_UNIQUE_INDEX_SQL)
except sqlite3.IntegrityError:
try:
cursor.execute(
"""UPDATE sessions AS older
cursor.execute("""UPDATE sessions AS older
SET title = NULL
WHERE title IS NOT NULL
AND EXISTS (
SELECT 1 FROM sessions AS newer
WHERE newer.title = older.title
AND newer.rowid > older.rowid
)"""
)
)""")
logger.warning(
"Cleared %d duplicate session title(s) while restoring the unique index", cursor.rowcount,
)
@@ -905,12 +852,9 @@ class SessionSchemaMixin:
# CJK was detached alongside the base indexes; its ensure path decides when it returns.
self._ensure_fts_cjk_schema(cursor)
else:
self._fts_enabled = False
self._trigram_available = False
self._fts_cjk_available = False
self._fts_enabled = self._trigram_available = self._fts_cjk_available = False
else:
base_sql, trigram_sql = _FTS_DDL[legacy_fts]
rebuild = self._rebuild_legacy_fts_indexes if legacy_fts else self._rebuild_fts_indexes
# Measure BEFORE the DDL below runs (pre-repair state). Whether the trigram half is
# creatable is only known AFTER _ensure_fts_schema, hence the halves combine at the `if`.
base_triggers_missing = self._fts_triggers_missing(cursor, _FTS_BASE_TRIGGERS)
@@ -922,7 +866,8 @@ class SessionSchemaMixin:
self._trigram_available = trigram_enabled
if base_triggers_missing or (trigram_enabled and trigram_triggers_missing):
self._run_admitted_startup_rebuild(
cursor, lambda: rebuild(cursor, include_trigram=trigram_enabled),
cursor,
lambda: self._rebuild_fts_indexes(cursor, legacy=legacy_fts, include_trigram=trigram_enabled),
)
if not legacy_fts:
# CJK-bigram index: strictly additive, gated on the loadable tokenizer.
@@ -950,9 +895,7 @@ class SessionSchemaMixin:
cursor.execute(_STALE_KEY_UPSERT_SQL, (FTS_STALE_KEY,))
self._drop_all_fts_triggers(cursor)
self._fts_stale = True
self._fts_enabled = False
self._trigram_available = False
self._fts_cjk_available = False
self._fts_enabled = self._trigram_available = self._fts_cjk_available = False
def _backfill_gateway_metadata_from_sessions_json(self, cursor: sqlite3.Cursor) -> None:
"""One-time v18 backfill of gateway metadata from sessions.json. Only fills NULL
@@ -986,13 +929,9 @@ class SessionSchemaMixin:
END
WHERE id = ?""",
(
entry.get("session_key") or key,
origin_dict.get("chat_id") if origin_dict is not None else None,
entry.get("chat_type"),
origin_dict.get("thread_id") if origin_dict is not None else None,
entry.get("display_name"),
json.dumps(origin) if origin_dict is not None else None,
1 if entry.get("expiry_finalized") or entry.get("memory_flushed") else 0,
str(session_id),
entry.get("session_key") or key, origin_dict.get("chat_id") if origin_dict is not None else None,
entry.get("chat_type"), origin_dict.get("thread_id") if origin_dict is not None else None,
entry.get("display_name"), json.dumps(origin) if origin_dict is not None else None,
1 if entry.get("expiry_finalized") or entry.get("memory_flushed") else 0, str(session_id),
),
)
+93 -176
View File
@@ -4,6 +4,7 @@ Plain mixin for ``hermes_state.SessionDB`` (no ``__init__``/state of its own).
Must never import hermes_state (cycle); shared constants live in hermes_state_common.
"""
import contextlib
import logging
import re
import sqlite3
@@ -143,8 +144,7 @@ def _search_select_sql(snippet_sql: str, from_sql: str, where: List[str], order_
def _search_filter_clauses(
where: List[str], params: list, *, include_inactive: bool, source_filter: Optional[List[str]],
exclude_sources: Optional[List[str]], role_filter: Optional[List[str]],
) -> None:
exclude_sources: Optional[List[str]], role_filter: Optional[List[str]]) -> None:
"""Append the visibility/source/role predicates every search route shares. Live rows
(active=1) AND compaction-archived rows (compacted=1) are discoverable; only
rewind/undo rows (active=0, compacted=0) are hidden."""
@@ -204,9 +204,8 @@ class SessionSearchMixin:
return self._rebuild_status("fts_cjk_rebuild")
def _rebuild_status(self, prefix: str) -> Optional[Dict[str, Any]]:
rows = self._read_all(
"SELECT key, value FROM state_meta WHERE key IN (?, ?)", (f"{prefix}_high_water", f"{prefix}_progress"),
)
rows = self._read_all("SELECT key, value FROM state_meta WHERE key IN (?, ?)",
(f"{prefix}_high_water", f"{prefix}_progress"))
meta = {r["key"]: r["value"] for r in rows}
high_water = meta.get(f"{prefix}_high_water")
if high_water is None or int(high_water) <= 0:
@@ -239,9 +238,8 @@ class SessionSearchMixin:
def _fts_cjk_rebuild_finish(self) -> None:
"""Boundary sweep + clear the cjk markers; index becomes servable."""
self._rebuild_finish("fts_cjk_rebuild", [
self._BOUNDARY_SWEEP_SQL.format(table="messages_fts_cjk", extra="AND m.role <> 'tool' ")
])
sweep = self._BOUNDARY_SWEEP_SQL.format(table="messages_fts_cjk", extra="AND m.role <> 'tool' ")
self._rebuild_finish("fts_cjk_rebuild", [sweep])
self._fts_cjk_available = True
logger.info("CJK FTS index backfill complete — serving CJK search.")
@@ -265,19 +263,16 @@ class SessionSearchMixin:
inserts = [self._CHUNK_INSERT_SQL.format(table="messages_fts", extra="")]
if self._trigram_available:
inserts.append(self._CHUNK_INSERT_SQL.format(table="messages_fts_trigram", extra=" AND role <> 'tool'"))
return self._rebuild_step(
"fts_rebuild", inserts, fail_msg="FTS rebuild chunk failed (will retry): %s",
finish=self._fts_rebuild_finish,
)
return self._rebuild_step("fts_rebuild", inserts, fail_msg="FTS rebuild chunk failed (will retry): %s",
finish=self._fts_rebuild_finish)
def fts_cjk_rebuild_step(self) -> bool:
"""Backfill one chunk of the CJK index. True while work remains."""
if not self._fts_enabled or not self._fts_cjk_loaded:
return False
return self._rebuild_step(
"fts_cjk_rebuild", [self._CHUNK_INSERT_SQL.format(table="messages_fts_cjk", extra=" AND role <> 'tool'")],
fail_msg="CJK FTS rebuild chunk failed (will retry): %s", finish=self._fts_cjk_rebuild_finish,
)
insert = self._CHUNK_INSERT_SQL.format(table="messages_fts_cjk", extra=" AND role <> 'tool'")
return self._rebuild_step("fts_cjk_rebuild", [insert], finish=self._fts_cjk_rebuild_finish,
fail_msg="CJK FTS rebuild chunk failed (will retry): %s")
def _rebuild_step(self, prefix: str, insert_sqls: List[str], *, fail_msg: str, finish) -> bool:
"""Shared chunk engine for the base and CJK deferred backfills."""
@@ -322,13 +317,10 @@ class SessionSearchMixin:
each chunk's scan is bounded (restarting the scan was O(n²)); compound-key tables
keep the chunked ``LIMIT`` delete — they are small by construction."""
with self._lock:
trash = [
r[0] for r in self._conn.execute(
"SELECT name FROM sqlite_master WHERE type = 'table' "
"AND name LIKE ? ESCAPE '\\'",
(self._FTS_TRASH_PREFIX.replace("_", "\\_") + "%",),
).fetchall()
]
trash = [r[0] for r in self._conn.execute(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE ? ESCAPE '\\'",
(self._FTS_TRASH_PREFIX.replace("_", "\\_") + "%",),
).fetchall()]
if not trash:
return False
tbl = trash[0]
@@ -358,9 +350,7 @@ class SessionSearchMixin:
cur = conn.execute(
f"DELETE FROM {tbl} WHERE ({key}) IN (SELECT {key} FROM {tbl} LIMIT {self._FTS_REBUILD_CHUNK_ROWS})"
)
if cur.rowcount == 0:
return _drop(conn)
return True # re-check: more trash tables / chunks may remain
return _drop(conn) if cur.rowcount == 0 else True # True: more trash tables / chunks may remain
def _drop(conn, marker_key: Optional[str] = None) -> bool:
"""Drained — the DROP is cheap now. True: re-check for more trash."""
@@ -411,29 +401,22 @@ class SessionSearchMixin:
except sqlite3.OperationalError:
return False # table absent / FTS disabled mid-init — not this failure class
def _fts_index_known_empty(self, conn) -> bool:
"""True when the base external-content index holds no rows; a missing table counts."""
try:
return int(conn.execute("SELECT COUNT(*) FROM messages_fts_docsize").fetchone()[0]) == 0
except sqlite3.OperationalError:
return True
def _reset_fts_index_to_empty(self, conn) -> None:
"""Truncate the v23 external-content tables via FTS5 ``'delete-all'`` (a plain DELETE is
O(rows) and corrupts the index when indexed rows diverged from ``messages``). The
backfill worker replays without an anti-join, so it needs a known-empty index."""
for tbl in ("messages_fts", "messages_fts_trigram"):
try:
conn.execute(f"INSERT INTO {tbl}({tbl}) VALUES('delete-all')")
except sqlite3.OperationalError:
pass # table absent — already an empty surface
def _reseed_missing_progress(self, conn) -> None:
"""high_water without progress: fts_rebuild_step reads missing progress as "done by
another process" and optimize would no-op then stamp. Reset to known-empty, re-seed."""
another process" and optimize would no-op then stamp. Reset to known-empty, re-seed.
Truncation goes through FTS5 ``'delete-all'`` (a plain DELETE is O(rows) and corrupts
the index when indexed rows diverged from ``messages``); the backfill worker replays
without an anti-join, so it needs a known-empty index. A missing docsize table counts
as empty."""
if _meta_row(conn, "fts_rebuild_progress") is None:
if not self._fts_index_known_empty(conn):
self._reset_fts_index_to_empty(conn)
try:
known_empty = int(conn.execute("SELECT COUNT(*) FROM messages_fts_docsize").fetchone()[0]) == 0
except sqlite3.OperationalError:
known_empty = True
if not known_empty:
for tbl in ("messages_fts", "messages_fts_trigram"):
with contextlib.suppress(sqlite3.OperationalError): # table absent — already an empty surface
conn.execute(f"INSERT INTO {tbl}({tbl}) VALUES('delete-all')")
self.set_meta("fts_rebuild_progress", "0", cursor=conn)
def _seed_fts_rebuild_markers(self, conn, *, force: bool = False) -> int:
@@ -494,26 +477,22 @@ class SessionSearchMixin:
def _stage(conn):
self._drop_fts_triggers(conn)
conn.execute("DROP VIEW IF EXISTS messages_fts_trigram_src")
had = bool(conn.execute(
if conn.execute(
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name IN ('messages_fts', 'messages_fts_trigram') "
"AND sql LIKE 'CREATE VIRTUAL TABLE%' LIMIT 1"
).fetchone())
if had:
).fetchone():
conn.execute("PRAGMA writable_schema=ON")
conn.execute(
"DELETE FROM sqlite_master WHERE type = 'table' "
"AND name IN ('messages_fts', 'messages_fts_trigram') AND sql LIKE 'CREATE VIRTUAL TABLE%'"
)
conn.execute("PRAGMA writable_schema=RESET")
shadows = [
r[0] for r in conn.execute(
"SELECT name FROM sqlite_master WHERE type = 'table' "
"AND (name LIKE 'messages_fts_%' ESCAPE '\\' "
"OR name LIKE 'messages_fts_trigram_%' ESCAPE '\\')"
).fetchall()
]
for sh in shadows:
conn.execute(f"ALTER TABLE {sh} RENAME TO fts_v22_trash_{sh}")
for row in conn.execute(
"SELECT name FROM sqlite_master WHERE type = 'table' "
"AND (name LIKE 'messages_fts_%' ESCAPE '\\' "
"OR name LIKE 'messages_fts_trigram_%' ESCAPE '\\')"
).fetchall():
conn.execute(f"ALTER TABLE {row[0]} RENAME TO fts_v22_trash_{row[0]}")
# Claim the backfill BEFORE the empty v23 tables exist so a crash before
# schema ensure resumes instead of stamping an empty index.
hw = self._seed_fts_rebuild_markers(conn, force=True)
@@ -536,17 +515,6 @@ class SessionSearchMixin:
raise sqlite3.OperationalError(failure_message)
self._conn.commit()
def _optimize_unsettled_reason(self, conn) -> Optional[str]:
"""Refusal reason while optimize work remains, else None. An empty base index against
non-empty messages also refuses (settling there meant permanent search-index loss)."""
if _meta_row(conn, "fts_rebuild_high_water") is not None:
return "backfill_incomplete"
if self._has_fts_trash(conn):
return "teardown_incomplete"
if self._fts_external_index_empty_with_messages(conn):
return "backfill_incomplete"
return None
def _optimize_vacuum(self) -> bool:
"""Phase 3: reclaim freed pages to the OS. False when VACUUM failed (usually no free disk
for its temp copy; a later VACUUM reclaims)."""
@@ -570,10 +538,15 @@ class SessionSearchMixin:
def _optimize_settle(self, conn) -> Optional[str]:
"""Phase 4 (inside the write transaction, so a concurrent writer cannot race a stamp past
incomplete work): stamp the FTS layout (source of truth for "optimized"), clear the
"available" flag, advance a lagging schema_version. Returns a refusal reason or None."""
refusal = self._optimize_unsettled_reason(conn)
if refusal is not None:
return refusal
"available" flag, advance a lagging schema_version. Returns a refusal reason or None.
Refuses while optimize work remains; an empty base index against non-empty messages
also refuses (settling there meant permanent search-index loss)."""
if _meta_row(conn, "fts_rebuild_high_water") is not None:
return "backfill_incomplete"
if self._has_fts_trash(conn):
return "teardown_incomplete"
if self._fts_external_index_empty_with_messages(conn):
return "backfill_incomplete"
self.set_meta("fts_storage_version", str(FTS_STORAGE_VERSION), cursor=conn)
_delete_meta(conn, "fts_optimize_available")
conn.execute("UPDATE schema_version SET version = ? WHERE version < ?", (SCHEMA_VERSION, SCHEMA_VERSION))
@@ -612,10 +585,8 @@ class SessionSearchMixin:
if progress_cb is None:
return
st = self.fts_rebuild_status() or self.fts_cjk_rebuild_status()
progress_cb({
"phase": phase, "percent": st["percent"] if st else 100,
"indexed": st["indexed"] if st else 0, "total": st["total"] if st else 0,
})
progress_cb({"phase": phase, "percent": st["percent"] if st else 100,
"indexed": st["indexed"] if st else 0, "total": st["total"] if st else 0})
def _drive(phase: str, step) -> None:
"""Run *step* to completion; the inter-chunk sleep is the single place the duty
@@ -636,17 +607,14 @@ class SessionSearchMixin:
# Phase 2: tear down the demoted legacy shadow tables in chunks.
_emit("teardown")
_drive("teardown", self._fts_teardown_trash_step)
with self._lock:
still_pending = _meta_row(self._conn, "fts_rebuild_high_water") is not None
still_trash = self._has_fts_trash(self._conn)
empty_index = self._fts_external_index_empty_with_messages(self._conn)
if still_pending or still_trash or empty_index:
reason = "backfill_incomplete" if still_pending or empty_index else "teardown_incomplete"
logger.warning(
"FTS storage optimization did not settle (%s): pending=%s trash=%s empty_index=%s",
reason, still_pending, still_trash, empty_index,
)
logger.warning("FTS storage optimization did not settle (%s): pending=%s trash=%s empty_index=%s",
reason, still_pending, still_trash, empty_index)
return {"ok": False, "reason": reason, "vacuumed": None}
vacuum_ok = None
@@ -666,8 +634,7 @@ class SessionSearchMixin:
def get_anchored_view(
self, session_id: str, around_message_id: int, window: int = 5, bookend: int = 3,
keep_roles: Optional[Tuple[str, ...]] = ("user", "assistant"),
) -> Dict[str, Any]:
keep_roles: Optional[Tuple[str, ...]] = ("user", "assistant")) -> Dict[str, Any]:
"""Anchored window (``get_messages_around``) plus session bookends, so one call yields the
goal and the resolution of a long session. ``window`` is filtered to ``keep_roles``
(None disables) EXCEPT the anchor; ``bookend_start`` / ``bookend_end`` are the
@@ -684,15 +651,11 @@ class SessionSearchMixin:
if keep_roles is not None:
keep_set = set(keep_roles)
filtered_window = [m for m in window_rows if m.get("id") == around_message_id or m.get("role") in keep_set]
bookend_start_rows: List[Any] = []
bookend_end_rows: List[Any] = []
if bookend > 0:
role_clause = ""
role_params: list = []
if keep_roles is not None:
role_clause = f" AND role IN ({','.join('?' for _ in keep_roles)})"
role_params = list(keep_roles)
role_clause = "" if keep_roles is None else f" AND role IN ({','.join('?' for _ in keep_roles)})"
role_params = [] if keep_roles is None else list(keep_roles)
with self._read_ctx() as conn:
def _bookend(op: str, boundary_id: int, order: str):
return conn.execute(
@@ -708,7 +671,6 @@ class SessionSearchMixin:
def _hydrate(row) -> Dict[str, Any]:
return self._row_to_message_dict(row, warn_context="get_anchored_view", summary_flag=False)
return {
"window": filtered_window, "messages_before": primitive["messages_before"],
"messages_after": primitive["messages_after"],
@@ -717,8 +679,7 @@ class SessionSearchMixin:
}
def list_recent_user_messages(
self, session_id: str, limit: int = 20, include_inactive: bool = False
) -> List[Dict[str, Any]]:
self, session_id: str, limit: int = 20, include_inactive: bool = False) -> List[Dict[str, Any]]:
"""The *limit* most-recent real user turns, newest first, as ``{id, timestamp, preview}``
(80 chars, whitespace collapsed); used by /rewind and ``/undo [N]``. Bookkeeping rows
(``display_kind`` set) are excluded. Legacy compaction handoffs are role='user' rows
@@ -727,17 +688,14 @@ class SessionSearchMixin:
with a DB pick that includes them."""
active_clause = "" if include_inactive else " AND active = 1"
display_clause = " AND (display_kind IS NULL OR display_kind = '')"
fetch_limit = int(limit) * 2 + 5
with self._lock:
rows = self._conn.execute(
"SELECT id, timestamp, content FROM messages WHERE session_id = ? AND role = 'user'"
f"{active_clause}{display_clause} "
"ORDER BY id DESC LIMIT ?",
(session_id, fetch_limit),
(session_id, int(limit) * 2 + 5),
).fetchall()
from agent.context_compressor import ContextCompressor
result: List[Dict[str, Any]] = []
for row in rows:
if len(result) >= int(limit):
@@ -745,8 +703,7 @@ class SessionSearchMixin:
decoded = self._decode_content(row["content"])
if ContextCompressor._is_context_summary_content(decoded):
continue # compaction handoff — never a user-originated turn
if isinstance(decoded, str):
# A /skill turn embeds the whole skill body; show what was typed.
if isinstance(decoded, str): # a /skill turn embeds the whole skill body; show what was typed
preview = describe_skill_invocation(decoded) or decoded
else:
preview = _flatten_text(decoded)
@@ -833,7 +790,8 @@ class SessionSearchMixin:
def _trigram_route_ok(self, raw_query: str) -> bool:
"""Per-token CJK length gate for the trigram index: ``广西 OR 桂林 OR 漓江`` has 6
CJK chars total but 2 per token, so trigram returns 0."""
return(self._count_cjk(raw_query) >= 3 and not self._has_short_cjk_token(raw_query) and self._trigram_available)
return (self._count_cjk(raw_query) >= 3 and not self._has_short_cjk_token(raw_query)
and self._trigram_available)
def _describe_search_path(self, query: str) -> str:
"""Best-effort name of the routing path a query takes (log-only)."""
@@ -848,26 +806,19 @@ class SessionSearchMixin:
raw = sanitized.strip('"').strip()
if self._fts_cjk_available and not self._has_lone_cjk_run(raw):
return "fts_cjk"
if self._trigram_route_ok(raw):
return "trigram"
return "like_scan"
return "trigram" if self._trigram_route_ok(raw) else "like_scan"
except Exception:
return "unknown"
# ── Query builders / runners ───────────────────────────────────────────
@staticmethod
def _fts_match_sql(
table: str, match_query: str, order_by_sql: str, *, include_inactive: bool, source_filter: Optional[List[str]],
exclude_sources: Optional[List[str]], role_filter: Optional[List[str]], limit: int, offset: int,
) -> Tuple[str, list]:
def _fts_match_sql(table: str, match_query: str, order_by_sql: str, *, limit: int, offset: int,
**filters) -> Tuple[str, list]:
"""MATCH query + params against one FTS5 index joined to messages/sessions."""
where = [f"{table} MATCH ?"]
params: list = [match_query]
_search_filter_clauses(
where, params, include_inactive=include_inactive, source_filter=source_filter,
exclude_sources=exclude_sources, role_filter=role_filter,
)
_search_filter_clauses(where, params, **filters)
params.extend([limit, offset])
sql = _search_select_sql(
f"snippet({table}, -1, '>>>', '<<<', '...', 40) AS snippet",
@@ -875,10 +826,8 @@ class SessionSearchMixin:
)
return sql, params
def _match_rows(
self, table: str, match_query: str, order_by_sql: str, *, fail_open: Optional[str] = None,
operational_debug: Optional[str] = None, **kwargs,
) -> Optional[List[Dict[str, Any]]]:
def _match_rows(self, table: str, match_query: str, order_by_sql: str, *, fail_open: Optional[str] = None,
operational_debug: Optional[str] = None, **kwargs) -> Optional[List[Dict[str, Any]]]:
"""Run one MATCH against *table*; ``None`` when the query cannot execute (tokenizer /
syntax) so the caller falls back. *fail_open* names the index for the
substring-capable routes: a corruption-class ``DatabaseError`` there detaches the
@@ -896,8 +845,7 @@ class SessionSearchMixin:
raise
logger.warning(
"%s FTS search hit a corruption error (%s); detached FTS and falling back to canonical LIKE.",
fail_open, exc,
)
fail_open, exc)
return None
def _like_rows(self, where: List[str], params: list, *, order_by: str, limit_sql: str) -> List[Dict[str, Any]]:
@@ -944,8 +892,7 @@ class SessionSearchMixin:
return " OR ".join(compiled_groups), params, snippet_term
def _search_messages_like_fallback(
self, query: str, *, limit: int, offset: int, sort: Optional[str], **filters
) -> List[Dict[str, Any]]:
self, query: str, *, limit: int, offset: int, sort: Optional[str], **filters) -> List[Dict[str, Any]]:
"""Search canonical messages while derived FTS state is stale."""
predicate, params, snippet_term = self._compile_like_boolean_query(query)
if not predicate or snippet_term is None:
@@ -953,10 +900,8 @@ class SessionSearchMixin:
where = [f"({predicate})"]
_search_filter_clauses(where, params, **filters)
order = "ASC" if isinstance(sort, str) and sort.strip().lower() == "oldest" else "DESC"
return self._like_rows(
where, [snippet_term, *params, limit, offset],
order_by=f"ORDER BY m.timestamp {order}, m.id {order}", limit_sql="LIMIT ? OFFSET ?",
)
return self._like_rows(where, [snippet_term, *params, limit, offset],
order_by=f"ORDER BY m.timestamp {order}, m.id {order}", limit_sql="LIMIT ? OFFSET ?")
def _refresh_fts_stale_state(self) -> None:
"""Observe fail-open initiated by another process sharing state.db."""
@@ -968,13 +913,10 @@ class SessionSearchMixin:
return
if stale is not None:
self._fts_stale = True
self._fts_enabled = False
self._trigram_available = False
self._fts_cjk_available = False
self._fts_enabled = self._trigram_available = self._fts_cjk_available = False
def _finalize_search_matches(
self, matches: List[Dict[str, Any]], result_fields: Optional[Collection[str]] = None
) -> List[Dict[str, Any]]:
self, matches: List[Dict[str, Any]], result_fields: Optional[Collection[str]] = None) -> List[Dict[str, Any]]:
"""Attach neighboring messages (1 before + after, only when the projection consumes
``context``) and trim full content. Each context query takes its own read txn."""
if result_fields is None or "context" in result_fields:
@@ -983,9 +925,8 @@ class SessionSearchMixin:
with self._read_ctx() as conn:
rows = conn.execute(_CONTEXT_WINDOW_SQL, (match["id"], match["id"])).fetchall()
match["context"] = [
{"role": row["role"], "content": _flatten_text(self._decode_content(row["content"]))[:200]}
for row in rows
]
{"role": r["role"], "content": _flatten_text(self._decode_content(r["content"]))[:200]}
for r in rows]
except Exception:
match["context"] = []
# No route selects full content; the pop guards any future one that does.
@@ -1009,16 +950,14 @@ class SessionSearchMixin:
try:
rows = self._search_messages_impl(
query, source_filter=source_filter, exclude_sources=exclude_sources, role_filter=role_filter,
limit=limit, offset=offset, sort=sort, include_inactive=include_inactive, fields=fields,
)
limit=limit, offset=offset, sort=sort, include_inactive=include_inactive, fields=fields)
return rows
finally:
elapsed_ms = (time.time() - started) * 1000.0
if elapsed_ms >= env_float("HERMES_SEARCH_SLOW_MS", 1000.0):
logger.info(
"slow session search: path=%s elapsed=%.0fms rows=%s query=%r", self._describe_search_path(query),
elapsed_ms, len(rows) if rows is not None else "err", query[: 200],
)
logger.info("slow session search: path=%s elapsed=%.0fms rows=%s query=%r",
self._describe_search_path(query), elapsed_ms, len(rows) if rows is not None else "err",
query[: 200])
def _search_messages_impl(
self, query: str, source_filter: List[str] = None, exclude_sources: List[str] = None,
@@ -1036,11 +975,8 @@ class SessionSearchMixin:
query = self._sanitize_fts5_query(query)
if not query:
return []
filters = dict(
include_inactive=include_inactive, source_filter=source_filter,
exclude_sources=exclude_sources, role_filter=role_filter,
)
filters = dict(include_inactive=include_inactive, source_filter=source_filter,
exclude_sources=exclude_sources, role_filter=role_filter)
self._refresh_fts_stale_state()
if self._fts_stale:
matches = self._search_messages_like_fallback(query, limit=limit, offset=offset, sort=sort, **filters)
@@ -1103,8 +1039,7 @@ class SessionSearchMixin:
if self._fts_cjk_available and not wants_tool_rows and not self._has_lone_cjk_run(raw_query):
matches = self._match_rows(
"messages_fts_cjk", match_query, fail_open="CJK-bigram",
operational_debug="messages_fts_cjk query failed; falling back to trigram/LIKE", **route,
)
operational_debug="messages_fts_cjk query failed; falling back to trigram/LIKE", **route)
if matches is not None:
return matches
if self._trigram_route_ok(raw_query) and not wants_tool_rows:
@@ -1112,17 +1047,13 @@ class SessionSearchMixin:
if matches is not None:
return matches
non_op_tokens = _non_operator_tokens(raw_query) or [raw_query]
like_params: list = []
for tok in non_op_tokens:
like_params += _like_params(tok)
like_params: list = [p for tok in non_op_tokens for p in _like_params(tok)]
like_where = [f"({' OR '.join([_LIKE_ANY_COLUMN_SQL] * len(non_op_tokens))})"]
filters = {k: route[k] for k in ("include_inactive", "source_filter", "exclude_sources", "role_filter")}
_search_filter_clauses(like_where, like_params, **filters)
# instr() for the snippet uses the first search token.
return self._like_rows(
like_where, [non_op_tokens[0], *like_params, route["limit"], route["offset"]],
order_by="ORDER BY m.timestamp DESC", limit_sql="LIMIT ? OFFSET ?",
)
return self._like_rows(like_where, [non_op_tokens[0], *like_params, route["limit"], route["offset"]],
order_by="ORDER BY m.timestamp DESC", limit_sql="LIMIT ? OFFSET ?")
def _search_unindexed_gap(self, fts_query: str, limit: int, **filters) -> List[Dict[str, Any]]:
"""LIKE-scan ids in (fts_rebuild_progress, fts_rebuild_high_water] — rows the deferred
@@ -1131,26 +1062,19 @@ class SessionSearchMixin:
status = self.fts_rebuild_status()
if status is None or limit <= 0:
return []
terms = [
tok for tok in (t.strip('"').strip("*").strip() for t in _LIKE_TOKEN_RE.findall(fts_query))
if tok and tok.upper() not in _LIKE_SKIP_TOKENS
]
terms = [tok for tok in (t.strip('"').strip("*").strip() for t in _LIKE_TOKEN_RE.findall(fts_query))
if tok and tok.upper() not in _LIKE_SKIP_TOKENS]
if not terms:
return []
where = ["m.id > ? AND m.id <= ?"]
params: list = [status["indexed"], status["total"]]
for term in terms:
where.append(_LIKE_ANY_COLUMN_SQL)
params += _like_params(term)
where = ["m.id > ? AND m.id <= ?", *([_LIKE_ANY_COLUMN_SQL] * len(terms))]
params: list = [status["indexed"], status["total"], *(p for term in terms for p in _like_params(term))]
_search_filter_clauses(where, params, **filters)
return self._like_rows(
where, [terms[0], *params, limit], order_by="ORDER BY m.timestamp DESC", limit_sql="LIMIT ?",
)
return self._like_rows(where, [terms[0], *params, limit], order_by="ORDER BY m.timestamp DESC",
limit_sql="LIMIT ?")
def search_sessions_by_id(
self, query: str, limit: int = 20, include_archived: bool = True, source: str = None,
sources: List[str] = None, exclude_sources: List[str] = None,
) -> List[Dict[str, Any]]:
sources: List[str] = None, exclude_sources: List[str] = None) -> List[Dict[str, Any]]:
"""Search surfaced sessions by exact/prefix/substring session id. Also matches
``_lineage_root_id`` so an old compression root id resolves to the live continuation."""
needle = (query or "").strip().lower()
@@ -1160,18 +1084,13 @@ class SessionSearchMixin:
# chain) into SQL; over-fetch so the in-Python ranking has candidates.
candidates = self.list_sessions_rich(
source=source, sources=sources, exclude_sources=exclude_sources, limit=max(limit * 4, limit),
offset=0, include_archived=include_archived, order_by_last_active=True, id_query=needle,
)
offset=0, include_archived=include_archived, order_by_last_active=True, id_query=needle)
def score(row: Dict[str, Any]) -> int:
ids = [str(row.get("id") or ""), str(row.get("_lineage_root_id") or "")]
normalized = [value.lower() for value in ids if value]
normalized = [v.lower() for v in (str(row.get("id") or ""), str(row.get("_lineage_root_id") or "")) if v]
if any(value == needle for value in normalized):
return 0
if any(value.startswith(needle) for value in normalized):
return 1
return 2
return 1 if any(value.startswith(needle) for value in normalized) else 2
ranked = sorted(enumerate(candidates), key=lambda item: (score(item[1]), item[0]))
return [row for _, row in ranked[:limit]]
@@ -1214,8 +1133,7 @@ class SessionSearchMixin:
with fts_rebuild_admission(self.db_path) as admitted:
if not admitted:
logger.warning(
"Deferred in-place FTS rebuild: another process holds the rebuild authority for this state.db."
)
"Deferred in-place FTS rebuild: another process holds the rebuild authority for this state.db.")
return 0
with self._lock:
for tbl in self._present_fts_tables():
@@ -1243,7 +1161,6 @@ class SessionSearchMixin:
if max_commands is None:
max_commands = self._FTS_MERGE_COMMANDS_PER_PASS
_positive_int("max_commands", max_commands)
executed = 0
with self._lock:
for tbl in self._present_fts_tables():
+40 -85
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import contextlib
import logging
import sqlite3
import time
@@ -99,20 +100,12 @@ class SessionTelegramTopicsMixin:
tables (nobody ran ``/topic``) by returning their empty value; only
``enable``/``bind`` run the migration."""
def _topic_read_one(self, sql: str, params, default=None):
"""``fetchone`` that treats an unmigrated table as *default*."""
return self._topic_read(self._read_one, sql, params, default)
def _topic_read_all(self, sql: str, params) -> list:
"""``fetchall`` that treats an unmigrated table as no rows."""
return self._topic_read(self._read_all, sql, params, [])
@staticmethod
def _topic_read(reader, sql: str, params, default):
def _topic_read_one(self, sql: str, params):
"""``fetchone`` that treats an unmigrated table as None."""
try:
return reader(sql, params)
return self._read_one(sql, params)
except sqlite3.OperationalError:
return default
return None
def apply_telegram_topic_migration(self) -> None:
"""Create Telegram DM topic-mode tables on explicit /topic opt-in. Deliberately NOT
@@ -129,25 +122,21 @@ class SessionTelegramTopicsMixin:
# v1/v2 → v3. SQLite can't ALTER a PK or FK, so rebuild (also supplies v2's
# ON DELETE CASCADE). Legacy rows land in "default" only.
legacy_columns = columns.replace("profile_name, ", "", 1)
conn.executescript(
f"""
conn.executescript(f"""
CREATE TABLE {table}_new ({ddl});
INSERT INTO {table}_new ({columns})
SELECT 'default', {legacy_columns} FROM {table};
DROP TABLE {table};
ALTER TABLE {table}_new RENAME TO {table};
"""
)
""")
# Indexes after any rebuild: the user index needs profile_name.
conn.executescript(
"""
conn.executescript("""
CREATE UNIQUE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_session
ON telegram_dm_topic_bindings(session_id);
CREATE INDEX IF NOT EXISTS idx_telegram_dm_topic_bindings_user
ON telegram_dm_topic_bindings(profile_name, user_id, chat_id);
"""
)
""")
conn.execute(
"INSERT INTO state_meta (key, value) VALUES (?, ?) "
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
@@ -168,8 +157,7 @@ class SessionTelegramTopicsMixin:
def _to_int(value: Optional[bool]) -> Optional[int]:
return None if value is None else (1 if value else 0)
self._write_sql(
"""
self._write_sql("""
INSERT INTO telegram_dm_topic_mode (
profile_name, chat_id, user_id, enabled, activated_at, updated_at,
has_topics_enabled, allows_users_to_create_topics,
@@ -182,10 +170,8 @@ class SessionTelegramTopicsMixin:
has_topics_enabled = excluded.has_topics_enabled,
allows_users_to_create_topics = excluded.allows_users_to_create_topics,
capability_checked_at = excluded.capability_checked_at
""",
(profile_name, str(chat_id), str(user_id), now, now,
_to_int(has_topics_enabled), _to_int(allows_users_to_create_topics), now),
)
""", (profile_name, str(chat_id), str(user_id), now, now,
_to_int(has_topics_enabled), _to_int(allows_users_to_create_topics), now))
def disable_telegram_topic_mode(
self, *, chat_id: str, profile_name: str = "default", clear_bindings: bool = True
@@ -196,7 +182,7 @@ class SessionTelegramTopicsMixin:
profile_name = _normalize_telegram_topic_profile_name(profile_name)
def _do(conn):
try:
with contextlib.suppress(sqlite3.OperationalError):
conn.execute(
"UPDATE telegram_dm_topic_mode SET enabled = 0, updated_at = ? "
"WHERE profile_name = ? AND chat_id = ?",
@@ -207,20 +193,15 @@ class SessionTelegramTopicsMixin:
"DELETE FROM telegram_dm_topic_bindings WHERE profile_name = ? AND chat_id = ?",
(profile_name, str(chat_id)),
)
except sqlite3.OperationalError:
return
self._execute_write(_do)
def is_telegram_topic_mode_enabled(self, *, chat_id: str, user_id: str, profile_name: str = "default") -> bool:
"""Return whether Telegram DM topic mode is enabled for this chat/user."""
profile_name = _normalize_telegram_topic_profile_name(profile_name)
row = self._topic_read_one(
"""
row = self._topic_read_one("""
SELECT enabled FROM telegram_dm_topic_mode
WHERE profile_name = ? AND chat_id = ? AND user_id = ?
""",
(profile_name, str(chat_id), str(user_id)),
)
""", (profile_name, str(chat_id), str(user_id)))
return bool(row[0]) if row is not None else False
def get_telegram_topic_binding(
@@ -228,13 +209,10 @@ class SessionTelegramTopicsMixin:
) -> Optional[Dict[str, Any]]:
"""Return the session binding for a Telegram DM topic, if present."""
profile_name = _normalize_telegram_topic_profile_name(profile_name)
row = self._topic_read_one(
"""
row = self._topic_read_one("""
SELECT * FROM telegram_dm_topic_bindings
WHERE profile_name = ? AND chat_id = ? AND thread_id = ?
""",
(profile_name, str(chat_id), str(thread_id)),
)
""", (profile_name, str(chat_id), str(thread_id)))
return dict(row) if row else None
def list_telegram_topic_bindings_for_chat(
@@ -242,21 +220,21 @@ class SessionTelegramTopicsMixin:
) -> List[Dict[str, Any]]:
"""All bindings for one chat, newest first ([] when the table is absent)."""
profile_name = _normalize_telegram_topic_profile_name(profile_name)
rows = self._topic_read_all(
"SELECT * FROM telegram_dm_topic_bindings WHERE profile_name = ? AND chat_id = ? ORDER BY updated_at DESC",
(profile_name, str(chat_id)),
)
try:
rows = self._read_all(
"SELECT * FROM telegram_dm_topic_bindings WHERE profile_name = ? AND chat_id = ? ORDER BY updated_at DESC",
(profile_name, str(chat_id)),
)
except sqlite3.OperationalError:
return []
return [dict(row) for row in rows]
def get_telegram_topic_binding_by_session(self, *, session_id: str) -> Optional[Dict[str, Any]]:
"""Reverse lookup via the UNIQUE INDEX on session_id; None when unbound."""
row = self._topic_read_one(
"""
row = self._topic_read_one("""
SELECT * FROM telegram_dm_topic_bindings
WHERE session_id = ?
""",
(str(session_id),),
)
""", (str(session_id),))
return dict(row) if row else None
def delete_telegram_topic_binding(self, *, chat_id: str, thread_id: str, profile_name: str = "default") -> int:
@@ -268,41 +246,32 @@ class SessionTelegramTopicsMixin:
transaction, or a user who disabled topics in the Telegram client (not via
``/topic off``) stays stuck. Returns the number of rows deleted; absent binding or
unmigrated tables are silent no-ops (never raise from a cleanup hot path)."""
chat_id = str(chat_id)
thread_id = str(thread_id)
chat_id, thread_id = str(chat_id), str(thread_id)
profile_name = _normalize_telegram_topic_profile_name(profile_name)
def _do(conn) -> int:
try:
deleted = conn.execute(
"""
deleted = conn.execute("""
DELETE FROM telegram_dm_topic_bindings
WHERE profile_name = ? AND chat_id = ? AND thread_id = ?
""",
(profile_name, chat_id, thread_id),
).rowcount or 0
""", (profile_name, chat_id, thread_id)).rowcount or 0
except sqlite3.OperationalError:
return 0
if not deleted:
return 0
# Last binding gone → disable topic mode in the same transaction (no
# read-after-prune race).
try:
remaining = conn.execute(
"""
# read-after-prune race). telegram_dm_topic_mode absent — binding prune still stands.
with contextlib.suppress(sqlite3.OperationalError):
remaining = conn.execute("""
SELECT 1 FROM telegram_dm_topic_bindings
WHERE profile_name = ? AND chat_id = ? LIMIT 1
""",
(profile_name, chat_id),
).fetchone()
""", (profile_name, chat_id)).fetchone()
if remaining is None:
conn.execute(
"UPDATE telegram_dm_topic_mode SET enabled = 0, updated_at = ? "
"WHERE profile_name = ? AND chat_id = ?",
(time.time(), profile_name, chat_id),
)
except sqlite3.OperationalError:
pass # telegram_dm_topic_mode absent — binding prune still stands.
return deleted
return self._execute_write(_do)
@@ -321,24 +290,16 @@ class SessionTelegramTopicsMixin:
profile_name = _normalize_telegram_topic_profile_name(profile_name)
def _do(conn):
existing_session = conn.execute(
"""
existing_session = conn.execute("""
SELECT profile_name, chat_id, thread_id
FROM telegram_dm_topic_bindings
WHERE session_id = ?
""",
(session_id,),
).fetchone()
""", (session_id,)).fetchone()
if existing_session is not None:
linked_profile, linked_chat, linked_thread = existing_session
if (
str(linked_profile) != profile_name
or str(linked_chat) != chat_id
or str(linked_thread) != thread_id
):
if (str(linked_profile), str(linked_chat), str(linked_thread)) != (profile_name, chat_id, thread_id):
raise ValueError("session is already linked to another Telegram topic")
conn.execute(
"""
conn.execute("""
INSERT INTO telegram_dm_topic_bindings (
profile_name, chat_id, thread_id, user_id, session_key, session_id,
managed_mode, linked_at, updated_at
@@ -349,22 +310,16 @@ class SessionTelegramTopicsMixin:
session_id = excluded.session_id,
managed_mode = excluded.managed_mode,
updated_at = excluded.updated_at
""",
(profile_name, chat_id, thread_id, user_id, session_key, session_id,
managed_mode, now, now),
)
""", (profile_name, chat_id, thread_id, user_id, session_key, session_id, managed_mode, now, now))
self._execute_write(_do)
def is_telegram_session_linked_to_topic(self, *, session_id: str) -> bool:
"""True if the session is bound to any Telegram DM topic (absent tables → False)."""
row = self._topic_read_one(
"""
row = self._topic_read_one("""
SELECT 1 FROM telegram_dm_topic_bindings
WHERE session_id = ?
LIMIT 1
""",
(str(session_id),),
)
""", (str(session_id),))
return row is not None
def list_unlinked_telegram_sessions_for_user(
+13 -28
View File
@@ -27,9 +27,8 @@ class SessionTitlesMixin:
def _title_rank(cls, source: Optional[str]) -> int:
"""Rank a stored title_source. NULL (pre-provenance rows) is indistinguishable from a
manual ``/title`` of that era, so it ranks as ``user``."""
if source is None:
return cls._TITLE_SOURCE_RANK[cls.TITLE_SOURCE_USER]
return cls._TITLE_SOURCE_RANK.get(str(source), 0)
rank = cls._TITLE_SOURCE_RANK
return rank[cls.TITLE_SOURCE_USER] if source is None else rank.get(str(source), 0)
@staticmethod
def sanitize_title(title: Optional[str]) -> Optional[str]:
@@ -53,8 +52,7 @@ class SessionTitlesMixin:
if not ancestor_id or not descendant_id or ancestor_id == descendant_id:
return False
edge = _COMPRESSION_CHILD_SQL.format(a="child")
row = conn.execute(
f"""
return conn.execute(f"""
WITH RECURSIVE ancestors(id) AS (
SELECT ?
UNION
@@ -65,10 +63,7 @@ class SessionTitlesMixin:
WHERE {edge}
)
SELECT 1 FROM ancestors WHERE id = ? AND id != ? LIMIT 1
""",
(descendant_id, ancestor_id, descendant_id),
).fetchone()
return row is not None
""", (descendant_id, ancestor_id, descendant_id)).fetchone() is not None
def _set_session_title(self, session_id: str, title: str, *, source: str) -> bool:
"""Write a title, enforcing provenance precedence. A ``user`` write always lands;
@@ -91,17 +86,12 @@ class SessionTitlesMixin:
# exact-title lookup on every open), so a rename orphans the conversation. Hidden
# is the discriminator: canonical chats are born hidden; a visible session merely
# named "Bot Chat" stays renameable. Provenance-blind.
if (
(current["title"] or "") == self.CANONICAL_BOT_CHAT_TITLE
and bool(current["hidden"])
and title != self.CANONICAL_BOT_CHAT_TITLE
):
if ((current["title"] or "") == self.CANONICAL_BOT_CHAT_TITLE and bool(current["hidden"])
and title != self.CANONICAL_BOT_CHAT_TITLE):
if is_user:
raise ValueError(
"This is the bot's canonical Bot Chat — its name is its "
"identity, and renaming it would orphan the conversation. "
"To start fresh, create a new bot instead."
)
raise ValueError("This is the bot's canonical Bot Chat — its name is its "
"identity, and renaming it would orphan the conversation. "
"To start fresh, create a new bot instead.")
return 0
if not is_user and current["title"] is not None and self._title_rank(current["title_source"]) >= new_rank:
return 0
@@ -119,11 +109,10 @@ class SessionTitlesMixin:
raise ValueError(f"Title '{title}' is already in use by session {conflict_id}")
# CAS on the values just read (``IS`` is NULL-safe): a concurrent write between
# the SELECT and here loses instead of being overwritten.
cursor = conn.execute(
return conn.execute(
"UPDATE sessions SET title = ?, title_source = ? WHERE id = ? AND title IS ? AND title_source IS ?",
(title, source if title else None, session_id, current["title"], current["title_source"]),
)
return cursor.rowcount
).rowcount
return self._execute_write(_do) > 0
@@ -150,9 +139,7 @@ class SessionTitlesMixin:
def get_session_title_source(self, session_id: str) -> Optional[str]:
"""Get the provenance of a session's title, or None when untitled."""
row = self._read_one("SELECT title, title_source FROM sessions WHERE id = ?", (session_id,))
if not row or row["title"] is None:
return None
return row["title_source"]
return row["title_source"] if row and row["title"] is not None else None
def set_session_title_source(self, session_id: str, source: str) -> bool:
"""Overwrite a title's provenance without touching the text (a title copied across a
@@ -179,9 +166,7 @@ class SessionTitlesMixin:
"SELECT id, title, started_at FROM sessions "
"WHERE title LIKE ? ESCAPE '\\' ORDER BY started_at DESC",
(f"{_escape_like(title)} #%",))
if numbered:
return numbered[0]["id"]
return exact["id"] if exact else None
return numbered[0]["id"] if numbered else (exact["id"] if exact else None)
def get_next_title_in_lineage(self, base_title: str) -> str:
"""Next title in a lineage ("my session" -> "my session #2"): strip any " #N" suffix,
+17 -33
View File
@@ -4,6 +4,7 @@ per-model usage rows, and billing-route columns. Writer thread state lives on th
from __future__ import annotations
import atexit
import contextlib
import logging
import threading
import time
@@ -91,16 +92,13 @@ class SessionUsageMixin:
self.flush_token_counts()
def _do(conn):
conn.execute(
"""UPDATE sessions SET
conn.execute("""UPDATE sessions SET
billing_provider = ?,
billing_base_url = ?,
billing_mode = COALESCE(?, billing_mode),
system_prompt = NULL,
system_prompt_hash = NULL
WHERE id = ?""",
(provider, base_url, billing_mode, session_id),
)
WHERE id = ?""", (provider, base_url, billing_mode, session_id))
self._delete_unreferenced_system_prompts(conn)
self._execute_write(_do)
@@ -217,7 +215,7 @@ class SessionUsageMixin:
for session_id, kwargs in batch:
key = None
if not kwargs.get("absolute"):
key = (session_id,) + tuple(kwargs.get(f) for f in self._TOKEN_DELTA_ROUTE_FIELDS)
key = (session_id, *(kwargs.get(f) for f in self._TOKEN_DELTA_ROUTE_FIELDS))
if groups and key is not None and groups[-1][0] == key:
merged = groups[-1][2]
for f in self._TOKEN_DELTA_SUM_FIELDS:
@@ -268,10 +266,8 @@ class SessionUsageMixin:
self._apply_claimed_batch(batch)
def _drain_token_queue_at_exit(self) -> None:
try:
with contextlib.suppress(Exception): # never fatal at interpreter shutdown
self._stop_token_writer()
except Exception:
pass # never fatal at interpreter shutdown
def update_token_counts(
self, session_id: str, input_tokens: int=0, output_tokens: int=0, model: str=None, cache_read_tokens: int=0,
@@ -288,10 +284,8 @@ class SessionUsageMixin:
# locking, and the UPDATE would silently affect 0 rows.
self._insert_session_row(session_id, "unknown", model=model)
sql = _TOKEN_UPDATE_ABSOLUTE_SQL if absolute else _TOKEN_UPDATE_DELTA_SQL
has_usage = bool(
input_tokens or output_tokens or cache_read_tokens
or cache_write_tokens or reasoning_tokens or api_call_count or estimated_cost_usd
)
has_usage = bool(input_tokens or output_tokens or cache_read_tokens or cache_write_tokens or reasoning_tokens
or api_call_count or estimated_cost_usd)
has_accounted_usage = bool(has_usage or actual_cost_usd)
params = (
input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, reasoning_tokens,
@@ -320,13 +314,10 @@ class SessionUsageMixin:
and (existing.get("model") != model or existing.get("billing_provider") != billing_provider)
)
if first_accounted_route:
conn.execute(
"""UPDATE sessions
conn.execute("""UPDATE sessions
SET model = ?, billing_provider = ?,
billing_base_url = ?, billing_mode = ?
WHERE id = ?""",
(model, billing_provider, billing_base_url, billing_mode, session_id),
)
WHERE id = ?""", (model, billing_provider, billing_base_url, billing_mode, session_id))
conn.execute(sql, params)
if record_model_usage:
self._record_model_usage(conn, session_id, **usage)
@@ -350,16 +341,12 @@ class SessionUsageMixin:
sess = dict(row) if (row is not None and not task) else {}
counts = [v or 0 for v in (input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, reasoning_tokens)]
now = time.time()
conn.execute(
_MODEL_USAGE_UPSERT_SQL,
(
session_id, model or sess.get("model") or "unknown",
billing_provider or sess.get("billing_provider") or "",
billing_base_url or sess.get("billing_base_url") or "",
billing_mode or sess.get("billing_mode") or "", task or "",
api_call_count or 0, *counts,
float(estimated_cost_usd or 0.0), float(actual_cost_usd or 0.0),
cost_status, cost_source, now, now))
conn.execute(_MODEL_USAGE_UPSERT_SQL, (
session_id, model or sess.get("model") or "unknown",
billing_provider or sess.get("billing_provider") or "",
billing_base_url or sess.get("billing_base_url") or "",
billing_mode or sess.get("billing_mode") or "", task or "", api_call_count or 0, *counts,
float(estimated_cost_usd or 0.0), float(actual_cost_usd or 0.0), cost_status, cost_source, now, now))
def record_auxiliary_usage(
self, session_id: str, task: str, *, model: Optional[str]=None, billing_provider: Optional[str]=None,
@@ -386,13 +373,10 @@ class SessionUsageMixin:
params: List[Any] = [min_message_count]
if not include_archived:
where.append("COALESCE(archived, 0) = 0")
row = self._read_one(
f"""
row = self._read_one(f"""
SELECT COALESCE(SUM(COALESCE(input_tokens, 0) + COALESCE(output_tokens, 0)), 0),
COALESCE(SUM(COALESCE(actual_cost_usd, estimated_cost_usd, 0)), 0)
FROM sessions
WHERE {' AND '.join(where)}
""",
params,
)
""", params)
return {"tokens": int(row[0] or 0), "cost_usd": float(row[1] or 0.0)}
+55 -93
View File
@@ -30,7 +30,7 @@ _WAL_INCOMPAT_MARKERS = ("locking protocol", "not authorized", "disk i/o error")
_WAL_SIZE_LIMIT_BYTES = 64 * 1024 * 1024 # 64 MiB
# Once-per-process-per-db_label dedup sets (kanban_db.connect() runs on every kanban operation, so an undeduped
# line would repeat per connection). Tests clear these via ``hermes_state.<name>``; ``_warn_once`` resolves them there.
# line would repeat per connection). Tests clear these via ``hermes_state.<name>``; ``_log_once`` resolves them there.
_wal_fallback_warned_paths: set[str] = set()
_wal_fallback_warned_lock = threading.Lock()
_wal_reset_bug_warned_paths: set[str] = set()
@@ -45,17 +45,6 @@ _CANNOT_VERIFY_DELETE_MSG = ("could not verify journal mode before applying conf
"does not exclusively own")
def _warn_once(lock: threading.Lock, set_name: str, key: str) -> bool:
"""True the first time *key* is seen in ``hermes_state.<set_name>``."""
import hermes_state
seen = getattr(hermes_state, set_name)
with lock:
if key in seen:
return False
seen.add(key)
return True
def _mode_from_row(row) -> str:
"""Lower-cased mode from a ``PRAGMA journal_mode`` row, ``""`` if no row."""
return str(row[0]).strip().lower() if row and row[0] is not None else ""
@@ -97,10 +86,9 @@ def _apply_wal_size_limit(conn: sqlite3.Connection) -> None:
def _darwin_pragma(conn: sqlite3.Connection, pragma: str) -> None:
"""Best-effort PRAGMA on macOS only (no-op elsewhere, never raises)."""
if sys.platform != "darwin":
return
with contextlib.suppress(sqlite3.OperationalError):
conn.execute(pragma)
if sys.platform == "darwin":
with contextlib.suppress(sqlite3.OperationalError):
conn.execute(pragma)
def _apply_macos_checkpoint_barrier(conn: sqlite3.Connection) -> None:
@@ -127,8 +115,7 @@ def _apply_wal_companions(conn: sqlite3.Connection) -> None:
def is_sqlite_wal_reset_vulnerable(version_info: Optional[tuple] = None) -> bool:
"""True when the linked SQLite has the WAL-reset bug (3.7.0–3.51.2; fixed 3.51.3+, backports 3.50.7 /
3.44.6). Pre-WAL libraries are safe. https://sqlite.org/wal.html#walresetbug"""
info = version_info if version_info is not None else sqlite3.sqlite_version_info
return _is_sqlite_wal_reset_vulnerable(info)
return _is_sqlite_wal_reset_vulnerable(sqlite3.sqlite_version_info if version_info is None else version_info)
def sqlite_source_id() -> str:
@@ -156,7 +143,6 @@ def resolve_journal_mode() -> str:
durability: macOS virtiofs, NFS, SMB). Invalid values fail safe to ``wal``."""
try:
from hermes_cli.config import load_config_readonly
database = (load_config_readonly() or {}).get("database", {})
raw = database.get("journal_mode", "wal") if isinstance(database, dict) else "wal"
except Exception:
@@ -194,13 +180,11 @@ def apply_wal_with_fallback(conn: sqlite3.Connection, *, db_label: str = "state.
from hermes_state import is_sqlite_wal_reset_vulnerable, resolve_journal_mode
configured = resolve_journal_mode()
# Vulnerable SQLite: never enable WAL on non-WAL files. Configured mode is
# resolved first so an explicit DELETE request is still verified.
# Vulnerable SQLite: never enable WAL on non-WAL files (configured mode resolved first so an explicit DELETE
# request is still verified).
if is_sqlite_wal_reset_vulnerable():
return _apply_delete_for_wal_reset_bug(conn, db_label=db_label, require_delete=configured == "delete")
# Read-only probe (no flock/checkpoint/WAL-SHM unlink): WAL-init must not
# unlink files other connections hold open.
# Read-only probe (no flock/checkpoint/WAL-SHM unlink): WAL-init must not unlink files other connections hold.
current_mode = _on_disk_journal_mode(conn)
if current_mode == "wal":
if configured == "delete":
@@ -214,15 +198,13 @@ def apply_wal_with_fallback(conn: sqlite3.Connection, *, db_label: str = "state.
# Probe failed (locked/busy): ownership not provably exclusive. Fail loudly.
raise sqlite3.OperationalError(_CANNOT_VERIFY_DELETE_MSG)
return _verify_configured_delete(_set_journal_mode_no_wait(conn, "DELETE"))
return _enable_wal(conn, db_label, require_wal, current_mode)
def _enable_wal(conn: sqlite3.Connection, db_label: str, require_wal: bool, current_mode: Optional[str]) -> str:
"""Flip a non-WAL, non-vulnerable connection to WAL, or fall back to DELETE."""
# Decide BEFORE the flip whether it overwrites a mode somebody chose (probe and
# page_count are only readable while the file is untouched). A 0-page DB has
# no prior choice, and every caller reaches this before creating schema.
# Decide BEFORE the flip whether it overwrites a mode somebody chose (probe and page_count are only readable
# while the file is untouched). A 0-page DB has no prior choice, and every caller reaches this before schema.
upgrading_existing_db = current_mode is not None and current_mode != "wal" and _database_has_content(conn)
def _wal_activated() -> str:
@@ -249,11 +231,23 @@ def _enable_wal(conn: sqlite3.Connection, db_label: str, require_wal: bool, curr
if not any(marker in msg for marker in _WAL_INCOMPAT_MARKERS):
raise # unrelated OperationalError — don't silently swallow
if "disk i/o error" in msg:
activated, exc = _retry_wal_after_eio(conn, exc)
if activated:
return _wal_activated()
# Never downgrade if WAL is on disk or the mode cannot be read (probe blocked
# by a concurrent opener) — ownership is not provably exclusive either way.
# Retry twice: EIO is either deterministic WAL-incompatibility (ZFS / APFS-CoW) or a one-shot transient,
# and treating a transient as a permanent downgrade produced mixed-mode corruption (A downgrades to
# DELETE while siblings set WAL). A non-EIO retry error propagates.
for _ in range(2):
time.sleep(0.05)
try:
row = conn.execute("PRAGMA journal_mode=WAL").fetchone()
except sqlite3.OperationalError as retry_exc:
if "disk i/o error" not in str(retry_exc).lower():
raise
exc = retry_exc
continue
if _mode_from_row(row) == "wal":
return _wal_activated()
break
# Never downgrade if WAL is on disk or the mode cannot be read (probe blocked by a concurrent opener) —
# ownership is not provably exclusive either way.
if _on_disk_journal_mode(conn) in ("wal", None):
raise
if require_wal:
@@ -263,24 +257,6 @@ def _enable_wal(conn: sqlite3.Connection, db_label: str, require_wal: bool, curr
return "delete"
def _retry_wal_after_eio(conn: sqlite3.Connection, exc: sqlite3.OperationalError):
"""Retry ``journal_mode=WAL`` twice after ``disk i/o error``: EIO is either deterministic
WAL-incompatibility (ZFS / APFS-CoW) or a one-shot transient, and treating a transient as a permanent
downgrade produced mixed-mode corruption (A downgrades to DELETE while siblings set WAL). Returns
``(wal_activated, last_exc)``; a non-EIO retry error propagates."""
for _ in range(2):
time.sleep(0.05)
try:
row = conn.execute("PRAGMA journal_mode=WAL").fetchone()
except sqlite3.OperationalError as retry_exc:
if "disk i/o error" not in str(retry_exc).lower():
raise
exc = retry_exc
continue
return _mode_from_row(row) == "wal", exc
return False, exc
def _set_journal_mode_no_wait(conn: sqlite3.Connection, mode: str) -> str:
"""Execute ``PRAGMA journal_mode=<mode>`` without waiting on other openers.
@@ -343,7 +319,6 @@ def _wal_reset_repair_hint() -> str:
"""Repair hint matching what ``hermes update`` can actually do for this install type."""
try:
from hermes_cli.config import detect_install_method, get_project_root, recommended_update_command_for_method
method = detect_install_method(get_project_root())
cmd = recommended_update_command_for_method(method)
if method in {"git", "unknown"}:
@@ -353,9 +328,8 @@ def _wal_reset_repair_hint() -> str:
return "install a Python build bundled with SQLite 3.51.3+ (or backports 3.50.7 / 3.44.6) and restart Hermes"
# Once-per-(process, db_label) log table. Levels are deliberate: falling back to
# DELETE and an ignored ``journal_mode: delete`` are real losses (ERROR); a non-WAL
# -> WAL flip is normally desirable and only its invisibility was the problem (WARNING).
# Once-per-(process, db_label) log table. Levels are deliberate: falling back to DELETE and an ignored
# ``journal_mode: delete`` are real losses (ERROR); a non-WAL -> WAL flip is normally desirable (WARNING).
_WAL_RESET_BUG_ACTIONS = {
"indeterminate": ("journal mode could not be verified or exclusively switched (database is locked — possible "
"concurrent openers); leaving the journal mode untouched (no live downgrade under concurrent "
@@ -364,50 +338,45 @@ _WAL_RESET_BUG_ACTIONS = {
"delete": "using journal_mode=DELETE instead of enabling WAL",
}
_ONCE_LOGS = {
"wal_reset_bug": (
_wal_reset_bug_warned_lock, "_wal_reset_bug_warned_paths", logging.WARNING,
# Install-type-aware so the warning never promises a repair path that
# doesn't exist for git/pip/system Python installs.
"wal_reset_bug": (_wal_reset_bug_warned_lock, "_wal_reset_bug_warned_paths", logging.WARNING,
# Install-type-aware so the warning never promises a repair path that doesn't exist for git/pip installs.
"%s: linked SQLite %s (interpreter %s) is vulnerable to the WAL-reset corruption bug "
"(https://sqlite.org/wal.html#walresetbug) — %s. Upgrade to SQLite 3.51.3+ (or backports 3.50.7 / 3.44.6); "
"%s. See `hermes doctor`. This warning fires once per process per database.",
),
"journal_upgrade": (
_journal_upgrade_warned_lock, "_journal_upgrade_warned_paths", logging.WARNING,
# journal_mode is a property of the FILE: switching an existing DB to WAL rewrites its header and
# outlives the process. Operators set DELETE on the file directly (the documented WAL-reset-bug
# mitigation) and nothing told them the next open would silently put WAL back.
"%s. See `hermes doctor`. This warning fires once per process per database."),
"journal_upgrade": (_journal_upgrade_warned_lock, "_journal_upgrade_warned_paths", logging.WARNING,
# journal_mode is a property of the FILE: switching an existing DB to WAL rewrites its header and outlives
# the process. Operators set DELETE on the file (the WAL-reset-bug mitigation) and nothing told them the
# next open would silently put WAL back.
"%s: on-disk journal_mode was %s and has been switched to WAL. This rewrites the database header and "
"persists after this process exits. If %s was a deliberate choice (for example the mitigation for the SQLite "
"WAL-reset bug, or a WAL-unsafe filesystem), setting it with PRAGMA on the file will not survive -- every "
"open re-applies the configured mode. Set `database.journal_mode: delete` in config.yaml to make it stick. "
"This message fires once per process per database.",
),
"wal_fallback": (
_wal_fallback_warned_lock, "_wal_fallback_warned_paths", logging.ERROR,
"This message fires once per process per database."),
"wal_fallback": (_wal_fallback_warned_lock, "_wal_fallback_warned_paths", logging.ERROR,
# Under kanban dispatcher + workers a DELETE-mode write blocks readers as SQLITE_BUSY.
"%s: WAL journal_mode unsupported on this filesystem (%s) — falling back to journal_mode=DELETE (slower "
"rollback-journal mode; reduces concurrency but works on NFS/SMB/FUSE/ZFS). See "
"https://www.sqlite.org/wal.html for details. This message fires once per process per database.",
),
"delete_overridden": (
_delete_overridden_warned_lock, "_delete_overridden_warned_paths", logging.ERROR,
# Never-live-downgrade keeps WAL; without this the operator never learns
# that ``database.journal_mode: delete`` had no effect.
"https://www.sqlite.org/wal.html for details. This message fires once per process per database."),
"delete_overridden": (_delete_overridden_warned_lock, "_delete_overridden_warned_paths", logging.ERROR,
# Never-live-downgrade keeps WAL; without this the operator never learns their delete had no effect.
"%s: database.journal_mode=delete is configured but the on-disk database is already WAL; keeping WAL (a live "
"downgrade under open connections can corrupt the DB). To apply journal_mode=DELETE, stop all connections to "
"this DB and run a one-time offline 'PRAGMA journal_mode=DELETE' on the file. This message fires once per "
"process per database.",
),
"process per database."),
}
def _log_once(kind: str, db_label: str, *args: Any) -> None:
"""Emit ``_ONCE_LOGS[kind]`` once per (process, db_label). Callable *args* are
resolved only after the dedupe check, so install-method probes run once."""
import hermes_state
lock, set_name, level, message = _ONCE_LOGS[kind]
if _warn_once(lock, set_name, db_label):
logger.log(level, message, db_label, *(a() if callable(a) else a for a in args))
seen = getattr(hermes_state, set_name)
with lock:
if db_label in seen:
return
seen.add(db_label)
logger.log(level, message, db_label, *(a() if callable(a) else a for a in args))
def _log_wal_reset_bug_once(db_label: str, *, kept_wal: bool, indeterminate: bool = False) -> None:
@@ -426,8 +395,7 @@ _log_wal_fallback_once = functools.partial(_log_once, "wal_fallback")
_log_configured_delete_overridden_once = functools.partial(_log_once, "delete_overridden")
# Operators write synchronous as a name; mapped here rather than passed through
# so a typo becomes a warning instead of a silently different durability level.
# Operators write synchronous as a name; mapped so a typo becomes a warning, not a silently different level.
_SYNCHRONOUS_LEVELS: Dict[str, int] = {"OFF": 0, "NORMAL": 1, "FULL": 2, "EXTRA": 3}
_SYNCHRONOUS_NAMES: Dict[int, str] = {v: k for k, v in _SYNCHRONOUS_LEVELS.items()}
_SYNCHRONOUS_FULL = 2
@@ -438,8 +406,7 @@ def resolve_synchronous_level(raw_value: Any) -> Optional[int]:
case, or ``0``-``3``) to its PRAGMA integer; None for anything else so the
caller warns and leaves the level untouched (guessing at durability is worse)."""
if isinstance(raw_value, bool):
# bool is an int subclass and YAML turns bare `on`/`off` into one.
# "off" is a real durability choice; True is meaningless.
# bool is an int subclass and YAML turns bare `on`/`off` into one; "off" is a real choice, True meaningless.
return 0 if raw_value is False else None
if isinstance(raw_value, int):
return raw_value if raw_value in _SYNCHRONOUS_NAMES else None
@@ -458,8 +425,7 @@ def _apply_synchronous_pragma(conn: sqlite3.Connection, raw_value: Any, *, db_la
the platter, so an unrecognised value must not fall through to "SQLite default" the way a bad
``cache_size`` can. Darwin floor: this runs after :func:`_enforce_macos_synchronous_full`, so a configured
``NORMAL`` would silently undo the btree protection — raising is allowed, lowering is refused out loud."""
level = resolve_synchronous_level(raw_value)
if level is None:
if (level := resolve_synchronous_level(raw_value)) is None:
logger.warning("%s: ignoring unrecognized database.synchronous=%r (expected OFF, NORMAL, FULL, EXTRA, or 0-3)",
db_label, raw_value)
return
@@ -481,15 +447,12 @@ def apply_database_pragmas(conn: sqlite3.Connection, *, db_label: str = "state.d
between bundled/distro/Homebrew builds). Best-effort: failures are ignored so DB init never breaks on a
malformed section. Applied to ALL connection types: writer, read_only, WAL readers."""
try:
# Local import avoids a circular import with hermes_cli.config.
from hermes_cli.config import cfg_get, load_config_readonly
from hermes_cli.config import cfg_get, load_config_readonly # local: avoids a circular import
cfg = load_config_readonly()
except Exception:
return
for pragma_name in ("cache_size", "mmap_size", "temp_store", "wal_autocheckpoint", "journal_size_limit"):
raw_value = cfg_get(cfg, "database", pragma_name, default=None)
if raw_value is None:
if (raw_value := cfg_get(cfg, "database", pragma_name, default=None)) is None:
continue
try:
value = int(str(raw_value).strip())
@@ -499,6 +462,5 @@ def apply_database_pragmas(conn: sqlite3.Connection, *, db_label: str = "state.d
with contextlib.suppress(sqlite3.OperationalError):
conn.execute(f"PRAGMA {pragma_name}={value}")
# Last: sizing pragmas cannot change durability (see _apply_synchronous_pragma).
raw_synchronous = cfg_get(cfg, "database", "synchronous", default=None)
if raw_synchronous is not None:
if (raw_synchronous := cfg_get(cfg, "database", "synchronous", default=None)) is not None:
_apply_synchronous_pragma(conn, raw_synchronous, db_label=db_label)