"""FTS5 index setup for SessionDB: the CJK-bigram (cjk_unicode61) index DDL and tokenizer loader, per-table schema ensure with tokenizer-capability fallback, FTS-scoped corruption detection and the atomic fail-open trigger detach.""" import logging import os import sqlite3 from pathlib import Path from hermes_constants import get_hermes_home from hermes_state_common import FTS_CJK_STALE_KEY, FTS_STALE_KEY, _FTS_CJK_TRIGGERS, _FTS_TRIGGERS # caplog tests pin the "hermes_state" logger name. logger = logging.getLogger("hermes_state") # ── CJK-bigram FTS index (replaces the trigram index when available) ──── # Trigram needs >=3 chars per term, so 1-2 char CJK terms fell through to a # LIKE table scan (3-6s CPU per query on multi-GB installs). ``cjk_unicode61`` # (native/fts5_cjk/, loadable) re-emits CJK runs as overlapping bigrams; FTS5 # phrase semantics then give exact substring matching down to 2 chars. # # Same v23 discipline as the trigram table: external-content over a # tool-row-excluding view, triggers gated on a DEDICATED marker pair # (fts_cjk_rebuild_high_water / _progress) so a cjk-only backfill never gates # the complete messages_fts triggers. The table exists ONLY when the tokenizer # loads (~/.hermes/lib/libfts5_cjk.so); a process that cannot load it drops the # cjk triggers (writes keep working; the index goes stale until the next # optimize-storage on a capable host). # # Split DDL: the table/view is safe to ensure any time; triggers are created # ONLY while the index is complete-or-marker-gated. A stale index must keep its # triggers DROPPED — an external-content 'delete' for a rowid the index never # held is the canonical FTS5 corruption hazard the marker gating prevents. FTS_CJK_TABLE_SQL = """ CREATE VIEW IF NOT EXISTS messages_fts_cjk_src AS SELECT id, role, content, tool_name, tool_calls FROM messages WHERE role <> 'tool'; CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_cjk USING fts5( content, tool_name, tool_calls, content='messages_fts_cjk_src', content_rowid='id', tokenize='cjk_unicode61' ); """ FTS_CJK_TRIGGER_SQL = """ CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_insert AFTER INSERT ON messages WHEN new.role <> 'tool' AND (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_cjk_rebuild_high_water'), -1) OR new.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_cjk_rebuild_progress'), -1)) BEGIN INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) VALUES (new.id, new.content, new.tool_name, new.tool_calls); END; CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_delete AFTER DELETE ON messages WHEN old.role <> 'tool' AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_cjk_rebuild_high_water'), -1) OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_cjk_rebuild_progress'), -1)) BEGIN INSERT INTO messages_fts_cjk(messages_fts_cjk, rowid, content, tool_name, tool_calls) VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls); END; CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_update AFTER UPDATE OF content, tool_name, tool_calls, role ON messages WHEN (old.content IS NOT new.content OR old.tool_name IS NOT new.tool_name OR old.tool_calls IS NOT new.tool_calls OR old.role IS NOT new.role) AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_cjk_rebuild_high_water'), -1) OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_cjk_rebuild_progress'), -1)) BEGIN INSERT INTO messages_fts_cjk(messages_fts_cjk, rowid, content, tool_name, tool_calls) SELECT 'delete', old.id, old.content, old.tool_name, old.tool_calls WHERE old.role <> 'tool'; INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) SELECT new.id, new.content, new.tool_name, new.tool_calls WHERE new.role <> 'tool'; END; """ def fts5_cjk_so_path() -> Path: """Location of the cjk_unicode61 loadable extension.""" env = os.getenv("HERMES_FTS5_CJK_SO") if env: return Path(env).expanduser() return get_hermes_home() / "lib" / "libfts5_cjk.so" def _cjk_fts_config_enabled() -> bool: """config.yaml ``sessions.cjk_fts`` (default on), via its env bridge.""" return os.getenv("HERMES_CJK_FTS", "1").strip().lower() not in ("0", "false", "off", "no") def load_fts5_cjk_extension(conn: sqlite3.Connection) -> bool: """Best-effort load of the cjk_unicode61 tokenizer. False (never raises) when the .so is absent, ``sessions.cjk_fts`` is off, or extension loading is compiled out — callers then behave as before the cjk index existed.""" if not _cjk_fts_config_enabled(): return False path = fts5_cjk_so_path() if not path.exists(): return False try: conn.enable_load_extension(True) try: conn.load_extension(str(path)) finally: conn.enable_load_extension(False) return True except Exception: logger.warning("fts5_cjk extension load failed (%s)", path, exc_info=True) return False class SessionFtsSetupMixin: """FTS table/trigger lifecycle shared by schema init, optimize and the write path.""" @staticmethod def _is_fts5_unavailable_error(exc: sqlite3.OperationalError) -> bool: # Builds with FTS5 but without the optional trigram tokenizer raise # "no such tokenizer: trigram" instead of "no such module"; the loadable # cjk_unicode61 tokenizer shows the same capability-error shape. Scoped # to those two tokenizers so unrelated tokenizer errors aren't masked. err = str(exc).lower() return ("no such module" in err and "fts5" in err) or SessionFtsSetupMixin._is_trigram_unavailable_error(exc) @staticmethod def _is_trigram_unavailable_error(exc: sqlite3.OperationalError) -> bool: """Only an optional tokenizer is missing (trigram needs SQLite >= 3.34; cjk_unicode61 is loadable): "this one index can't be served", never "disable FTS".""" err = str(exc).lower() return ("no such tokenizer: trigram" in err or "no such tokenizer: cjk_unicode61" in err) @staticmethod def _db_has_legacy_inline_fts(cursor: sqlite3.Cursor) -> bool: """messages_fts exists in ANY pre-v23 shape. v23 is external-content over content/tool_name/tool_calls; every legacy shape (inline single-column v11..v22, or the v10-era external single-column) lacks tool_name, so "stored CREATE lacks tool_name" catches both. False when absent (fresh DB).""" row = cursor.execute( "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'messages_fts'" ).fetchone() return row is not None and "tool_name" not in (row[0] or "") def _warn_trigram_unavailable(self, exc: sqlite3.OperationalError) -> None: """Log once that the trigram tokenizer is missing; base FTS5 stays enabled.""" if getattr(self, "_trigram_unavailable_warned", False): return self._trigram_unavailable_warned = True logger.info( "SQLite trigram tokenizer unavailable for %s " "(requires SQLite >= 3.34, this build is %s); " "CJK/substring search will fall back to LIKE: %s", self.db_path, sqlite3.sqlite_version, exc, ) def _warn_fts5_unavailable(self, exc: sqlite3.OperationalError) -> None: self._fts_enabled = False if self._fts_unavailable_warned: return self._fts_unavailable_warned = True logger.warning( "SQLite FTS5 unavailable for %s; full-text session search " "disabled. Run `hermes update` to rebuild the venv with a " "current Python (managed uv guarantees FTS5). (underlying error: %s)", self.db_path, exc, ) def _ensure_fts_cjk_schema(self, cursor) -> None: """Create / repair / self-heal the CJK-bigram index (see the module comment). Sets ``_fts_cjk_available``; never raises. Loaded + absent → create (a populated DB gets the backfill markers and is NOT served until optimize-storage backfills); loaded + present → ensure triggers, honour the stale breadcrumb; NOT loaded + live triggers → drop them so INSERTs don't fail at trigger time and leave the breadcrumb.""" try: cjk_present = bool(cursor.execute( "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'messages_fts_cjk'" ).fetchone()) if not self._fts_cjk_loaded: if cjk_present: live = [r[0] for r in cursor.execute( "SELECT name FROM sqlite_master WHERE type = 'trigger' " f"AND name IN ({','.join('?' for _ in _FTS_CJK_TRIGGERS)})", _FTS_CJK_TRIGGERS, ).fetchall()] if live: # Breadcrumb FIRST (a crash between the two statements # is merely conservative), then drop. logger.warning( "messages_fts_cjk triggers present but the " "cjk_unicode61 tokenizer is unavailable (%s) — " "dropping the cjk triggers so message writes keep " "working. CJK search falls back to trigram/LIKE; " "run `hermes sessions optimize-storage` on a host " "with the extension to rebuild.", fts5_cjk_so_path(), ) cursor.execute( "INSERT INTO state_meta (key, value) VALUES (?, '1') " "ON CONFLICT(key) DO UPDATE SET value = '1'", (FTS_CJK_STALE_KEY,), ) for trig in live: cursor.execute(f"DROP TRIGGER IF EXISTS {trig}") self._fts_cjk_available = False return except sqlite3.OperationalError: logger.warning( "messages_fts_cjk presence check failed; CJK search stays on " "trigram/LIKE", exc_info=True, ) self._fts_cjk_available = False return try: cursor.executescript(FTS_CJK_TABLE_SQL) if not cjk_present: # Any old stale breadcrumb refers to a table that no longer exists. cursor.execute("DELETE FROM state_meta WHERE key = ?", (FTS_CJK_STALE_KEY,)) # Empty DB: index complete by construction (triggers cover everything), # no markers. Populated DB: the marker pair keeps the id-gated triggers # correct while old rows await optimize-storage; the index is NOT # served until that backfill completes. if cursor.execute("SELECT COUNT(*) FROM messages WHERE role <> 'tool'").fetchone()[0] > 0: hw = cursor.execute("SELECT COALESCE(MAX(id), 0) FROM messages").fetchone()[0] for k, v in ( ("fts_cjk_rebuild_high_water", str(hw)), ("fts_cjk_rebuild_progress", "0"), ): cursor.execute( "INSERT INTO state_meta (key, value) VALUES (?, ?) " "ON CONFLICT(key) DO UPDATE SET value = excluded.value", (k, v), ) if cursor.execute("SELECT 1 FROM state_meta WHERE key = ?", (FTS_CJK_STALE_KEY,)).fetchone(): # Gap of unknown extent: do NOT reinstall triggers (an # external-content 'delete' for an unindexed rowid corrupts the # index); the next optimize-storage rebuilds from scratch. self._fts_cjk_available = False return cursor.executescript(FTS_CJK_TRIGGER_SQL) backfill_pending = cursor.execute( "SELECT 1 FROM state_meta WHERE key = 'fts_cjk_rebuild_high_water' LIMIT 1" ).fetchone() self._fts_cjk_available = not backfill_pending except sqlite3.OperationalError: # incl. "no such tokenizer" after a failed registration logger.warning( "messages_fts_cjk ensure failed; CJK search stays on " "trigram/LIKE", exc_info=True, ) self._fts_cjk_available = False @staticmethod def _drop_fts_triggers(cursor: sqlite3.Cursor) -> None: for trigger in _FTS_TRIGGERS: try: cursor.execute(f"DROP TRIGGER IF EXISTS {trigger}") except sqlite3.OperationalError: pass def _ensure_fts_schema(self, cursor: sqlite3.Cursor, table_name: str, ddl: str) -> bool: status = self._fts_table_probe(cursor, table_name) if status is None: return False try: # Run even when the table exists: recreates triggers a no-FTS5 runtime dropped. cursor.executescript(ddl) return True except sqlite3.OperationalError as exc: if not self._is_fts5_unavailable_error(exc): raise # A missing tokenizer disables only that table; the base FTS5 table is fine. if self._is_trigram_unavailable_error(exc): self._warn_trigram_unavailable(exc) else: self._warn_fts5_unavailable(exc) return False @staticmethod def _is_fts_write_corruption_error(exc: sqlite3.DatabaseError) -> bool: """Corruption SQLite identifies as FTS-scoped: SQLITE_CORRUPT_VTAB, or (older builds) an ``fts5:`` message. A bare malformed-image error is structural and must not trigger live FTS maintenance.""" error_code = getattr(exc, "sqlite_errorcode", None) if error_code is not None: return error_code == getattr(sqlite3, "SQLITE_CORRUPT_VTAB", 267) msg = str(exc).lower() return msg.startswith("fts5:") and "corrupt structure" in msg def _enter_fts_fail_open(self, exc: sqlite3.DatabaseError) -> bool: """Detach corrupt FTS indexes so canonical writes can continue. Stale breadcrumb + trigger drop commit atomically: once triggers are absent the index has a gap of unknown extent, so no process may reinstall them without rebuilding every row.""" if not self._fts_enabled or not self._is_fts_write_corruption_error(exc): return False self._raise_if_db_corrupt() self._halt_if_db_generation_changed() try: with self._lock: self._conn.execute("BEGIN IMMEDIATE") try: self._conn.execute( "INSERT INTO state_meta (key, value) VALUES (?, '1') " "ON CONFLICT(key) DO UPDATE SET value = excluded.value", (FTS_STALE_KEY,), ) cjk_triggers_present = self._conn.execute( "SELECT 1 FROM sqlite_master WHERE type = 'trigger' " f"AND name IN ({','.join('?' for _ in _FTS_CJK_TRIGGERS)}) " "LIMIT 1", _FTS_CJK_TRIGGERS, ).fetchone() if cjk_triggers_present: self._conn.execute( "INSERT INTO state_meta (key, value) VALUES (?, '1') " "ON CONFLICT(key) DO UPDATE SET value = excluded.value", (FTS_CJK_STALE_KEY,), ) self._drop_all_fts_triggers(self._conn.cursor()) self._conn.commit() except BaseException: self._conn.rollback() raise except sqlite3.Error as detach_exc: logger.error( "Could not detach corrupt FTS indexes; canonical write still cannot proceed: %s", detach_exc, ) return False self._fts_stale = True self._fts_enabled = False self._trigram_available = False self._fts_cjk_available = False logger.error( "state.db FTS indexes remain corrupt (%s); disabled FTS sync and " "retrying the canonical write. Search temporarily uses LIKE until " "a later SessionDB open rebuilds the indexes.", exc, ) return True # ── Chunked FTS rebuild engine (v23 opt-in optimize) ── # One blocking rebuild held the write lock ~16 minutes on a 25 GB DB, so the # backfill runs in small chunks, each its own short transaction (resumable # from fts_rebuild_progress; concurrent runners claim chunks by CAS). # THROTTLING: a greedy loop owned the lock ~85% of the time and starved # other processes' writers; 500-row chunks plus a pause of max(MIN_PAUSE, # chunk cost x DUTY_FACTOR) cap our duty cycle unconditionally (works # cross-process, unlike any same-process activity stamp). _FTS_REBUILD_CHUNK_ROWS = 500 _FTS_REBUILD_DUTY_FACTOR = 4.0 # sleep >= 4x chunk cost (≤20% duty) _FTS_REBUILD_MIN_PAUSE = 0.2 # seconds — floor between chunks # Demoted v22 FTS shadow tables awaiting teardown: DROP of a multi-GB vtable # blocks for minutes, so the v23 migration demotes the vtable definitions # out of sqlite_master and renames the orphaned shadow tables (now plain # tables) to fts_v22_trash_*; the worker empties them in chunks, then drops. _FTS_TRASH_PREFIX = "fts_v22_trash_" def _has_fts_trash(self, conn) -> bool: """True when demoted v22 shadow tables are still awaiting teardown. Caller must hold ``self._lock`` (or pass a migration-time cursor).""" return bool(conn.execute( "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name LIKE ? ESCAPE '\\' LIMIT 1", (self._FTS_TRASH_PREFIX.replace("_", "\\_") + "%",), ).fetchone()) # FTS5 tables merged on optimize; trigram may be disabled and cjk exists only # with the loadable tokenizer, so each is probed before touching (optimize_fts). _FTS_TABLES = ("messages_fts", "messages_fts_trigram", "messages_fts_cjk")