diff --git a/hermes_state_common.py b/hermes_state_common.py index d482862808..95914e704a 100644 --- a/hermes_state_common.py +++ b/hermes_state_common.py @@ -354,7 +354,7 @@ def _sql_session_last_active_by_id(session_id_expr: str) -> str: ) -SCHEMA_VERSION = 28 +SCHEMA_VERSION = 29 # FTS storage-layout version, tracked INDEPENDENTLY of SCHEMA_VERSION in the @@ -772,20 +772,18 @@ END; # matching. The trigram tokenizer creates overlapping 3-byte sequences so # substring queries work natively for any script (CJK, Thai, etc.). # -# The trigram index is the most expensive index in state.db (~2.6x the size -# of the text it covers), and ``role='tool'`` rows are ~90% of message bytes -# while being almost entirely machine noise (base64 payloads, file dumps, -# delegation transcripts). The index therefore reads through -# ``messages_fts_trigram_src``, a view that excludes tool rows — they stay -# fully stored in ``messages`` and fully searchable via the standard -# ``messages_fts`` index; they just don't get trigram (CJK substring) -# treatment. ``search_messages`` routes CJK queries that filter on -# ``role='tool'`` to the LIKE fallback for the same reason. +# The trigram index is the most expensive index in state.db, and tool output +# plus cron transcripts are overwhelmingly machine-generated text. The index +# therefore reads through ``messages_fts_trigram_src``, a view that excludes +# both classes. They stay fully stored in ``messages`` and searchable via the +# standard ``messages_fts`` index; they just don't get trigram treatment. +# ``search_messages`` routes explicit tool/cron CJK searches to LIKE. FTS_TRIGRAM_SQL = """ CREATE VIEW IF NOT EXISTS messages_fts_trigram_src AS - SELECT id, role, content, tool_name, tool_calls - FROM messages - WHERE role <> 'tool'; + SELECT m.id, m.role, m.content, m.tool_name, m.tool_calls + FROM messages AS m + JOIN sessions AS s ON s.id = m.session_id + WHERE m.role <> 'tool' AND s.source <> 'cron'; CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5( content, @@ -798,6 +796,8 @@ CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5( CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_insert AFTER INSERT ON messages WHEN new.role <> 'tool' + AND EXISTS (SELECT 1 FROM sessions + WHERE id = new.session_id AND source <> 'cron') AND (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_rebuild_high_water'), -1) OR new.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta @@ -809,6 +809,8 @@ END; CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_delete AFTER DELETE ON messages WHEN old.role <> 'tool' + AND EXISTS (SELECT 1 FROM sessions + WHERE id = old.session_id AND source <> 'cron') AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_rebuild_high_water'), -1) OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta @@ -831,10 +833,14 @@ WHEN (old.content IS NOT new.content BEGIN INSERT INTO messages_fts_trigram(messages_fts_trigram, rowid, content, tool_name, tool_calls) SELECT 'delete', old.id, old.content, old.tool_name, old.tool_calls - WHERE old.role <> 'tool'; + WHERE old.role <> 'tool' + AND EXISTS (SELECT 1 FROM sessions + WHERE id = old.session_id AND source <> 'cron'); INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) SELECT new.id, new.content, new.tool_name, new.tool_calls - WHERE new.role <> 'tool'; + WHERE new.role <> 'tool' + AND EXISTS (SELECT 1 FROM sessions + WHERE id = new.session_id AND source <> 'cron'); END; """ diff --git a/hermes_state_schema.py b/hermes_state_schema.py index de6c383334..7cbe089fb9 100644 --- a/hermes_state_schema.py +++ b/hermes_state_schema.py @@ -414,6 +414,44 @@ class SessionSchemaMixin: exc_info=True, ) + def _migrate_trigram_cron_exclusion(self, cursor: sqlite3.Cursor) -> bool: + """Install the cron-filtered trigram view and purge historical rows. + + Legacy inline indexes remain opt-in: their content is private to the + virtual table and cannot adopt this external-content view. For an + external layout, replacing the view and triggers is cheap, but the + existing inverted index still contains cron rows until FTS5 rebuilds + from the new view. Run that rebuild under the shared cross-process + admission gate used by every startup FTS repair. + """ + if self._db_has_legacy_inline_fts(cursor): + return True + trigram_exists = self._fts_table_probe(cursor, "messages_fts_trigram") + if trigram_exists is not True: + # Let the normal ensure path create/backfill a missing optional + # trigram table. ``None`` means this runtime cannot safely inspect + # an existing one, so leave the schema version behind for retry. + return trigram_exists is False + for name in _FTS_TRIGRAM_TRIGGERS: + cursor.execute(f"DROP TRIGGER IF EXISTS {name}") + cursor.execute("DROP VIEW IF EXISTS messages_fts_trigram_src") + if not self._ensure_fts_schema( + cursor, "messages_fts_trigram", FTS_TRIGRAM_SQL + ): + return False + # Always rebuild while schema_version is behind, even if the view + # already has the new predicate. A process can die after replacing the + # view but before rebuilding/stamping; view text alone cannot prove the + # old cron postings were purged. + self._run_admitted_startup_rebuild( + cursor, + lambda: cursor.execute( + "INSERT INTO messages_fts_trigram(messages_fts_trigram) " + "VALUES('rebuild')" + ), + ) + return True + @staticmethod def _rebuild_fts_indexes( @@ -1490,6 +1528,15 @@ class SessionSchemaMixin: # rows, but clear migrated rows so future writes do not keep # one large prompt copy per session. self._dedupe_legacy_system_prompts(cursor) + if current_version < 29 and fts5_available: + # v29 (was v27 in the original PR; main had already reached + # v28 with column-reconciliation bumps, so a `< 27` gate would + # never fire on existing installs): cron sessions remain canonical and stay in the standard + # word index, but no longer inflate the trigram substring index. + # Rebuild once so rows indexed by older trigger/view definitions + # do not survive indefinitely as stale matches and disk usage. + if not self._migrate_trigram_cron_exclusion(cursor): + fts_migrations_complete = False # The FTS storage layout is versioned independently of the main # schema (see the v23 note above). Stamp the current layout so the diff --git a/hermes_state_search.py b/hermes_state_search.py index 7c17b49ae4..c487eee09d 100644 --- a/hermes_state_search.py +++ b/hermes_state_search.py @@ -167,8 +167,9 @@ class SessionSearchMixin: conn.execute( "INSERT INTO messages_fts_trigram(rowid, content, tool_name, tool_calls) " "SELECT m.id, m.content, m.tool_name, m.tool_calls " - "FROM messages m " + "FROM messages m JOIN sessions s ON s.id = m.session_id " "WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' " + "AND s.source <> 'cron' " "AND NOT EXISTS (SELECT 1 FROM messages_fts_trigram_docsize d WHERE d.id = m.id)", (lo, hi), ) @@ -325,8 +326,10 @@ class SessionSearchMixin: conn.execute( "INSERT INTO messages_fts_trigram" "(rowid, content, tool_name, tool_calls) " - "SELECT id, content, tool_name, tool_calls FROM messages " - "WHERE id > ? AND id <= ? AND role <> 'tool'", + "SELECT m.id, m.content, m.tool_name, m.tool_calls " + "FROM messages m JOIN sessions s ON s.id = m.session_id " + "WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' " + "AND s.source <> 'cron'", (progress, upper), ) # Publish progress in the same transaction as the rows it @@ -1905,6 +1908,7 @@ class SessionSearchMixin: # query explicitly filtering on role='tool' must therefore use # the LIKE fallback, which scans the base table directly. _wants_tool_rows = bool(role_filter) and "tool" in role_filter + _wants_cron_rows = bool(source_filter) and "cron" in source_filter # ── CJK-bigram route (messages_fts_cjk, cjk_unicode61) ────── # When the bigram index is available it serves EVERY CJK query @@ -1919,6 +1923,7 @@ class SessionSearchMixin: if ( self._fts_cjk_available and not _wants_tool_rows + and not _wants_cron_rows and not self._has_lone_cjk_run(raw_query) ): tokens = raw_query.split() @@ -1992,6 +1997,7 @@ class SessionSearchMixin: and not _any_short_cjk and self._trigram_available and not _wants_tool_rows + and not _wants_cron_rows ): # Trigram FTS5 path — quote each non-operator token to handle # FTS5 special chars (%, *, etc.) while preserving boolean diff --git a/tests/state/test_fts_trigram_cron_exclusion.py b/tests/state/test_fts_trigram_cron_exclusion.py new file mode 100644 index 0000000000..14f85ce2f8 --- /dev/null +++ b/tests/state/test_fts_trigram_cron_exclusion.py @@ -0,0 +1,171 @@ +"""Cron-source exclusion from the external-content trigram FTS index.""" + +from __future__ import annotations + +import sqlite3 + +import pytest + +from hermes_state import FTS_TRIGRAM_SQL, SCHEMA_VERSION, SessionDB + + +@pytest.fixture +def db(tmp_path): + session_db = SessionDB(db_path=tmp_path / "state.db") + if not session_db._trigram_available: + session_db.close() + pytest.skip("trigram tokenizer unavailable in this SQLite build") + yield session_db + session_db.close() + + +def _trigram_rowids(db: SessionDB) -> set[int]: + return { + row[0] + for row in db._conn.execute( + "SELECT id FROM messages_fts_trigram_docsize ORDER BY id" + ).fetchall() + } + + +def _install_pre_v27_trigram(db: SessionDB) -> None: + db._conn.executescript( + """ + DROP TRIGGER messages_fts_trigram_insert; + DROP TRIGGER messages_fts_trigram_delete; + DROP TRIGGER messages_fts_trigram_update; + DROP VIEW messages_fts_trigram_src; + CREATE VIEW messages_fts_trigram_src AS + SELECT id, role, content, tool_name, tool_calls + FROM messages WHERE role <> 'tool'; + CREATE TRIGGER messages_fts_trigram_insert AFTER INSERT ON messages + WHEN new.role <> 'tool' + BEGIN + INSERT INTO messages_fts_trigram( + rowid, content, tool_name, tool_calls + ) VALUES (new.id, new.content, new.tool_name, new.tool_calls); + END; + """ + ) + + +def test_fresh_trigram_indexes_conversations_but_not_cron(db: SessionDB): + db.create_session("cli", source="cli") + db.create_session("cron", source="cron") + cli_id = db.append_message("cli", role="user", content="交付状态正常") + cron_id = db.append_message("cron", role="user", content="定时任务状态正常") + + assert _trigram_rowids(db) == {cli_id} + assert cron_id not in _trigram_rowids(db) + assert db._conn.execute( + "SELECT id FROM messages_fts_docsize WHERE id = ?", (cron_id,) + ).fetchone() is not None + + +def test_cron_remains_searchable_via_standard_fts_and_explicit_cjk_fallback( + db: SessionDB, +): + db.create_session("cron", source="cron") + db.append_message( + "cron", role="assistant", content="quarterly archive 大别山项目 complete" + ) + + assert [row["session_id"] for row in db.search_messages("quarterly")] == [ + "cron" + ] + assert [ + row["session_id"] + for row in db.search_messages("大别山项目", source_filter=["cron"]) + ] == ["cron"] + + +def test_deferred_rebuild_does_not_reintroduce_cron(db: SessionDB): + db.create_session("cli", source="cli") + db.create_session("cron", source="cron") + cli_id = db.append_message("cli", role="assistant", content="交互会话内容") + db.append_message("cron", role="assistant", content="定时会话内容") + + with db._lock: + db._reset_fts_index_to_empty(db._conn) + db._seed_fts_rebuild_markers(db._conn, force=True) + db._conn.commit() + while db.fts_rebuild_step(): + pass + + assert _trigram_rowids(db) == {cli_id} + + +def test_existing_external_layout_rebuilds_trigram_on_upgrade(tmp_path): + db_path = tmp_path / "state.db" + old = SessionDB(db_path=db_path) + if not old._trigram_available: + old.close() + pytest.skip("trigram tokenizer unavailable in this SQLite build") + # The virtual table keeps referring to the view by name, so this recreates + # the exact old external-content boundary without reading source text. + _install_pre_v27_trigram(old) + old.create_session("cli", source="cli") + old.create_session("cron", source="cron") + cli_id = old.append_message("cli", role="user", content="交互迁移内容") + cron_id = old.append_message("cron", role="user", content="定时迁移内容") + assert _trigram_rowids(old) == {cli_id, cron_id} + old._conn.execute("UPDATE schema_version SET version = ?", (SCHEMA_VERSION - 1,)) + old._conn.commit() + old.close() + + migrated = SessionDB(db_path=db_path) + try: + assert _trigram_rowids(migrated) == {cli_id} + view_sql = migrated._conn.execute( + "SELECT sql FROM sqlite_master " + "WHERE type = 'view' AND name = 'messages_fts_trigram_src'" + ).fetchone()[0] + assert "sessions" in view_sql + assert "cron" in view_sql + migrated._conn.execute( + "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('integrity-check')" + ) + finally: + migrated.close() + + +def test_partial_upgrade_view_does_not_skip_historical_rebuild(tmp_path): + db_path = tmp_path / "state.db" + old = SessionDB(db_path=db_path) + if not old._trigram_available: + old.close() + pytest.skip("trigram tokenizer unavailable in this SQLite build") + _install_pre_v27_trigram(old) + old.create_session("cron", source="cron") + cron_id = old.append_message("cron", role="assistant", content="迁移中断内容") + assert _trigram_rowids(old) == {cron_id} + + # Simulate a crash after new DDL landed but before the rebuild/schema stamp. + for name in ( + "messages_fts_trigram_insert", + "messages_fts_trigram_delete", + "messages_fts_trigram_update", + ): + old._conn.execute(f"DROP TRIGGER IF EXISTS {name}") + old._conn.execute("DROP VIEW messages_fts_trigram_src") + old._conn.executescript(FTS_TRIGRAM_SQL) + old._conn.execute("UPDATE schema_version SET version = ?", (SCHEMA_VERSION - 1,)) + old._conn.commit() + old.close() + + migrated = SessionDB(db_path=db_path) + try: + assert _trigram_rowids(migrated) == set() + finally: + migrated.close() + + +def test_delete_of_unindexed_cron_row_keeps_trigram_consistent(db: SessionDB): + db.create_session("cron", source="cron") + cron_id = db.append_message("cron", role="user", content="不会进入索引") + assert cron_id not in _trigram_rowids(db) + + db._conn.execute("DELETE FROM messages WHERE id = ?", (cron_id,)) + db._conn.execute( + "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('integrity-check')" + )