diff --git a/hermes_state_common.py b/hermes_state_common.py index 9b04fa5ba3..eec87c62bf 100644 --- a/hermes_state_common.py +++ b/hermes_state_common.py @@ -354,7 +354,7 @@ def _sql_session_last_active_by_id(session_id_expr: str) -> str: ) -SCHEMA_VERSION = 29 +SCHEMA_VERSION = 30 # FTS storage-layout version, tracked INDEPENDENTLY of SCHEMA_VERSION in the @@ -782,12 +782,40 @@ END; # LIKE for the same reason. Structured ``tool_calls`` JSON likewise stays # searchable through ``messages_fts``; excluding it here avoids indexing # repetitive JSON syntax as trigrams (FTS_STORAGE_VERSION 2). -FTS_TRIGRAM_SQL = """ +# +# Delegate-child (subagent) transcripts are excluded the same way (v30): +# on a fan-out-heavy install they were ~70% of all message bytes and +# ``session_search`` hides ``source='subagent'`` sessions anyway. A child +# is recognised by its source OR by the ``_delegate_from`` creation marker +# (children spawned under a gateway turn inherit the gateway's source). +# Compression/branch continuations of interactive sessions also carry +# ``parent_session_id`` but NOT the marker, so they stay trigram-indexed. +FTS_TRIGRAM_EXCLUDED_SOURCES = ("cron", "subagent") + +# Predicate over a ``sessions`` row (unqualified column names) selecting +# sessions whose rows belong in the trigram index. Shared by the view, the +# sync triggers, and the deferred-backfill INSERT ... SELECTs so they can +# never disagree about the index boundary. +FTS_TRIGRAM_SESSION_SQL = ( + "source NOT IN (" + + ", ".join(f"'{src}'" for src in FTS_TRIGRAM_EXCLUDED_SOURCES) + + ") AND json_extract(COALESCE(model_config, '{}'), '$._delegate_from') IS NULL" +) + + +def fts_trigram_session_sql(alias: str) -> str: + """``FTS_TRIGRAM_SESSION_SQL`` with every column qualified by ``alias``.""" + return FTS_TRIGRAM_SESSION_SQL.replace("source ", f"{alias}.source ").replace( + "COALESCE(model_config", f"COALESCE({alias}.model_config" + ) + + +FTS_TRIGRAM_SQL = f""" CREATE VIEW IF NOT EXISTS messages_fts_trigram_src AS SELECT m.id, m.role, m.content, m.tool_name FROM messages AS m JOIN sessions AS s ON s.id = m.session_id - WHERE m.role <> 'tool' AND s.source <> 'cron'; + WHERE m.role <> 'tool' AND {fts_trigram_session_sql('s')}; CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5( content, @@ -800,7 +828,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5( CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_insert AFTER INSERT ON messages WHEN new.role <> 'tool' AND EXISTS (SELECT 1 FROM sessions - WHERE id = new.session_id AND source <> 'cron') + WHERE id = new.session_id AND {FTS_TRIGRAM_SESSION_SQL}) AND (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_rebuild_high_water'), -1) OR new.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta @@ -813,7 +841,7 @@ END; CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_delete AFTER DELETE ON messages WHEN old.role <> 'tool' AND EXISTS (SELECT 1 FROM sessions - WHERE id = old.session_id AND source <> 'cron') + WHERE id = old.session_id AND {FTS_TRIGRAM_SESSION_SQL}) AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_rebuild_high_water'), -1) OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta @@ -837,12 +865,12 @@ BEGIN SELECT 'delete', old.id, old.content, old.tool_name WHERE old.role <> 'tool' AND EXISTS (SELECT 1 FROM sessions - WHERE id = old.session_id AND source <> 'cron'); + WHERE id = old.session_id AND {FTS_TRIGRAM_SESSION_SQL}); INSERT INTO messages_fts_trigram(rowid, content, tool_name) SELECT new.id, new.content, new.tool_name WHERE new.role <> 'tool' AND EXISTS (SELECT 1 FROM sessions - WHERE id = new.session_id AND source <> 'cron'); + WHERE id = new.session_id AND {FTS_TRIGRAM_SESSION_SQL}); END; """ diff --git a/hermes_state_schema.py b/hermes_state_schema.py index 38d7ba3cd0..7896134dc2 100644 --- a/hermes_state_schema.py +++ b/hermes_state_schema.py @@ -415,7 +415,10 @@ class SessionSchemaMixin: ) def _migrate_trigram_cron_exclusion(self, cursor: sqlite3.Cursor) -> bool: - """Install the cron-filtered trigram view and purge historical rows. + """Install the source-filtered trigram view and purge historical rows. + + Covers the v29 cron exclusion and the v30 subagent exclusion — both + only change the view/trigger predicate and rebuild from it. Legacy inline indexes remain opt-in: their content is private to the virtual table and cannot adopt this external-content view. For an @@ -1550,11 +1553,11 @@ class SessionSchemaMixin: # rows, but clear migrated rows so future writes do not keep # one large prompt copy per session. self._dedupe_legacy_system_prompts(cursor) - if current_version < 29 and fts5_available: - # v29 (was v27 in the original PR; main had already reached - # v28 with column-reconciliation bumps, so a `< 27` gate would - # never fire on existing installs): cron sessions remain canonical and stay in the standard + if current_version < 30 and fts5_available: + # v29: cron sessions remain canonical and stay in the standard # word index, but no longer inflate the trigram substring index. + # v30: delegate-child (subagent) transcripts get the same + # treatment (FTS_TRIGRAM_EXCLUDED_SOURCES + _delegate_from). # Rebuild once so rows indexed by older trigger/view definitions # do not survive indefinitely as stale matches and disk usage. if not self._migrate_trigram_cron_exclusion(cursor): diff --git a/hermes_state_search.py b/hermes_state_search.py index ae1ce11397..91e95045e7 100644 --- a/hermes_state_search.py +++ b/hermes_state_search.py @@ -24,7 +24,9 @@ from hermes_state_common import ( FTS_STORAGE_VERSION, FTS_TOOL_CONTENT_PREFIX_CHARS, FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY, + FTS_TRIGRAM_EXCLUDED_SOURCES, FTS_TRIGRAM_SQL, + fts_trigram_session_sql, MAX_FTS5_QUERY_CHARS, SCHEMA_VERSION, _FTS_CJK_TRIGGERS, @@ -169,7 +171,7 @@ class SessionSearchMixin: "SELECT m.id, m.content, m.tool_name " "FROM messages m JOIN sessions s ON s.id = m.session_id " "WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' " - "AND s.source <> 'cron' " + f"AND {fts_trigram_session_sql('s')} " "AND NOT EXISTS (SELECT 1 FROM messages_fts_trigram_docsize d WHERE d.id = m.id)", (lo, hi), ) @@ -329,7 +331,7 @@ class SessionSearchMixin: "SELECT m.id, m.content, m.tool_name " "FROM messages m JOIN sessions s ON s.id = m.session_id " "WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' " - "AND s.source <> 'cron'", + f"AND {fts_trigram_session_sql('s')}", (progress, upper), ) # Publish progress in the same transaction as the rows it @@ -1922,7 +1924,12 @@ class SessionSearchMixin: # query explicitly filtering on role='tool' must therefore use # the LIKE fallback, which scans the base table directly. _wants_tool_rows = bool(role_filter) and "tool" in role_filter - _wants_cron_rows = bool(source_filter) and "cron" in source_filter + # Cron and subagent transcripts are excluded too (see + # FTS_TRIGRAM_EXCLUDED_SOURCES); an explicit filter for them + # must likewise scan the base table. + _wants_cron_rows = bool(source_filter) and any( + src in FTS_TRIGRAM_EXCLUDED_SOURCES for src in source_filter + ) # ── CJK-bigram route (messages_fts_cjk, cjk_unicode61) ────── # When the bigram index is available it serves EVERY CJK query diff --git a/tests/state/test_fts_trigram_subagent_exclusion.py b/tests/state/test_fts_trigram_subagent_exclusion.py new file mode 100644 index 0000000000..00d1a63fa6 --- /dev/null +++ b/tests/state/test_fts_trigram_subagent_exclusion.py @@ -0,0 +1,161 @@ +"""Delegate-child (subagent) transcripts stay out of the trigram FTS index (v30). + +Mirrors ``test_fts_trigram_cron_exclusion.py``: children are canonical rows +in ``messages`` and stay searchable through the standard ``messages_fts`` +word index; only the trigram (CJK substring) shadow index skips them. +""" + +from __future__ import annotations + +import pytest + +from hermes_state import SCHEMA_VERSION, SessionDB +from hermes_state_common import FTS_TRIGRAM_EXCLUDED_SOURCES, fts_trigram_session_sql + + +@pytest.fixture +def db(tmp_path): + session_db = SessionDB(db_path=tmp_path / "state.db") + if not session_db._trigram_available: + session_db.close() + pytest.skip("trigram tokenizer unavailable in this SQLite build") + yield session_db + session_db.close() + + +def _trigram_rowids(db: SessionDB) -> set[int]: + return { + row[0] + for row in db._conn.execute("SELECT id FROM messages_fts_trigram_docsize").fetchall() + } + + +def _fts_rowids(db: SessionDB) -> set[int]: + return { + row[0] for row in db._conn.execute("SELECT id FROM messages_fts_docsize").fetchall() + } + + +def _seed(db: SessionDB) -> dict[str, int]: + db.create_session("root", source="cli") + # delegate_tool children: source='subagent' via platform, plus the + # _delegate_from creation marker. + db.create_session( + "kid", source="subagent", parent_session_id="root", + model_config={"_delegate_from": "root"}, + ) + # A child spawned under a gateway turn inherits the gateway's source but + # still carries the marker. + db.create_session( + "gw-kid", source="telegram", parent_session_id="root", + model_config={"_delegate_from": "root"}, + ) + # Compression continuation: parent_session_id but NO marker -> indexed. + db.create_session("cont", source="cli", parent_session_id="root") + return { + "root": db.append_message("root", role="user", content="交付状态正常 root-word"), + "kid": db.append_message("kid", role="assistant", content="子任务状态正常 kid-word"), + "gw-kid": db.append_message("gw-kid", role="assistant", content="网关子任务 gwkid-word"), + "cont": db.append_message("cont", role="assistant", content="继续会话内容 cont-word"), + } + + +def test_subagent_rows_skip_trigram_but_stay_in_standard_fts(db: SessionDB): + ids = _seed(db) + assert _trigram_rowids(db) == {ids["root"], ids["cont"]} + assert _fts_rowids(db) >= set(ids.values()) + + +def test_subagent_rows_remain_word_searchable(db: SessionDB): + _seed(db) + assert [r["session_id"] for r in db.search_messages("kid-word")] == ["kid"] + assert [r["session_id"] for r in db.search_messages("gwkid-word")] == ["gw-kid"] + # Explicit CJK search scoped to the excluded source falls back to LIKE. + assert [ + r["session_id"] + for r in db.search_messages("子任务状态", source_filter=["subagent"]) + ] == ["kid"] + # Top-level CJK substring search unaffected. + assert [r["session_id"] for r in db.search_messages("交付状态")] == ["root"] + + +def test_update_and_delete_of_unindexed_child_row_keep_trigram_consistent(db: SessionDB): + ids = _seed(db) + db._conn.execute( + "UPDATE messages SET content = ? WHERE id = ?", ("改写后的内容", ids["kid"]) + ) + db._conn.execute("DELETE FROM messages WHERE id = ?", (ids["kid"],)) + db._conn.execute( + "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('integrity-check')" + ) + assert _trigram_rowids(db) == {ids["root"], ids["cont"]} + + +def test_deferred_rebuild_does_not_reintroduce_children(db: SessionDB): + ids = _seed(db) + with db._lock: + db._reset_fts_index_to_empty(db._conn) + db._seed_fts_rebuild_markers(db._conn, force=True) + db._conn.commit() + while db.fts_rebuild_step(): + pass + assert _trigram_rowids(db) == {ids["root"], ids["cont"]} + assert _fts_rowids(db) >= set(ids.values()) + + +def test_full_rebuild_honours_exclusion(db: SessionDB): + ids = _seed(db) + db.rebuild_fts() + assert _trigram_rowids(db) == {ids["root"], ids["cont"]} + + +def test_v29_install_purges_child_rows_on_upgrade(tmp_path): + db_path = tmp_path / "state.db" + old = SessionDB(db_path=db_path) + if not old._trigram_available: + old.close() + pytest.skip("trigram tokenizer unavailable in this SQLite build") + # Recreate the v29 (cron-only) view/trigger boundary. + old._conn.executescript( + """ + DROP TRIGGER messages_fts_trigram_insert; + DROP TRIGGER messages_fts_trigram_delete; + DROP TRIGGER messages_fts_trigram_update; + DROP VIEW messages_fts_trigram_src; + CREATE VIEW messages_fts_trigram_src AS + SELECT m.id, m.role, m.content, m.tool_name + FROM messages AS m JOIN sessions AS s ON s.id = m.session_id + WHERE m.role <> 'tool' AND s.source <> 'cron'; + CREATE TRIGGER messages_fts_trigram_insert AFTER INSERT ON messages + WHEN new.role <> 'tool' + AND EXISTS (SELECT 1 FROM sessions WHERE id = new.session_id AND source <> 'cron') + BEGIN + INSERT INTO messages_fts_trigram(rowid, content, tool_name) + VALUES (new.id, new.content, new.tool_name); + END; + """ + ) + ids = _seed(old) + assert _trigram_rowids(old) == set(ids.values()) + old._conn.execute("UPDATE schema_version SET version = 29") + old._conn.commit() + old.close() + + migrated = SessionDB(db_path=db_path) + try: + assert _trigram_rowids(migrated) == {ids["root"], ids["cont"]} + assert migrated._conn.execute( + "SELECT version FROM schema_version" + ).fetchone()[0] == SCHEMA_VERSION + migrated._conn.execute( + "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('integrity-check')" + ) + finally: + migrated.close() + + +def test_predicate_constants_agree(): + assert "subagent" in FTS_TRIGRAM_EXCLUDED_SOURCES + assert "cron" in FTS_TRIGRAM_EXCLUDED_SOURCES + sql = fts_trigram_session_sql("s") + assert sql.startswith("s.source NOT IN (") and "s.model_config" in sql diff --git a/website/docs/developer-guide/session-storage.md b/website/docs/developer-guide/session-storage.md index 0f80accfde..4627b0ffa6 100644 --- a/website/docs/developer-guide/session-storage.md +++ b/website/docs/developer-guide/session-storage.md @@ -169,6 +169,8 @@ The `schema_version` table stores a single integer. Simple column additions are | 20 | Per-model usage attribution — seed `session_model_usage` rows from historical per-session aggregate totals | | 22 | Task-dimension usage attribution — rebuild `session_model_usage` so the `task` column participates in the PRIMARY KEY | | 23 | FTS storage redesign — external-content FTS tables replacing the v11 inline-mode copies (opt-in transition for existing DBs) | +| 29 | Cron sessions leave the trigram (substring/CJK) index; `messages_fts_trigram_src` view + triggers filter on `sessions.source`, one-time rebuild purges historical rows | +| 30 | Delegate-child (subagent) sessions leave the trigram index too — `source='subagent'` or the `$._delegate_from` marker (`FTS_TRIGRAM_SESSION_SQL`). Rows stay in `messages` and the standard `messages_fts` word index, so `session_search` still finds them; only the ~2.6× trigram shadow tables shrink. Same one-time rebuild as v29 | Versions not listed above were declarative column additions handled by `_reconcile_columns()` (version bump only, no data migration).