perf(state): keep delegate-child transcripts out of the trigram FTS index (schema v30)

On a fan-out-heavy install state.db reached 3.4 GB; 70% of message bytes
belonged to subagent sessions, and every one of those rows was also
indexed into messages_fts_trigram, whose shadow tables are ~2.6x the
text they cover (1,029 MB trigram vs 350 MB standard FTS on that DB).
session_search already hides source='subagent' sessions, so the
substring/CJK index bought nothing for them.

Extend the v29 cron exclusion: the messages_fts_trigram_src view, the
three sync triggers, and both deferred-backfill INSERT...SELECTs now use
one shared predicate (FTS_TRIGRAM_SESSION_SQL / fts_trigram_session_sql)
that skips sessions with source IN ('cron','subagent') or the
$._delegate_from creation marker (children spawned under a gateway turn
inherit the gateway's source). Compression/branch continuations carry
parent_session_id without the marker and stay indexed. Child rows remain
canonical in `messages` and fully indexed in the standard messages_fts
word index; explicit source_filter=['subagent'] CJK searches route to
LIKE like cron already did.

The v29 migration gate becomes `< 30` and reuses the same view-swap +
admitted rebuild, so existing installs purge historical child postings
once on open. Fresh DB with 2,000 x 2 KB child messages: 22.4 MB ->
12.5 MB (trigram shadow 10.09 MB -> 0.02 MB).

hermes_state_common.py

@@ -354,7 +354,7 @@ def _sql_session_last_active_by_id(session_id_expr: str) -> str:
     )
 
 
-SCHEMA_VERSION = 29
+SCHEMA_VERSION = 30
 
 
 # FTS storage-layout version, tracked INDEPENDENTLY of SCHEMA_VERSION in the
@@ -782,12 +782,40 @@ END;
 # LIKE for the same reason. Structured ``tool_calls`` JSON likewise stays
 # searchable through ``messages_fts``; excluding it here avoids indexing
 # repetitive JSON syntax as trigrams (FTS_STORAGE_VERSION 2).
-FTS_TRIGRAM_SQL = """
+#
+# Delegate-child (subagent) transcripts are excluded the same way (v30):
+# on a fan-out-heavy install they were ~70% of all message bytes and
+# ``session_search`` hides ``source='subagent'`` sessions anyway. A child
+# is recognised by its source OR by the ``_delegate_from`` creation marker
+# (children spawned under a gateway turn inherit the gateway's source).
+# Compression/branch continuations of interactive sessions also carry
+# ``parent_session_id`` but NOT the marker, so they stay trigram-indexed.
+FTS_TRIGRAM_EXCLUDED_SOURCES = ("cron", "subagent")
+
+# Predicate over a ``sessions`` row (unqualified column names) selecting
+# sessions whose rows belong in the trigram index. Shared by the view, the
+# sync triggers, and the deferred-backfill INSERT ... SELECTs so they can
+# never disagree about the index boundary.
+FTS_TRIGRAM_SESSION_SQL = (
+    "source NOT IN ("
+    + ", ".join(f"'{src}'" for src in FTS_TRIGRAM_EXCLUDED_SOURCES)
+    + ") AND json_extract(COALESCE(model_config, '{}'), '$._delegate_from') IS NULL"
+)
+
+
+def fts_trigram_session_sql(alias: str) -> str:
+    """``FTS_TRIGRAM_SESSION_SQL`` with every column qualified by ``alias``."""
+    return FTS_TRIGRAM_SESSION_SQL.replace("source ", f"{alias}.source ").replace(
+        "COALESCE(model_config", f"COALESCE({alias}.model_config"
+    )
+
+
+FTS_TRIGRAM_SQL = f"""
 CREATE VIEW IF NOT EXISTS messages_fts_trigram_src AS
     SELECT m.id, m.role, m.content, m.tool_name
     FROM messages AS m
     JOIN sessions AS s ON s.id = m.session_id
-    WHERE m.role <> 'tool' AND s.source <> 'cron';
+    WHERE m.role <> 'tool' AND {fts_trigram_session_sql('s')};
 
 CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5(
     content,
@@ -800,7 +828,7 @@ CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_trigram USING fts5(
 CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_insert AFTER INSERT ON messages
 WHEN new.role <> 'tool'
    AND EXISTS (SELECT 1 FROM sessions
-               WHERE id = new.session_id AND source <> 'cron')
+               WHERE id = new.session_id AND {FTS_TRIGRAM_SESSION_SQL})
    AND (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
                            WHERE key = 'fts_rebuild_high_water'), -1)
      OR new.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
@@ -813,7 +841,7 @@ END;
 CREATE TRIGGER IF NOT EXISTS messages_fts_trigram_delete AFTER DELETE ON messages
 WHEN old.role <> 'tool'
    AND EXISTS (SELECT 1 FROM sessions
-               WHERE id = old.session_id AND source <> 'cron')
+               WHERE id = old.session_id AND {FTS_TRIGRAM_SESSION_SQL})
    AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
                            WHERE key = 'fts_rebuild_high_water'), -1)
      OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
@@ -837,12 +865,12 @@ BEGIN
     SELECT 'delete', old.id, old.content, old.tool_name
     WHERE old.role <> 'tool'
       AND EXISTS (SELECT 1 FROM sessions
-                  WHERE id = old.session_id AND source <> 'cron');
+                  WHERE id = old.session_id AND {FTS_TRIGRAM_SESSION_SQL});
     INSERT INTO messages_fts_trigram(rowid, content, tool_name)
     SELECT new.id, new.content, new.tool_name
     WHERE new.role <> 'tool'
       AND EXISTS (SELECT 1 FROM sessions
-                  WHERE id = new.session_id AND source <> 'cron');
+                  WHERE id = new.session_id AND {FTS_TRIGRAM_SESSION_SQL});
 END;
 """
 

hermes_state_schema.py

@@ -415,7 +415,10 @@ class SessionSchemaMixin:
             )
 
     def _migrate_trigram_cron_exclusion(self, cursor: sqlite3.Cursor) -> bool:
-        """Install the cron-filtered trigram view and purge historical rows.
+        """Install the source-filtered trigram view and purge historical rows.
+
+        Covers the v29 cron exclusion and the v30 subagent exclusion — both
+        only change the view/trigger predicate and rebuild from it.
 
         Legacy inline indexes remain opt-in: their content is private to the
         virtual table and cannot adopt this external-content view. For an
@@ -1550,11 +1553,11 @@ class SessionSchemaMixin:
                 # rows, but clear migrated rows so future writes do not keep
                 # one large prompt copy per session.
                 self._dedupe_legacy_system_prompts(cursor)
-            if current_version < 29 and fts5_available:
-                # v29 (was v27 in the original PR; main had already reached
-                # v28 with column-reconciliation bumps, so a `< 27` gate would
-                # never fire on existing installs): cron sessions remain canonical and stay in the standard
+            if current_version < 30 and fts5_available:
+                # v29: cron sessions remain canonical and stay in the standard
                 # word index, but no longer inflate the trigram substring index.
+                # v30: delegate-child (subagent) transcripts get the same
+                # treatment (FTS_TRIGRAM_EXCLUDED_SOURCES + _delegate_from).
                 # Rebuild once so rows indexed by older trigger/view definitions
                 # do not survive indefinitely as stale matches and disk usage.
                 if not self._migrate_trigram_cron_exclusion(cursor):

hermes_state_search.py

@@ -24,7 +24,9 @@ from hermes_state_common import (
     FTS_STORAGE_VERSION,
     FTS_TOOL_CONTENT_PREFIX_CHARS,
     FTS_TOOL_FULL_CONTENT_HIGH_WATER_KEY,
+    FTS_TRIGRAM_EXCLUDED_SOURCES,
     FTS_TRIGRAM_SQL,
+    fts_trigram_session_sql,
     MAX_FTS5_QUERY_CHARS,
     SCHEMA_VERSION,
     _FTS_CJK_TRIGGERS,
@@ -169,7 +171,7 @@ class SessionSearchMixin:
                         "SELECT m.id, m.content, m.tool_name "
                         "FROM messages m JOIN sessions s ON s.id = m.session_id "
                         "WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' "
-                        "AND s.source <> 'cron' "
+                        f"AND {fts_trigram_session_sql('s')} "
                         "AND NOT EXISTS (SELECT 1 FROM messages_fts_trigram_docsize d WHERE d.id = m.id)",
                         (lo, hi),
                     )
@@ -329,7 +331,7 @@ class SessionSearchMixin:
                     "SELECT m.id, m.content, m.tool_name "
                     "FROM messages m JOIN sessions s ON s.id = m.session_id "
                     "WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' "
-                    "AND s.source <> 'cron'",
+                    f"AND {fts_trigram_session_sql('s')}",
                     (progress, upper),
                 )
             # Publish progress in the same transaction as the rows it
@@ -1922,7 +1924,12 @@ class SessionSearchMixin:
             # query explicitly filtering on role='tool' must therefore use
             # the LIKE fallback, which scans the base table directly.
             _wants_tool_rows = bool(role_filter) and "tool" in role_filter
-            _wants_cron_rows = bool(source_filter) and "cron" in source_filter
+            # Cron and subagent transcripts are excluded too (see
+            # FTS_TRIGRAM_EXCLUDED_SOURCES); an explicit filter for them
+            # must likewise scan the base table.
+            _wants_cron_rows = bool(source_filter) and any(
+                src in FTS_TRIGRAM_EXCLUDED_SOURCES for src in source_filter
+            )
 
             # ── CJK-bigram route (messages_fts_cjk, cjk_unicode61) ──────
             # When the bigram index is available it serves EVERY CJK query

tests/state/test_fts_trigram_subagent_exclusion.py

@@ -0,0 +1,161 @@
+"""Delegate-child (subagent) transcripts stay out of the trigram FTS index (v30).
+
+Mirrors ``test_fts_trigram_cron_exclusion.py``: children are canonical rows
+in ``messages`` and stay searchable through the standard ``messages_fts``
+word index; only the trigram (CJK substring) shadow index skips them.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from hermes_state import SCHEMA_VERSION, SessionDB
+from hermes_state_common import FTS_TRIGRAM_EXCLUDED_SOURCES, fts_trigram_session_sql
+
+
+@pytest.fixture
+def db(tmp_path):
+    session_db = SessionDB(db_path=tmp_path / "state.db")
+    if not session_db._trigram_available:
+        session_db.close()
+        pytest.skip("trigram tokenizer unavailable in this SQLite build")
+    yield session_db
+    session_db.close()
+
+
+def _trigram_rowids(db: SessionDB) -> set[int]:
+    return {
+        row[0]
+        for row in db._conn.execute("SELECT id FROM messages_fts_trigram_docsize").fetchall()
+    }
+
+
+def _fts_rowids(db: SessionDB) -> set[int]:
+    return {
+        row[0] for row in db._conn.execute("SELECT id FROM messages_fts_docsize").fetchall()
+    }
+
+
+def _seed(db: SessionDB) -> dict[str, int]:
+    db.create_session("root", source="cli")
+    # delegate_tool children: source='subagent' via platform, plus the
+    # _delegate_from creation marker.
+    db.create_session(
+        "kid", source="subagent", parent_session_id="root",
+        model_config={"_delegate_from": "root"},
+    )
+    # A child spawned under a gateway turn inherits the gateway's source but
+    # still carries the marker.
+    db.create_session(
+        "gw-kid", source="telegram", parent_session_id="root",
+        model_config={"_delegate_from": "root"},
+    )
+    # Compression continuation: parent_session_id but NO marker -> indexed.
+    db.create_session("cont", source="cli", parent_session_id="root")
+    return {
+        "root": db.append_message("root", role="user", content="交付状态正常 root-word"),
+        "kid": db.append_message("kid", role="assistant", content="子任务状态正常 kid-word"),
+        "gw-kid": db.append_message("gw-kid", role="assistant", content="网关子任务 gwkid-word"),
+        "cont": db.append_message("cont", role="assistant", content="继续会话内容 cont-word"),
+    }
+
+
+def test_subagent_rows_skip_trigram_but_stay_in_standard_fts(db: SessionDB):
+    ids = _seed(db)
+    assert _trigram_rowids(db) == {ids["root"], ids["cont"]}
+    assert _fts_rowids(db) >= set(ids.values())
+
+
+def test_subagent_rows_remain_word_searchable(db: SessionDB):
+    _seed(db)
+    assert [r["session_id"] for r in db.search_messages("kid-word")] == ["kid"]
+    assert [r["session_id"] for r in db.search_messages("gwkid-word")] == ["gw-kid"]
+    # Explicit CJK search scoped to the excluded source falls back to LIKE.
+    assert [
+        r["session_id"]
+        for r in db.search_messages("子任务状态", source_filter=["subagent"])
+    ] == ["kid"]
+    # Top-level CJK substring search unaffected.
+    assert [r["session_id"] for r in db.search_messages("交付状态")] == ["root"]
+
+
+def test_update_and_delete_of_unindexed_child_row_keep_trigram_consistent(db: SessionDB):
+    ids = _seed(db)
+    db._conn.execute(
+        "UPDATE messages SET content = ? WHERE id = ?", ("改写后的内容", ids["kid"])
+    )
+    db._conn.execute("DELETE FROM messages WHERE id = ?", (ids["kid"],))
+    db._conn.execute(
+        "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('integrity-check')"
+    )
+    assert _trigram_rowids(db) == {ids["root"], ids["cont"]}
+
+
+def test_deferred_rebuild_does_not_reintroduce_children(db: SessionDB):
+    ids = _seed(db)
+    with db._lock:
+        db._reset_fts_index_to_empty(db._conn)
+        db._seed_fts_rebuild_markers(db._conn, force=True)
+        db._conn.commit()
+    while db.fts_rebuild_step():
+        pass
+    assert _trigram_rowids(db) == {ids["root"], ids["cont"]}
+    assert _fts_rowids(db) >= set(ids.values())
+
+
+def test_full_rebuild_honours_exclusion(db: SessionDB):
+    ids = _seed(db)
+    db.rebuild_fts()
+    assert _trigram_rowids(db) == {ids["root"], ids["cont"]}
+
+
+def test_v29_install_purges_child_rows_on_upgrade(tmp_path):
+    db_path = tmp_path / "state.db"
+    old = SessionDB(db_path=db_path)
+    if not old._trigram_available:
+        old.close()
+        pytest.skip("trigram tokenizer unavailable in this SQLite build")
+    # Recreate the v29 (cron-only) view/trigger boundary.
+    old._conn.executescript(
+        """
+        DROP TRIGGER messages_fts_trigram_insert;
+        DROP TRIGGER messages_fts_trigram_delete;
+        DROP TRIGGER messages_fts_trigram_update;
+        DROP VIEW messages_fts_trigram_src;
+        CREATE VIEW messages_fts_trigram_src AS
+            SELECT m.id, m.role, m.content, m.tool_name
+            FROM messages AS m JOIN sessions AS s ON s.id = m.session_id
+            WHERE m.role <> 'tool' AND s.source <> 'cron';
+        CREATE TRIGGER messages_fts_trigram_insert AFTER INSERT ON messages
+        WHEN new.role <> 'tool'
+           AND EXISTS (SELECT 1 FROM sessions WHERE id = new.session_id AND source <> 'cron')
+        BEGIN
+            INSERT INTO messages_fts_trigram(rowid, content, tool_name)
+            VALUES (new.id, new.content, new.tool_name);
+        END;
+        """
+    )
+    ids = _seed(old)
+    assert _trigram_rowids(old) == set(ids.values())
+    old._conn.execute("UPDATE schema_version SET version = 29")
+    old._conn.commit()
+    old.close()
+
+    migrated = SessionDB(db_path=db_path)
+    try:
+        assert _trigram_rowids(migrated) == {ids["root"], ids["cont"]}
+        assert migrated._conn.execute(
+            "SELECT version FROM schema_version"
+        ).fetchone()[0] == SCHEMA_VERSION
+        migrated._conn.execute(
+            "INSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('integrity-check')"
+        )
+    finally:
+        migrated.close()
+
+
+def test_predicate_constants_agree():
+    assert "subagent" in FTS_TRIGRAM_EXCLUDED_SOURCES
+    assert "cron" in FTS_TRIGRAM_EXCLUDED_SOURCES
+    sql = fts_trigram_session_sql("s")
+    assert sql.startswith("s.source NOT IN (") and "s.model_config" in sql

website/docs/developer-guide/session-storage.md

@@ -169,6 +169,8 @@ The `schema_version` table stores a single integer. Simple column additions are
 | 20 | Per-model usage attribution — seed `session_model_usage` rows from historical per-session aggregate totals |
 | 22 | Task-dimension usage attribution — rebuild `session_model_usage` so the `task` column participates in the PRIMARY KEY |
 | 23 | FTS storage redesign — external-content FTS tables replacing the v11 inline-mode copies (opt-in transition for existing DBs) |
+| 29 | Cron sessions leave the trigram (substring/CJK) index; `messages_fts_trigram_src` view + triggers filter on `sessions.source`, one-time rebuild purges historical rows |
+| 30 | Delegate-child (subagent) sessions leave the trigram index too — `source='subagent'` or the `$._delegate_from` marker (`FTS_TRIGRAM_SESSION_SQL`). Rows stay in `messages` and the standard `messages_fts` word index, so `session_search` still finds them; only the ~2.6× trigram shadow tables shrink. Same one-time rebuild as v29 |
 
 Versions not listed above were declarative column additions handled by `_reconcile_columns()` (version bump only, no data migration).