perf(state): batch message hydration in export_all

This commit is contained in:
Xipong
2026-09-09 00:28:09 +03:00
committed by Teknium
parent 16e4496d90
commit 78f85112d9
2 changed files with 57 additions and 1 deletions
+16 -1
View File
@@ -258,7 +258,22 @@ class SessionPortabilityMixin:
def export_all(self, source: str = None) -> List[Dict[str, Any]]:
"""Export all sessions (with messages) as dicts, e.g. for JSONL backup."""
return [self._with_messages(s) for s in self.search_sessions(source=source, limit=100000)]
sessions = self.search_sessions(source=source, limit=100000)
messages_by_session = {session["id"]: [] for session in sessions}
session_ids = list(messages_by_session)
# Stay below SQLite's legacy 999-variable limit while replacing the per-session N+1 reads.
for start in range(0, len(session_ids), 900):
chunk = session_ids[start:start + 900]
rows = self._read_all(
f"SELECT * FROM messages WHERE session_id IN ({','.join('?' for _ in chunk)}) "
"AND active = 1 ORDER BY session_id, id",
chunk,
)
for row in rows:
messages_by_session[row["session_id"]].append(
self._row_to_message_dict(row, warn_context="get_messages", summary_flag=True)
)
return [{**session, "messages": messages_by_session[session["id"]]} for session in sessions]
def adopt_session_lineage_from(self, donor_db: Any, session_id: str, *, retire_donor: bool = True) -> Dict[str, Any]:
"""Adopt *session_id*'s full compression lineage from *donor_db* (stranded-bot-session