fix(state): widen surrogate scrub to remaining raw-str bind sites

Follow-up to the cherry-picked fix: the same UnicodeEncodeError bind
failure was live at sibling sites the PR didn't cover —

- api_content sidecar (append_message, _insert_message_rows,
  set_latest_user_api_content): bound raw; a surrogate in the composed
  api_content aborted the whole row/UPDATE. Scrubbing is wire-accurate:
  the conversation loop already scrubs every outgoing payload, so the
  scrubbed form IS what was sent.
- tool_name (both INSERT sites): raw bind.
- sanitize_title: session titles from LLM title generation or /title
  could carry surrogates; scrub before validation.

E2E-verified each site raised UnicodeEncodeError on main and persists
after this commit. Tests added for all five paths.
This commit is contained in:
Kshitij Kapoor
2026-07-19 13:28:28 +05:30
committed by kshitij
parent 81d4619707
commit 653a95f9fa
2 changed files with 43 additions and 6 deletions
+13 -6
View File
@@ -3304,6 +3304,10 @@ class SessionDB:
if not title:
return None
# Lone surrogates cannot be bound by sqlite3 (UnicodeEncodeError at
# UTF-8 encode time) — scrub them like every other write path here.
title = _sanitize_surrogates(title)
# Remove ASCII control characters (0x00-0x1F, 0x7F) but keep
# whitespace chars (\t=0x09, \n=0x0A, \r=0x0D) so they can be
# normalized to spaces by the whitespace collapsing step below
@@ -4181,7 +4185,10 @@ class SessionDB:
``api_content`` is the exact content string sent to the API for this
message when it differs from ``content`` (ephemeral memory/plugin
injections, persist overrides). It is a byte-fidelity sidecar for
prompt-cache-stable replay — stored verbatim, never sanitized.
prompt-cache-stable replay — stored as sent, except lone surrogates
(which sqlite3 cannot bind and which the conversation loop scrubs
from every outgoing payload anyway, so the scrubbed form IS the
wire bytes).
"""
# Serialize structured fields to JSON before entering the write txn
reasoning_details_json = (
@@ -4229,7 +4236,7 @@ class SessionDB:
stored_content,
tool_call_id,
tool_calls_json,
tool_name,
_scrub_surrogates(tool_name),
effect_disposition,
message_timestamp,
token_count,
@@ -4242,7 +4249,7 @@ class SessionDB:
platform_message_id,
1 if observed else 0,
1,
api_content if isinstance(api_content, str) else None,
_scrub_surrogates(api_content) if isinstance(api_content, str) else None,
),
)
msg_id = cursor.lastrowid
@@ -4325,7 +4332,7 @@ class SessionDB:
self._encode_content(msg.get("content")),
msg.get("tool_call_id"),
tool_calls_json,
msg.get("tool_name"),
_scrub_surrogates(msg.get("tool_name")),
msg.get("effect_disposition"),
message_timestamp,
msg.get("token_count"),
@@ -4338,7 +4345,7 @@ class SessionDB:
platform_msg_id,
1 if msg.get("observed") else 0,
1,
api_content if isinstance(api_content, str) else None,
_scrub_surrogates(api_content) if isinstance(api_content, str) else None,
),
)
inserted += 1
@@ -4489,7 +4496,7 @@ class SessionDB:
"WHERE session_id = ? AND role = 'user' AND active = 1 "
"ORDER BY id DESC LIMIT 1"
") AND content IS ?",
(api_content, session_id, encoded),
(_scrub_surrogates(api_content), session_id, encoded),
)
return cursor.rowcount
+30
View File
@@ -6278,3 +6278,33 @@ class TestLoneSurrogatePersistence:
benign = "Ünïcödé ok — 日本語 🎉 emoji fine"
db.append_message("s1", "assistant", benign)
assert db.get_messages("s1")[0]["content"] == benign
# -- sibling raw-str bind sites (follow-up widening of the same bug class)
def test_append_message_survives_lone_surrogate_api_content(self, db):
db.create_session("s1", source="cli")
db.append_message("s1", "user", "clean", api_content=self.DIRTY)
assert db.get_messages("s1")[0]["api_content"] == "scraped \ufffd price"
def test_append_message_survives_lone_surrogate_tool_name(self, db):
db.create_session("s1", source="cli")
db.append_message("s1", "tool", "ok", tool_name="web\ud835search")
assert len(db.get_messages("s1")) == 1
def test_replace_messages_survives_lone_surrogate_api_content(self, db):
db.create_session("s1", source="cli")
db.replace_messages(
"s1", [{"role": "user", "content": "u1", "api_content": self.DIRTY}]
)
assert db.get_messages("s1")[0]["api_content"] == "scraped \ufffd price"
def test_set_latest_user_api_content_survives_lone_surrogate(self, db):
db.create_session("s1", source="cli")
db.append_message("s1", "user", "turn text")
assert db.set_latest_user_api_content("s1", "turn text", self.DIRTY) == 1
def test_session_title_survives_lone_surrogate(self, db):
db.create_session("s1", source="cli")
assert db.set_session_title("s1", "title \ud835 bad") is True
assert db.get_session("s1")["title"] == "title \ufffd bad"