fix(search): strip % from non-CJK FTS5 queries

Closes the residual the contributor's own triage comment flagged: % was
excluded from the special-char class to protect the CJK LIKE fallback,
but a non-CJK query never reaches that fallback (is_cjk gates it), so
'50%' still hit MATCH raw and silently returned zero results. Strip %
whenever the sanitized query contains no CJK; the CJK path keeps its
pre-existing contract. Regression tests for both directions.
This commit is contained in:
Teknium
2026-08-08 18:45:11 -07:00
parent c595dcb955
commit 90311ee75f
2 changed files with 24 additions and 0 deletions
+8
View File
@@ -1230,6 +1230,14 @@ class SessionSearchMixin:
# ``50%`` all raised before the class was completed.
sanitized = _FTS5_SPECIAL_RE.sub(" ", sanitized)
# Step 2b: ``%`` is excluded from the class above only to protect the
# CJK LIKE-fallback path (LIKE treats % as a wildcard the fallback
# builds itself). A non-CJK query never reaches that fallback
# (``is_cjk`` gates it), so ``50%`` would sail into MATCH raw and
# raise like the rest. Strip it whenever the query has no CJK.
if "%" in sanitized and not SessionSearchMixin._contains_cjk(sanitized):
sanitized = sanitized.replace("%", " ")
# Step 3: Collapse repeated * (e.g. "***") into a single one,
# and remove leading * (prefix-only needs at least one char before *)
sanitized = re.sub(r"\*+", "*", sanitized)
+16
View File
@@ -4570,3 +4570,19 @@ class TestFts5SanitizerCharacterClass:
"SELECT count(*) FROM t WHERE t MATCH ?", (sanitized,)
).fetchone()
assert rows[0] == 1
def test_percent_stripped_for_non_cjk_query(self):
# % is kept only for the CJK LIKE fallback; a non-CJK query never
# reaches that fallback, so % must be stripped before MATCH.
conn = self._fts_table()
sanitized = self._sanitize("50%")
assert "%" not in sanitized
conn.execute(
"SELECT count(*) FROM t WHERE t MATCH ?", (sanitized,)
).fetchone()
def test_percent_preserved_for_cjk_query(self):
# The CJK LIKE fallback builds its own pattern from the sanitized
# text; keep % intact there (pre-existing contract).
sanitized = self._sanitize("完成50%")
assert "%" in sanitized