fix(search): strip % from non-CJK FTS5 queries
Closes the residual the contributor's own triage comment flagged: % was excluded from the special-char class to protect the CJK LIKE fallback, but a non-CJK query never reaches that fallback (is_cjk gates it), so '50%' still hit MATCH raw and silently returned zero results. Strip % whenever the sanitized query contains no CJK; the CJK path keeps its pre-existing contract. Regression tests for both directions.
This commit is contained in:
@@ -1230,6 +1230,14 @@ class SessionSearchMixin:
|
||||
# ``50%`` all raised before the class was completed.
|
||||
sanitized = _FTS5_SPECIAL_RE.sub(" ", sanitized)
|
||||
|
||||
# Step 2b: ``%`` is excluded from the class above only to protect the
|
||||
# CJK LIKE-fallback path (LIKE treats % as a wildcard the fallback
|
||||
# builds itself). A non-CJK query never reaches that fallback
|
||||
# (``is_cjk`` gates it), so ``50%`` would sail into MATCH raw and
|
||||
# raise like the rest. Strip it whenever the query has no CJK.
|
||||
if "%" in sanitized and not SessionSearchMixin._contains_cjk(sanitized):
|
||||
sanitized = sanitized.replace("%", " ")
|
||||
|
||||
# Step 3: Collapse repeated * (e.g. "***") into a single one,
|
||||
# and remove leading * (prefix-only needs at least one char before *)
|
||||
sanitized = re.sub(r"\*+", "*", sanitized)
|
||||
|
||||
@@ -4570,3 +4570,19 @@ class TestFts5SanitizerCharacterClass:
|
||||
"SELECT count(*) FROM t WHERE t MATCH ?", (sanitized,)
|
||||
).fetchone()
|
||||
assert rows[0] == 1
|
||||
|
||||
def test_percent_stripped_for_non_cjk_query(self):
|
||||
# % is kept only for the CJK LIKE fallback; a non-CJK query never
|
||||
# reaches that fallback, so % must be stripped before MATCH.
|
||||
conn = self._fts_table()
|
||||
sanitized = self._sanitize("50%")
|
||||
assert "%" not in sanitized
|
||||
conn.execute(
|
||||
"SELECT count(*) FROM t WHERE t MATCH ?", (sanitized,)
|
||||
).fetchone()
|
||||
|
||||
def test_percent_preserved_for_cjk_query(self):
|
||||
# The CJK LIKE fallback builds its own pattern from the sanitized
|
||||
# text; keep % intact there (pre-existing contract).
|
||||
sanitized = self._sanitize("完成50%")
|
||||
assert "%" in sanitized
|
||||
|
||||
Reference in New Issue
Block a user