Teknium
f13f845116
Integration layer for the cjk_unicode61 tokenizer, rebuilt on the v23 schema (the contributed integration in PR #65544 predated it): - messages_fts_cjk: external-content FTS5 over a tool-row-excluding view (same v23 storage discipline as the trigram index it supersedes — zero inline text copies). Serves EVERY CJK query shape the legacy routing split between trigram (>=3 chars/token) and LIKE full scans (1-2 char tokens). Lone 1-char CJK runs and role_filter=['tool'] queries keep their legacy routes. - Dedicated marker pair (fts_cjk_rebuild_high_water/progress) gates the id-scoped triggers, so a cjk-only backfill never gates the complete messages_fts/trigram triggers. - Transitions ride (the existing throttled/resumable chunk engine): fresh DBs are born with the index; legacy v22 DBs land on v23+cjk in one run; already-optimized v23 DBs gaining the tokenizer get a marker-gated backfill; live writes are indexed immediately in every case. - Tokenizer-loss self-heal: a process that can't load the extension drops the cjk triggers (writes keep working), leaves a stale breadcrumb, and the index is rebuilt from scratch on the next optimize run — triggers are never reinstalled over a gap (external-content 'delete' on an unindexed rowid is the FTS5 corruption hazard the marker gating exists to prevent). - Capability classification: 'no such tokenizer: cjk_unicode61' joins the degraded-runtime error class everywhere (read probe, write probe, repair) so tokenizer absence is never misclassified as corruption. - Config: sessions.cjk_fts (default on, inert without the .so) and sessions.search_slow_ms in config.yaml, bridged to env by CLI + gateway (startup + per-turn reload). build.sh falls back to vendored SQLite headers so no libsqlite3-dev is needed. Slow-query log path attribution updated: fts_cjk / fts5 / trigram / like_scan. Tests: 14 lifecycle tests (fresh/legacy/stale/backfill paths, tokenizer-loss round-trip) + 5 config-bridge tests + slow-log suite.
71 lines
2.6 KiB
Python
71 lines
2.6 KiB
Python
"""config.yaml sessions.* bridges for the search-index knobs (config-authoritative).
|
|
|
|
Salvaged from PR #65544 (adapted: agent.fts_v2_read → sessions.cjk_fts).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
import gateway.run as gateway_run
|
|
|
|
|
|
def _write_home(tmp_path: Path, sessions_cfg: dict, env_text: str = "") -> Path:
|
|
hermes_home = tmp_path / ".hermes"
|
|
hermes_home.mkdir()
|
|
(hermes_home / "config.yaml").write_text(
|
|
yaml.safe_dump({"sessions": sessions_cfg}), encoding="utf-8"
|
|
)
|
|
(hermes_home / ".env").write_text(env_text, encoding="utf-8")
|
|
return hermes_home
|
|
|
|
|
|
def test_cjk_fts_bridged_from_config(tmp_path, monkeypatch):
|
|
home = _write_home(tmp_path, {"cjk_fts": False})
|
|
monkeypatch.setattr(gateway_run, "_hermes_home", home)
|
|
monkeypatch.setenv("HERMES_CJK_FTS", "1")
|
|
gateway_run._reload_runtime_env_preserving_config_authority()
|
|
assert os.environ["HERMES_CJK_FTS"] == "False"
|
|
|
|
|
|
def test_search_slow_ms_bridged_from_config(tmp_path, monkeypatch):
|
|
home = _write_home(tmp_path, {"search_slow_ms": 250})
|
|
monkeypatch.setattr(gateway_run, "_hermes_home", home)
|
|
monkeypatch.delenv("HERMES_SEARCH_SLOW_MS", raising=False)
|
|
gateway_run._reload_runtime_env_preserving_config_authority()
|
|
assert os.environ["HERMES_SEARCH_SLOW_MS"] == "250"
|
|
|
|
|
|
def test_env_survives_when_config_omits_search_knobs(tmp_path, monkeypatch):
|
|
home = _write_home(tmp_path, {"auto_prune": False})
|
|
monkeypatch.setattr(gateway_run, "_hermes_home", home)
|
|
monkeypatch.setenv("HERMES_CJK_FTS", "0")
|
|
monkeypatch.setenv("HERMES_SEARCH_SLOW_MS", "700")
|
|
gateway_run._reload_runtime_env_preserving_config_authority()
|
|
assert os.environ["HERMES_CJK_FTS"] == "0"
|
|
assert os.environ["HERMES_SEARCH_SLOW_MS"] == "700"
|
|
|
|
|
|
def test_search_knobs_have_documented_defaults():
|
|
"""The advertised config surface must exist in DEFAULT_CONFIG (no
|
|
user-facing env switch): cjk index default ON, slow-search log at 1s."""
|
|
from hermes_cli.config import DEFAULT_CONFIG
|
|
|
|
assert DEFAULT_CONFIG["sessions"]["cjk_fts"] is True
|
|
assert DEFAULT_CONFIG["sessions"]["search_slow_ms"] == 1000
|
|
|
|
|
|
def test_config_false_disables_cjk_semantics(tmp_path, monkeypatch):
|
|
"""The bridged 'False' string must parse as OFF in hermes_state."""
|
|
from hermes_state import _cjk_fts_config_enabled
|
|
|
|
monkeypatch.setenv("HERMES_CJK_FTS", "False")
|
|
assert not _cjk_fts_config_enabled()
|
|
monkeypatch.setenv("HERMES_CJK_FTS", "True")
|
|
assert _cjk_fts_config_enabled()
|
|
monkeypatch.delenv("HERMES_CJK_FTS", raising=False)
|
|
assert _cjk_fts_config_enabled() # default on
|