feat(state): messages_fts_cjk — CJK-bigram index on the v23 external-content layout
Integration layer for the cjk_unicode61 tokenizer, rebuilt on the v23 schema (the contributed integration in PR #65544 predated it): - messages_fts_cjk: external-content FTS5 over a tool-row-excluding view (same v23 storage discipline as the trigram index it supersedes — zero inline text copies). Serves EVERY CJK query shape the legacy routing split between trigram (>=3 chars/token) and LIKE full scans (1-2 char tokens). Lone 1-char CJK runs and role_filter=['tool'] queries keep their legacy routes. - Dedicated marker pair (fts_cjk_rebuild_high_water/progress) gates the id-scoped triggers, so a cjk-only backfill never gates the complete messages_fts/trigram triggers. - Transitions ride (the existing throttled/resumable chunk engine): fresh DBs are born with the index; legacy v22 DBs land on v23+cjk in one run; already-optimized v23 DBs gaining the tokenizer get a marker-gated backfill; live writes are indexed immediately in every case. - Tokenizer-loss self-heal: a process that can't load the extension drops the cjk triggers (writes keep working), leaves a stale breadcrumb, and the index is rebuilt from scratch on the next optimize run — triggers are never reinstalled over a gap (external-content 'delete' on an unindexed rowid is the FTS5 corruption hazard the marker gating exists to prevent). - Capability classification: 'no such tokenizer: cjk_unicode61' joins the degraded-runtime error class everywhere (read probe, write probe, repair) so tokenizer absence is never misclassified as corruption. - Config: sessions.cjk_fts (default on, inert without the .so) and sessions.search_slow_ms in config.yaml, bridged to env by CLI + gateway (startup + per-turn reload). build.sh falls back to vendored SQLite headers so no libsqlite3-dev is needed. Slow-query log path attribution updated: fts_cjk / fts5 / trigram / like_scan. Tests: 14 lifecycle tests (fresh/legacy/stale/backfill paths, tokenizer-loss round-trip) + 5 config-bridge tests + slow-log suite.
This commit is contained in:
Teknium
@@ -169,3 +169,4 @@ apps/desktop/demo/
|
||||
# PR body is the archive. See the hermes-agent-dev skill's
|
||||
# pr-infographic-workflow reference (storage rule + lapse #8 / #COMMIT-1).
|
||||
infographic/
|
||||
native/fts5_cjk/*.so
|
||||
|
||||
@@ -772,6 +772,16 @@ def load_cli_config() -> Dict[str, Any]:
|
||||
if redact is not None:
|
||||
os.environ["HERMES_REDACT_SECRETS"] = str(redact).lower()
|
||||
|
||||
# Session-search index knobs (hermes_state reads the env carriers).
|
||||
sessions_config = defaults.get("sessions", {})
|
||||
if isinstance(sessions_config, dict):
|
||||
if "cjk_fts" in sessions_config:
|
||||
os.environ["HERMES_CJK_FTS"] = str(sessions_config["cjk_fts"])
|
||||
if "search_slow_ms" in sessions_config:
|
||||
os.environ["HERMES_SEARCH_SLOW_MS"] = str(
|
||||
sessions_config["search_slow_ms"]
|
||||
)
|
||||
|
||||
return defaults
|
||||
|
||||
# Load configuration at module startup
|
||||
|
||||
@@ -1481,6 +1481,14 @@ def _bridge_max_turns_from_config(home: "Path") -> None:
|
||||
agent_cfg = cfg.get("agent", {})
|
||||
if isinstance(agent_cfg, dict) and "max_turns" in agent_cfg:
|
||||
os.environ["HERMES_MAX_ITERATIONS"] = str(agent_cfg["max_turns"])
|
||||
# config-authoritative knobs for the session-search index (config.yaml
|
||||
# sessions.* wins over stale env; env stays the cross-process carrier).
|
||||
sessions_cfg = cfg.get("sessions", {})
|
||||
if isinstance(sessions_cfg, dict):
|
||||
if "cjk_fts" in sessions_cfg:
|
||||
os.environ["HERMES_CJK_FTS"] = str(sessions_cfg["cjk_fts"])
|
||||
if "search_slow_ms" in sessions_cfg:
|
||||
os.environ["HERMES_SEARCH_SLOW_MS"] = str(sessions_cfg["search_slow_ms"])
|
||||
|
||||
|
||||
def _current_max_iterations() -> int:
|
||||
@@ -1763,6 +1771,16 @@ if _config_path.exists():
|
||||
os.environ["HERMES_AUTO_CONTINUE_FRESHNESS"] = str(
|
||||
_agent_cfg["gateway_auto_continue_freshness"]
|
||||
)
|
||||
# config-authoritative knobs for the session-search index; same
|
||||
# bridge semantics as the agent settings above.
|
||||
_sessions_cfg = _cfg.get("sessions", {})
|
||||
if _sessions_cfg and isinstance(_sessions_cfg, dict):
|
||||
if "cjk_fts" in _sessions_cfg:
|
||||
os.environ["HERMES_CJK_FTS"] = str(_sessions_cfg["cjk_fts"])
|
||||
if "search_slow_ms" in _sessions_cfg:
|
||||
os.environ["HERMES_SEARCH_SLOW_MS"] = str(
|
||||
_sessions_cfg["search_slow_ms"]
|
||||
)
|
||||
_display_cfg = _cfg.get("display", {})
|
||||
if _display_cfg and isinstance(_display_cfg, dict):
|
||||
if "busy_input_mode" in _display_cfg:
|
||||
|
||||
@@ -3249,6 +3249,20 @@ DEFAULT_CONFIG = {
|
||||
# enforcement is a copy/gating change, not new migration code.
|
||||
# "off": suppress the notice entirely.
|
||||
"fts_optimize_notice": "advise",
|
||||
# CJK-bigram search index (messages_fts_cjk, cjk_unicode61 loadable
|
||||
# tokenizer). When the extension is built (native/fts5_cjk/build.sh →
|
||||
# ~/.hermes/lib/libfts5_cjk.so), 1-2 char CJK terms (일본, 项目, ...)
|
||||
# get index-speed exact matching instead of LIKE full-table scans.
|
||||
# True (default): use the index when the extension is present; the
|
||||
# setting is inert when it isn't. False: never load the extension or
|
||||
# serve the cjk index. Bridged to HERMES_CJK_FTS (internal carrier).
|
||||
"cjk_fts": True,
|
||||
# Slow session-search log threshold in milliseconds: searches at or
|
||||
# above it log one INFO line with the routing path taken (fts_cjk /
|
||||
# fts5 / trigram / like_scan) so latency regressions stay
|
||||
# attributable per query shape. 0 logs every search. Bridged to
|
||||
# HERMES_SEARCH_SLOW_MS (internal carrier).
|
||||
"search_slow_ms": 1000,
|
||||
},
|
||||
|
||||
# Contextual first-touch onboarding hints (see agent/onboarding.py).
|
||||
|
||||
@@ -6406,6 +6406,10 @@ def _print_fts_optimize_available_notice() -> None:
|
||||
"SELECT 1 FROM sqlite_master WHERE type = 'table' "
|
||||
"AND name LIKE 'fts\\_v22\\_trash\\_%' ESCAPE '\\' LIMIT 1"
|
||||
).fetchone()
|
||||
or db._conn.execute(
|
||||
"SELECT 1 FROM state_meta WHERE key IN "
|
||||
"('fts_cjk_rebuild_high_water', 'fts_cjk_stale') LIMIT 1"
|
||||
).fetchone()
|
||||
)
|
||||
except Exception:
|
||||
return
|
||||
|
||||
+597
-13
@@ -17,6 +17,7 @@ Key design decisions:
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import sqlite3
|
||||
@@ -582,6 +583,13 @@ def _db_opens_cleanly(db_path: Path) -> Optional[str]:
|
||||
"""
|
||||
conn = sqlite3.connect(str(db_path), isolation_level=None)
|
||||
try:
|
||||
# Best-effort tokenizer load: a DB carrying the messages_fts_cjk
|
||||
# index needs the cjk_unicode61 extension before any statement can
|
||||
# touch that table — including the trigger-driven write probe below.
|
||||
# Without it, this probe sees the DB exactly as a tokenizer-less
|
||||
# SessionDB open would (which drops the cjk triggers to keep writes
|
||||
# working), so tokenizer absence must never classify as corruption.
|
||||
load_fts5_cjk_extension(conn)
|
||||
conn.execute("PRAGMA journal_mode").fetchone()
|
||||
rows = conn.execute("PRAGMA integrity_check").fetchall()
|
||||
problems = [str(r[0]) for r in rows if r and str(r[0]).lower() != "ok"]
|
||||
@@ -603,7 +611,7 @@ def _db_opens_cleanly(db_path: Path) -> Optional[str]:
|
||||
# Catch the full sqlite3 exception hierarchy (not just
|
||||
# OperationalError) so the malformed-shadow-table class is reported
|
||||
# rather than letting it crash the caller.
|
||||
for fts_table in ("messages_fts", "messages_fts_trigram"):
|
||||
for fts_table in ("messages_fts", "messages_fts_trigram", "messages_fts_cjk"):
|
||||
try:
|
||||
# No-op queries against the actual FTS5 APIs the search
|
||||
# tools use. The trigram table is included because it backs
|
||||
@@ -674,6 +682,12 @@ def _db_opens_cleanly(db_path: Path) -> Optional[str]:
|
||||
msg = str(exc).lower()
|
||||
if "no such table" in msg or "no such column" in msg:
|
||||
return None
|
||||
if "no such tokenizer: cjk_unicode61" in msg:
|
||||
# This probe process couldn't load the cjk extension while
|
||||
# the DB carries the cjk index — capability gap, not
|
||||
# corruption. A tokenizer-capable SessionDB serves it fine;
|
||||
# a tokenizer-less one self-heals by dropping the triggers.
|
||||
return None
|
||||
return str(exc)
|
||||
return None
|
||||
except sqlite3.DatabaseError as exc:
|
||||
@@ -737,13 +751,19 @@ def repair_state_db_schema(db_path: Path, *, backup: bool = True) -> Dict[str, A
|
||||
try:
|
||||
conn = sqlite3.connect(str(db_path), isolation_level=None)
|
||||
try:
|
||||
for table_name in ("messages_fts", "messages_fts_trigram"):
|
||||
# The cjk index can only be rebuilt with its tokenizer loaded;
|
||||
# best-effort (a tokenizer-less host skips it at the probe below).
|
||||
load_fts5_cjk_extension(conn)
|
||||
for table_name in (
|
||||
"messages_fts", "messages_fts_trigram", "messages_fts_cjk"
|
||||
):
|
||||
try:
|
||||
conn.execute(
|
||||
f"INSERT INTO {table_name}({table_name}) VALUES('rebuild')"
|
||||
)
|
||||
except sqlite3.OperationalError:
|
||||
# Table absent (FTS disabled / trigram off) — skip it.
|
||||
# Table absent (FTS disabled / trigram off / cjk not
|
||||
# present or tokenizer unavailable) — skip it.
|
||||
continue
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -1149,6 +1169,151 @@ BEGIN
|
||||
END;
|
||||
"""
|
||||
|
||||
# ── CJK-bigram FTS index (replaces the trigram index when available) ────
|
||||
#
|
||||
# The trigram tokenizer needs >=3 chars per query term, so 1-2 char CJK
|
||||
# terms (ubiquitous in Korean/Chinese: 일본, 구글, 项目, ...) fall through
|
||||
# to a LIKE full-table scan — measured 3-6s CPU per query on multi-GB
|
||||
# installs and the dominant base cost of session_search on CJK workloads.
|
||||
#
|
||||
# ``cjk_unicode61`` (native/fts5_cjk/, a ~250-line loadable FTS5 tokenizer
|
||||
# with no dependencies) wraps unicode61: maximal CJK runs are re-emitted as
|
||||
# overlapping character bigrams (Lucene CJKAnalyzer semantics), everything
|
||||
# else passes through unchanged. FTS5 phrase semantics turn a query term's
|
||||
# consecutive bigrams into exact substring matching down to 2 chars at
|
||||
# index speed. Contributed by Soju06 (PR #65544).
|
||||
#
|
||||
# Same v23 storage discipline as the trigram table it replaces:
|
||||
# external-content over a tool-row-excluding view (zero inline text
|
||||
# copies; tool rows stay searchable via ``messages_fts``), triggers gated
|
||||
# on a DEDICATED marker pair (``fts_cjk_rebuild_high_water`` /
|
||||
# ``fts_cjk_rebuild_progress``) so a cjk-only backfill — e.g. the
|
||||
# trigram→cjk upgrade on an already-optimized DB — never gates the
|
||||
# complete ``messages_fts`` index's triggers.
|
||||
#
|
||||
# The table exists ONLY when the loadable tokenizer is available
|
||||
# (``~/.hermes/lib/libfts5_cjk.so``, built by ``native/fts5_cjk/build.sh``).
|
||||
# A process that cannot load it self-heals by dropping the cjk triggers
|
||||
# (message writes keep working; the index goes stale and is rebuilt by the
|
||||
# next ``hermes sessions optimize-storage`` on a capable host).
|
||||
#
|
||||
# Split DDL: the table/view part is safe to ensure any time; the triggers
|
||||
# are created ONLY while the index is complete-or-marker-gated. A stale
|
||||
# index (trigger gap of unknown extent) must keep its triggers DROPPED —
|
||||
# an external-content 'delete' op for a rowid the index never held is the
|
||||
# canonical FTS5 index-corruption hazard the v23 marker gating exists to
|
||||
# prevent.
|
||||
FTS_CJK_TABLE_SQL = """
|
||||
CREATE VIEW IF NOT EXISTS messages_fts_cjk_src AS
|
||||
SELECT id, role, content, tool_name, tool_calls
|
||||
FROM messages
|
||||
WHERE role <> 'tool';
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_cjk USING fts5(
|
||||
content,
|
||||
tool_name,
|
||||
tool_calls,
|
||||
content='messages_fts_cjk_src',
|
||||
content_rowid='id',
|
||||
tokenize='cjk_unicode61'
|
||||
);
|
||||
"""
|
||||
|
||||
FTS_CJK_TRIGGER_SQL = """
|
||||
CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_insert AFTER INSERT ON messages
|
||||
WHEN new.role <> 'tool'
|
||||
AND (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_high_water'), -1)
|
||||
OR new.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_progress'), -1))
|
||||
BEGIN
|
||||
INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls)
|
||||
VALUES (new.id, new.content, new.tool_name, new.tool_calls);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_delete AFTER DELETE ON messages
|
||||
WHEN old.role <> 'tool'
|
||||
AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_high_water'), -1)
|
||||
OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_progress'), -1))
|
||||
BEGIN
|
||||
INSERT INTO messages_fts_cjk(messages_fts_cjk, rowid, content, tool_name, tool_calls)
|
||||
VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_update AFTER UPDATE ON messages
|
||||
WHEN (old.content IS NOT new.content
|
||||
OR old.tool_name IS NOT new.tool_name
|
||||
OR old.tool_calls IS NOT new.tool_calls
|
||||
OR old.role IS NOT new.role)
|
||||
AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_high_water'), -1)
|
||||
OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta
|
||||
WHERE key = 'fts_cjk_rebuild_progress'), -1))
|
||||
BEGIN
|
||||
INSERT INTO messages_fts_cjk(messages_fts_cjk, rowid, content, tool_name, tool_calls)
|
||||
SELECT 'delete', old.id, old.content, old.tool_name, old.tool_calls
|
||||
WHERE old.role <> 'tool';
|
||||
INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls)
|
||||
SELECT new.id, new.content, new.tool_name, new.tool_calls
|
||||
WHERE new.role <> 'tool';
|
||||
END;
|
||||
"""
|
||||
|
||||
_FTS_CJK_TRIGGERS = (
|
||||
"messages_fts_cjk_insert",
|
||||
"messages_fts_cjk_delete",
|
||||
"messages_fts_cjk_update",
|
||||
)
|
||||
|
||||
# state_meta breadcrumb set when a tokenizer-less process had to drop the
|
||||
# cjk triggers to keep message writes alive: rows written from that moment
|
||||
# on are missing from the cjk index, so it must not serve reads until
|
||||
# `hermes sessions optimize-storage` rebuilds it on a capable host.
|
||||
FTS_CJK_STALE_KEY = "fts_cjk_stale"
|
||||
|
||||
|
||||
def fts5_cjk_so_path() -> Path:
|
||||
"""Location of the cjk_unicode61 loadable extension."""
|
||||
env = os.getenv("HERMES_FTS5_CJK_SO")
|
||||
if env:
|
||||
return Path(env).expanduser()
|
||||
return get_hermes_home() / "lib" / "libfts5_cjk.so"
|
||||
|
||||
|
||||
def _cjk_fts_config_enabled() -> bool:
|
||||
"""config.yaml ``sessions.cjk_fts`` (default on), via its env bridge."""
|
||||
return os.getenv("HERMES_CJK_FTS", "1").strip().lower() not in (
|
||||
"0", "false", "off", "no",
|
||||
)
|
||||
|
||||
|
||||
def load_fts5_cjk_extension(conn: sqlite3.Connection) -> bool:
|
||||
"""Best-effort load of the cjk_unicode61 tokenizer into ``conn``.
|
||||
|
||||
Returns False (never raises) when the .so is absent, the feature is
|
||||
disabled via ``sessions.cjk_fts``, or this Python build has extension
|
||||
loading compiled out — every caller treats False as "behave exactly as
|
||||
before the cjk index existed".
|
||||
"""
|
||||
if not _cjk_fts_config_enabled():
|
||||
return False
|
||||
path = fts5_cjk_so_path()
|
||||
if not path.exists():
|
||||
return False
|
||||
try:
|
||||
conn.enable_load_extension(True)
|
||||
try:
|
||||
conn.load_extension(str(path))
|
||||
finally:
|
||||
conn.enable_load_extension(False)
|
||||
return True
|
||||
except Exception:
|
||||
logger.warning("fts5_cjk extension load failed (%s)", path, exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
|
||||
# ── Legacy (v22 / inline-content) FTS DDL ──────────────────────────────
|
||||
# Used ONLY to keep an existing pre-v23 install's search working and its
|
||||
@@ -1269,6 +1434,12 @@ class SessionDB:
|
||||
self._fts_runtime_rebuild_attempted = False
|
||||
self._fts_enabled = False
|
||||
self._trigram_available = False
|
||||
# CJK-bigram index (cjk_unicode61 loadable tokenizer). _fts_cjk_loaded:
|
||||
# extension present on the writer connection; _fts_cjk_available: the
|
||||
# messages_fts_cjk table is queryable AND not marked stale. Set during
|
||||
# _init_schema / _probe_fts_cjk.
|
||||
self._fts_cjk_loaded = False
|
||||
self._fts_cjk_available = False
|
||||
self._fts_unavailable_warned = False
|
||||
self._conn = None
|
||||
try:
|
||||
@@ -1309,6 +1480,7 @@ class SessionDB:
|
||||
self._conn.row_factory = sqlite3.Row
|
||||
apply_wal_with_fallback(self._conn, db_label="state.db")
|
||||
self._conn.execute("PRAGMA foreign_keys=ON")
|
||||
self._fts_cjk_loaded = load_fts5_cjk_extension(self._conn)
|
||||
self._init_schema()
|
||||
|
||||
try:
|
||||
@@ -1372,12 +1544,25 @@ class SessionDB:
|
||||
# Scope to trigram specifically to avoid masking unrelated tokenizer errors.
|
||||
if "no such tokenizer: trigram" in err:
|
||||
return True
|
||||
# The cjk_unicode61 tokenizer is a loadable extension — a process
|
||||
# that couldn't load it sees the same capability-error shape.
|
||||
if "no such tokenizer: cjk_unicode61" in err:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _is_trigram_unavailable_error(exc: sqlite3.OperationalError) -> bool:
|
||||
"""True when only the trigram tokenizer is missing (FTS5 itself works)."""
|
||||
return "no such tokenizer: trigram" in str(exc).lower()
|
||||
"""True when only an optional tokenizer is missing (FTS5 itself works).
|
||||
|
||||
Covers the built-in trigram tokenizer (needs SQLite >= 3.34) and the
|
||||
loadable cjk_unicode61 tokenizer — both mean "this one index can't be
|
||||
served here", never "disable FTS".
|
||||
"""
|
||||
err = str(exc).lower()
|
||||
return (
|
||||
"no such tokenizer: trigram" in err
|
||||
or "no such tokenizer: cjk_unicode61" in err
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _db_has_legacy_inline_fts(cursor: sqlite3.Cursor) -> bool:
|
||||
@@ -1444,6 +1629,124 @@ class SessionDB:
|
||||
self._warn_fts5_unavailable(exc)
|
||||
return False
|
||||
|
||||
def _ensure_fts_cjk_schema(self, cursor) -> None:
|
||||
"""Create / repair / self-heal the CJK-bigram index surface.
|
||||
|
||||
``cursor`` may be a Cursor or a Connection (both expose execute /
|
||||
executescript). Called only for v23-shape DBs with the base FTS
|
||||
surface healthy. Sets ``self._fts_cjk_available``. Never raises;
|
||||
every failure mode degrades to "no cjk index" (trigram/LIKE routing
|
||||
keeps working).
|
||||
|
||||
Cases:
|
||||
tokenizer loaded, table absent → create. Empty DB: index is
|
||||
complete by construction (triggers cover everything). Populated
|
||||
DB: set the cjk backfill markers so the id-gated triggers stay
|
||||
correct and `optimize-storage` can backfill; the index is NOT
|
||||
served until the backfill completes.
|
||||
tokenizer loaded, table present → ensure triggers (recreates any
|
||||
dropped by a tokenizer-less process), honour the stale
|
||||
breadcrumb (serve only when absent and no backfill pending).
|
||||
tokenizer NOT loaded, table present with live triggers → drop the
|
||||
cjk triggers so message INSERTs don't fail at trigger time,
|
||||
and leave the stale breadcrumb (#self-heal). The table itself
|
||||
stays for a later capable open to rebuild.
|
||||
"""
|
||||
cjk_present = bool(cursor.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type = 'table' "
|
||||
"AND name = 'messages_fts_cjk'"
|
||||
).fetchone())
|
||||
|
||||
if not self._fts_cjk_loaded:
|
||||
if cjk_present:
|
||||
live = [
|
||||
r[0] for r in cursor.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type = 'trigger' "
|
||||
f"AND name IN ({','.join('?' for _ in _FTS_CJK_TRIGGERS)})",
|
||||
_FTS_CJK_TRIGGERS,
|
||||
).fetchall()
|
||||
]
|
||||
if live:
|
||||
# Self-heal: this process cannot tokenize, so every
|
||||
# message INSERT would die inside the cjk trigger.
|
||||
# Breadcrumb FIRST (crash between the two statements is
|
||||
# merely conservative), then drop.
|
||||
logger.warning(
|
||||
"messages_fts_cjk triggers present but the "
|
||||
"cjk_unicode61 tokenizer is unavailable (%s) — "
|
||||
"dropping the cjk triggers so message writes keep "
|
||||
"working. CJK search falls back to trigram/LIKE; "
|
||||
"run `hermes sessions optimize-storage` on a host "
|
||||
"with the extension to rebuild.",
|
||||
fts5_cjk_so_path(),
|
||||
)
|
||||
cursor.execute(
|
||||
"INSERT INTO state_meta (key, value) VALUES (?, '1') "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = '1'",
|
||||
(FTS_CJK_STALE_KEY,),
|
||||
)
|
||||
for trig in live:
|
||||
cursor.execute(f"DROP TRIGGER IF EXISTS {trig}")
|
||||
self._fts_cjk_available = False
|
||||
return
|
||||
|
||||
try:
|
||||
cursor.executescript(FTS_CJK_TABLE_SQL)
|
||||
if not cjk_present:
|
||||
# Freshly created. An empty DB's index is complete by
|
||||
# construction (triggers will cover every future row); a
|
||||
# populated DB (e.g. a v23 install predating the cjk index)
|
||||
# gets the dedicated marker pair so the id-gated triggers
|
||||
# keep NEW rows indexed while old rows await the
|
||||
# `optimize-storage` backfill. Either way any old stale
|
||||
# breadcrumb refers to a table that no longer exists.
|
||||
cursor.execute(
|
||||
"DELETE FROM state_meta WHERE key = ?",
|
||||
(FTS_CJK_STALE_KEY,),
|
||||
)
|
||||
n_msgs = cursor.execute(
|
||||
"SELECT COUNT(*) FROM messages WHERE role <> 'tool'"
|
||||
).fetchone()[0]
|
||||
if n_msgs > 0:
|
||||
hw = cursor.execute(
|
||||
"SELECT COALESCE(MAX(id), 0) FROM messages"
|
||||
).fetchone()[0]
|
||||
for k, v in (
|
||||
("fts_cjk_rebuild_high_water", str(hw)),
|
||||
("fts_cjk_rebuild_progress", "0"),
|
||||
):
|
||||
cursor.execute(
|
||||
"INSERT INTO state_meta (key, value) VALUES (?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(k, v),
|
||||
)
|
||||
stale = cursor.execute(
|
||||
"SELECT 1 FROM state_meta WHERE key = ?",
|
||||
(FTS_CJK_STALE_KEY,),
|
||||
).fetchone()
|
||||
if stale:
|
||||
# A tokenizer-less process dropped the triggers at some
|
||||
# unknown point — the index has a gap of unknown extent.
|
||||
# Do NOT reinstall triggers (an external-content 'delete'
|
||||
# for an unindexed rowid corrupts the index); the next
|
||||
# `optimize-storage` run rebuilds from scratch.
|
||||
self._fts_cjk_available = False
|
||||
return
|
||||
cursor.executescript(FTS_CJK_TRIGGER_SQL)
|
||||
backfill_pending = cursor.execute(
|
||||
"SELECT 1 FROM state_meta "
|
||||
"WHERE key = 'fts_cjk_rebuild_high_water' LIMIT 1"
|
||||
).fetchone()
|
||||
self._fts_cjk_available = not backfill_pending
|
||||
except sqlite3.OperationalError:
|
||||
# Includes "no such tokenizer: cjk_unicode61" if the extension
|
||||
# loaded but registration failed — degrade to trigram/LIKE.
|
||||
logger.warning(
|
||||
"messages_fts_cjk ensure failed; CJK search stays on "
|
||||
"trigram/LIKE", exc_info=True,
|
||||
)
|
||||
self._fts_cjk_available = False
|
||||
|
||||
@staticmethod
|
||||
def _drop_fts_triggers(cursor: sqlite3.Cursor) -> None:
|
||||
for trigger in _FTS_TRIGGERS:
|
||||
@@ -1968,6 +2271,136 @@ class SessionDB:
|
||||
return False
|
||||
return bool(more)
|
||||
|
||||
# ── CJK-bigram index backfill (dedicated marker pair) ──
|
||||
#
|
||||
# Same chunk engine as the main deferred rebuild, but on the
|
||||
# ``fts_cjk_rebuild_*`` markers so a cjk-only backfill (the common case:
|
||||
# an already-optimized v23 DB gaining the cjk index) never gates the
|
||||
# complete ``messages_fts`` / trigram triggers.
|
||||
|
||||
def fts_cjk_rebuild_status(self) -> Optional[Dict[str, Any]]:
|
||||
"""CJK-index backfill progress, or None when none is pending."""
|
||||
high_water = self.get_meta("fts_cjk_rebuild_high_water")
|
||||
if high_water is None:
|
||||
return None
|
||||
progress = int(self.get_meta("fts_cjk_rebuild_progress") or 0)
|
||||
total = int(high_water)
|
||||
if total <= 0:
|
||||
return None
|
||||
pct = min(100, int(100 * progress / total))
|
||||
return {"pending": True, "total": total, "indexed": progress, "percent": pct}
|
||||
|
||||
def fts_cjk_rebuild_step(self) -> bool:
|
||||
"""Backfill one chunk of the CJK index. True while work remains."""
|
||||
if not self._fts_enabled or not self._fts_cjk_loaded:
|
||||
return False
|
||||
high_water_raw = self.get_meta("fts_cjk_rebuild_high_water")
|
||||
if high_water_raw is None:
|
||||
return False
|
||||
high_water = int(high_water_raw)
|
||||
chunk = self._FTS_REBUILD_CHUNK_ROWS
|
||||
|
||||
def _do(conn):
|
||||
row = conn.execute(
|
||||
"SELECT value FROM state_meta "
|
||||
"WHERE key = 'fts_cjk_rebuild_progress'"
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return False # finished (or cleared) by another process
|
||||
progress = int(row[0])
|
||||
if progress >= high_water:
|
||||
return False
|
||||
upper = min(progress + chunk, high_water)
|
||||
conn.execute(
|
||||
"INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) "
|
||||
"SELECT id, content, tool_name, tool_calls FROM messages "
|
||||
"WHERE id > ? AND id <= ? AND role <> 'tool'",
|
||||
(progress, upper),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE state_meta SET value = ? "
|
||||
"WHERE key = 'fts_cjk_rebuild_progress'",
|
||||
(str(upper),),
|
||||
)
|
||||
return upper < high_water
|
||||
|
||||
try:
|
||||
more = self._execute_write(_do)
|
||||
except sqlite3.OperationalError as exc:
|
||||
logger.debug("CJK FTS rebuild chunk failed (will retry): %s", exc)
|
||||
return True
|
||||
if more is False:
|
||||
status = self.fts_cjk_rebuild_status()
|
||||
if status is not None and status["indexed"] >= status["total"]:
|
||||
self._fts_cjk_rebuild_finish()
|
||||
return False
|
||||
return bool(more)
|
||||
|
||||
def _fts_cjk_rebuild_finish(self) -> None:
|
||||
"""Boundary sweep + clear the cjk markers; index becomes servable."""
|
||||
def _do(conn):
|
||||
hw_row = conn.execute(
|
||||
"SELECT value FROM state_meta "
|
||||
"WHERE key = 'fts_cjk_rebuild_high_water'"
|
||||
).fetchone()
|
||||
if hw_row is not None:
|
||||
hw = int(hw_row[0])
|
||||
lo, hi = hw - 1000, hw + 1000
|
||||
conn.execute(
|
||||
"INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) "
|
||||
"SELECT m.id, m.content, m.tool_name, m.tool_calls "
|
||||
"FROM messages m "
|
||||
"WHERE m.id > ? AND m.id <= ? AND m.role <> 'tool' "
|
||||
"AND NOT EXISTS (SELECT 1 FROM messages_fts_cjk_docsize d WHERE d.id = m.id)",
|
||||
(lo, hi),
|
||||
)
|
||||
conn.execute(
|
||||
"DELETE FROM state_meta WHERE key IN "
|
||||
"('fts_cjk_rebuild_high_water', 'fts_cjk_rebuild_progress')"
|
||||
)
|
||||
self._execute_write(_do)
|
||||
self._fts_cjk_available = True
|
||||
logger.info("CJK FTS index backfill complete — serving CJK search.")
|
||||
|
||||
def _fts_cjk_reset_if_stale(self) -> None:
|
||||
"""Rebuild path for a stale cjk index (triggers were dropped).
|
||||
|
||||
The gap's extent is unknown, so the only safe recovery is a from-
|
||||
scratch rebuild: drop the table + triggers, clear the breadcrumb,
|
||||
recreate via ``_ensure_fts_cjk_schema`` (which sets fresh backfill
|
||||
markers on a populated DB). Called from ``optimize_fts_storage`` on
|
||||
a tokenizer-capable host; no-op when not stale.
|
||||
"""
|
||||
if not self._fts_cjk_loaded:
|
||||
return
|
||||
|
||||
def _do(conn):
|
||||
stale = conn.execute(
|
||||
"SELECT 1 FROM state_meta WHERE key = ?",
|
||||
(FTS_CJK_STALE_KEY,),
|
||||
).fetchone()
|
||||
if not stale:
|
||||
return False
|
||||
for trig in _FTS_CJK_TRIGGERS:
|
||||
conn.execute(f"DROP TRIGGER IF EXISTS {trig}")
|
||||
conn.execute("DROP TABLE IF EXISTS messages_fts_cjk")
|
||||
conn.execute("DROP VIEW IF EXISTS messages_fts_cjk_src")
|
||||
conn.execute(
|
||||
"DELETE FROM state_meta WHERE key IN "
|
||||
f"('{FTS_CJK_STALE_KEY}', 'fts_cjk_rebuild_high_water', "
|
||||
"'fts_cjk_rebuild_progress')"
|
||||
)
|
||||
return True
|
||||
was_stale = self._execute_write(_do)
|
||||
if was_stale:
|
||||
# Recreate outside the write transaction — _ensure_fts_cjk_schema
|
||||
# uses executescript(), which implicitly commits any pending
|
||||
# transaction and must not run inside _execute_write's BEGIN
|
||||
# IMMEDIATE. Sets fresh backfill markers on a populated DB.
|
||||
with self._lock:
|
||||
self._ensure_fts_cjk_schema(self._conn)
|
||||
self._conn.commit()
|
||||
|
||||
# ── Opt-in v23 FTS storage optimization (`hermes sessions optimize-storage`) ──
|
||||
#
|
||||
# This is the ONLY path that migrates an existing legacy (v22 inline) DB
|
||||
@@ -1983,8 +2416,10 @@ class SessionDB:
|
||||
is a legacy inline-FTS install that can be optimized to the v23
|
||||
external-content schema, or a previous optimize run was interrupted
|
||||
(legacy vtables already demoted, but backfill markers and/or trash
|
||||
tables remain) and re-running would resume it. False for fresh and
|
||||
fully-optimized installs (and when FTS5 is unavailable)."""
|
||||
tables remain) and re-running would resume it, or the CJK-bigram
|
||||
index needs a backfill/rebuild on this tokenizer-capable host.
|
||||
False for fresh and fully-optimized installs (and when FTS5 is
|
||||
unavailable)."""
|
||||
if not self._fts_enabled or self.read_only:
|
||||
return False
|
||||
with self._lock:
|
||||
@@ -2000,6 +2435,14 @@ class SessionDB:
|
||||
"WHERE key = 'fts_rebuild_high_water' LIMIT 1"
|
||||
).fetchone():
|
||||
return True
|
||||
# CJK-bigram index work — only offerable when THIS process can
|
||||
# tokenize: a pending backfill (markers set at creation on a
|
||||
# populated DB) or a stale index awaiting a from-scratch rebuild.
|
||||
if self._fts_cjk_loaded and self._conn.execute(
|
||||
"SELECT 1 FROM state_meta WHERE key IN "
|
||||
f"('fts_cjk_rebuild_high_water', '{FTS_CJK_STALE_KEY}') LIMIT 1"
|
||||
).fetchone():
|
||||
return True
|
||||
return self._has_fts_trash(self._conn)
|
||||
|
||||
def _has_fts_trash(self, conn) -> bool:
|
||||
@@ -2089,10 +2532,24 @@ class SessionDB:
|
||||
if legacy and not pending:
|
||||
self._demote_legacy_fts_to_trash()
|
||||
|
||||
# A stale CJK index (triggers dropped by a tokenizer-less process)
|
||||
# can only be recovered from scratch — reset it now so the cjk
|
||||
# backfill phase below rebuilds it. No-op without the tokenizer.
|
||||
self._fts_cjk_reset_if_stale()
|
||||
# An optimized v23 DB gaining the cjk index for the first time (no
|
||||
# legacy work left, tokenizer newly installed): ensure the table +
|
||||
# markers exist so the backfill phase has work to claim.
|
||||
if self._fts_cjk_loaded:
|
||||
with self._lock:
|
||||
self._ensure_fts_cjk_schema(self._conn)
|
||||
self._conn.commit()
|
||||
|
||||
def _emit(phase: str) -> None:
|
||||
if progress_cb is None:
|
||||
return
|
||||
st = self.fts_rebuild_status()
|
||||
if st is None:
|
||||
st = self.fts_cjk_rebuild_status()
|
||||
progress_cb({
|
||||
"phase": phase,
|
||||
"percent": st["percent"] if st else 100,
|
||||
@@ -2125,6 +2582,15 @@ class SessionDB:
|
||||
_pause(time.monotonic() - _t0)
|
||||
_emit("backfill")
|
||||
|
||||
# Phase 1b: backfill the CJK-bigram index (its own marker pair; a
|
||||
# no-op when the tokenizer isn't loadable or nothing is pending).
|
||||
while True:
|
||||
_t0 = time.monotonic()
|
||||
if not self.fts_cjk_rebuild_step():
|
||||
break
|
||||
_emit("backfill")
|
||||
_pause(time.monotonic() - _t0)
|
||||
|
||||
# Phase 2: tear down the demoted legacy shadow tables in chunks.
|
||||
_emit("teardown")
|
||||
while True:
|
||||
@@ -2687,6 +3153,9 @@ class SessionDB:
|
||||
cursor,
|
||||
include_trigram=trigram_enabled,
|
||||
)
|
||||
# CJK-bigram index (cjk_unicode61). Strictly additive to
|
||||
# the surfaces above and gated on the loadable tokenizer:
|
||||
self._ensure_fts_cjk_schema(cursor)
|
||||
|
||||
self._conn.commit()
|
||||
|
||||
@@ -6290,6 +6759,24 @@ class SessionDB:
|
||||
"""Count CJK characters in text."""
|
||||
return sum(1 for ch in text if cls._is_cjk_codepoint(ord(ch)))
|
||||
|
||||
@classmethod
|
||||
def _has_lone_cjk_run(cls, query: str) -> bool:
|
||||
"""True when any maximal CJK run in the query is a single char.
|
||||
|
||||
The cjk-bigram index stores bigrams for runs >=2 chars and unigrams
|
||||
only for isolated chars, so a 1-char CJK term can't match inside
|
||||
longer runs there — those queries keep the LIKE substring route.
|
||||
"""
|
||||
run = 0
|
||||
for ch in query:
|
||||
if cls._is_cjk_codepoint(ord(ch)):
|
||||
run += 1
|
||||
else:
|
||||
if run == 1:
|
||||
return True
|
||||
run = 0
|
||||
return run == 1
|
||||
|
||||
def search_messages(
|
||||
self,
|
||||
query: str,
|
||||
@@ -6344,11 +6831,11 @@ class SessionDB:
|
||||
sanitized = self._sanitize_fts5_query(query or "")
|
||||
if not sanitized:
|
||||
return "empty"
|
||||
if self._fts_v2_query_allowed(sanitized):
|
||||
return "fts_v2"
|
||||
if not self._contains_cjk(sanitized):
|
||||
return "fts5"
|
||||
raw = sanitized.strip('"').strip()
|
||||
if self._fts_cjk_available and not self._has_lone_cjk_run(raw):
|
||||
return "fts_cjk"
|
||||
tokens = [
|
||||
t for t in raw.split()
|
||||
if t.upper() not in {"AND", "OR", "NOT"} and self._contains_cjk(t)
|
||||
@@ -6507,8 +6994,104 @@ class SessionDB:
|
||||
# query explicitly filtering on role='tool' must therefore use
|
||||
# the LIKE fallback, which scans the base table directly.
|
||||
_wants_tool_rows = bool(role_filter) and "tool" in role_filter
|
||||
|
||||
# ── CJK-bigram route (messages_fts_cjk, cjk_unicode61) ──────
|
||||
# When the bigram index is available it serves EVERY CJK query
|
||||
# shape the legacy code split between trigram (>=3 chars/token)
|
||||
# and LIKE full scans (1-2 char tokens) — the whole point of the
|
||||
# index (PR #65544). Exceptions stay on the legacy routes:
|
||||
# - role_filter=['tool'] queries (tool rows aren't in the cjk
|
||||
# index, same exclusion as trigram),
|
||||
# - queries containing a LONE 1-char CJK run: the index stores
|
||||
# bigrams for runs >=2, so a single-char term can only match
|
||||
# isolated chars — LIKE substring semantics are broader.
|
||||
if (
|
||||
cjk_count >= 3
|
||||
self._fts_cjk_available
|
||||
and not _wants_tool_rows
|
||||
and not self._has_lone_cjk_run(raw_query)
|
||||
):
|
||||
tokens = raw_query.split()
|
||||
parts = []
|
||||
for tok in tokens:
|
||||
if tok.upper() in {"AND", "OR", "NOT"}:
|
||||
parts.append(tok)
|
||||
else:
|
||||
parts.append('"' + tok.replace('"', '""') + '"')
|
||||
cjk_query = " ".join(parts)
|
||||
cjk_where = ["messages_fts_cjk MATCH ?"]
|
||||
cjk_params: list = [cjk_query]
|
||||
if not include_inactive:
|
||||
cjk_where.append("(m.active = 1 OR m.compacted = 1)")
|
||||
if source_filter is not None:
|
||||
cjk_where.append(f"s.source IN ({','.join('?' for _ in source_filter)})")
|
||||
cjk_params.extend(source_filter)
|
||||
if exclude_sources is not None:
|
||||
cjk_where.append(f"s.source NOT IN ({','.join('?' for _ in exclude_sources)})")
|
||||
cjk_params.extend(exclude_sources)
|
||||
if role_filter:
|
||||
cjk_where.append(f"m.role IN ({','.join('?' for _ in role_filter)})")
|
||||
cjk_params.extend(role_filter)
|
||||
cjk_sql = f"""
|
||||
SELECT
|
||||
m.id,
|
||||
m.session_id,
|
||||
m.role,
|
||||
snippet(messages_fts_cjk, -1, '>>>', '<<<', '...', 40) AS snippet,
|
||||
m.content,
|
||||
m.timestamp,
|
||||
m.tool_name,
|
||||
s.source,
|
||||
s.model,
|
||||
s.started_at AS session_started
|
||||
FROM messages_fts_cjk
|
||||
JOIN messages m ON m.id = messages_fts_cjk.rowid
|
||||
JOIN sessions s ON s.id = m.session_id
|
||||
WHERE {' AND '.join(cjk_where)}
|
||||
{order_by_sql}
|
||||
LIMIT ? OFFSET ?
|
||||
"""
|
||||
cjk_params.extend([limit, offset])
|
||||
try:
|
||||
with self._lock:
|
||||
cjk_cursor = self._conn.execute(cjk_sql, cjk_params)
|
||||
matches = [dict(row) for row in cjk_cursor.fetchall()]
|
||||
_trigram_succeeded = True
|
||||
except sqlite3.OperationalError:
|
||||
# Tokenizer missing on this connection / query syntax —
|
||||
# the trigram + LIKE routes below still answer.
|
||||
logger.debug(
|
||||
"messages_fts_cjk query failed; falling back to "
|
||||
"trigram/LIKE", exc_info=True,
|
||||
)
|
||||
except sqlite3.DatabaseError as exc:
|
||||
# Same corruption class as the other FTS reads: rebuild
|
||||
# in place once and retry; on refusal/failure fall back.
|
||||
if self._try_runtime_fts_rebuild(exc):
|
||||
try:
|
||||
with self._lock:
|
||||
cjk_cursor = self._conn.execute(
|
||||
cjk_sql, cjk_params
|
||||
)
|
||||
matches = [
|
||||
dict(row) for row in cjk_cursor.fetchall()
|
||||
]
|
||||
_trigram_succeeded = True
|
||||
except sqlite3.DatabaseError:
|
||||
logger.warning(
|
||||
"CJK-bigram FTS search still failing after "
|
||||
"in-place rebuild; falling back to "
|
||||
"trigram/LIKE."
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"CJK-bigram FTS search hit a corruption error "
|
||||
"(%s) and no in-place rebuild was possible; "
|
||||
"falling back to trigram/LIKE.", exc,
|
||||
)
|
||||
|
||||
if (
|
||||
not _trigram_succeeded
|
||||
and cjk_count >= 3
|
||||
and not _any_short_cjk
|
||||
and self._trigram_available
|
||||
and not _wants_tool_rows
|
||||
@@ -8604,9 +9187,10 @@ class SessionDB:
|
||||
# ── Space reclamation ──
|
||||
|
||||
# FTS5 virtual tables whose b-tree segments we merge on optimize. The
|
||||
# trigram table is created lazily / may be disabled, so we probe before
|
||||
# touching it (see optimize_fts).
|
||||
_FTS_TABLES = ("messages_fts", "messages_fts_trigram")
|
||||
# trigram table is created lazily / may be disabled, and the cjk-bigram
|
||||
# table only exists (and is only queryable) when the loadable tokenizer
|
||||
# is present — so we probe each before touching it (see optimize_fts).
|
||||
_FTS_TABLES = ("messages_fts", "messages_fts_trigram", "messages_fts_cjk")
|
||||
|
||||
def _fts_table_exists(self, name: str) -> bool:
|
||||
"""True if an FTS5 virtual table is queryable in this DB."""
|
||||
|
||||
@@ -1,12 +1,24 @@
|
||||
# fts5_cjk — cjk_unicode61 FTS5 tokenizer
|
||||
|
||||
unicode61 + CJK character bigrams (Lucene CJKAnalyzer semantics). Fixes
|
||||
2-char Korean/CJK terms falling through to LIKE full-table scans.
|
||||
1-2 char Korean/Chinese/Japanese terms falling through to LIKE full-table
|
||||
scans in session search.
|
||||
|
||||
Build & install to ~/.hermes/lib/:
|
||||
Build & install to `~/.hermes/lib/`:
|
||||
|
||||
./build.sh
|
||||
|
||||
Then run `scripts/fts_v2_migrate.py` to create + backfill messages_fts_v2,
|
||||
and set `HERMES_FTS_V2_READ=1` in ~/.hermes/.env to cut reads over.
|
||||
Override the .so location with `HERMES_FTS5_CJK_SO`.
|
||||
Uses the system `sqlite3ext.h` when available, else the vendored copy in
|
||||
`vendor/` — no libsqlite3-dev required.
|
||||
|
||||
Once the extension is installed, the next `SessionDB` open creates the
|
||||
`messages_fts_cjk` index (external-content, tool rows excluded — same v23
|
||||
storage discipline as the other indexes). On a populated database, run
|
||||
|
||||
hermes sessions optimize-storage
|
||||
|
||||
to backfill it; new messages are indexed live either way. Set
|
||||
`sessions.cjk_fts: false` in `~/.hermes/config.yaml` to disable. Override
|
||||
the .so location with `HERMES_FTS5_CJK_SO`.
|
||||
|
||||
Contributed by Soju06 (PR #65544).
|
||||
|
||||
@@ -1,8 +1,18 @@
|
||||
#!/bin/bash
|
||||
# Build libfts5_cjk.so and install to ~/.hermes/lib/ (or $1).
|
||||
#
|
||||
# Uses the system sqlite3ext.h when present, else the vendored copy in
|
||||
# vendor/ (public-domain SQLite amalgamation headers) so the build works
|
||||
# without libsqlite3-dev installed.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
gcc -shared -fPIC -O2 -Wall -Wextra fts5_cjk.c -o libfts5_cjk.so
|
||||
|
||||
CFLAGS_EXTRA=""
|
||||
if ! echo '#include <sqlite3ext.h>' | gcc -E -xc - >/dev/null 2>&1; then
|
||||
CFLAGS_EXTRA="-Ivendor"
|
||||
fi
|
||||
|
||||
gcc -shared -fPIC -O2 -Wall -Wextra $CFLAGS_EXTRA fts5_cjk.c -o libfts5_cjk.so
|
||||
dest="${1:-$HOME/.hermes/lib}"
|
||||
mkdir -p "$dest"
|
||||
install -m 0644 libfts5_cjk.so "$dest/libfts5_cjk.so"
|
||||
|
||||
Vendored
+13775
File diff suppressed because it is too large
Load Diff
Vendored
+723
@@ -0,0 +1,723 @@
|
||||
/*
|
||||
** 2006 June 7
|
||||
**
|
||||
** The author disclaims copyright to this source code. In place of
|
||||
** a legal notice, here is a blessing:
|
||||
**
|
||||
** May you do good and not evil.
|
||||
** May you find forgiveness for yourself and forgive others.
|
||||
** May you share freely, never taking more than you give.
|
||||
**
|
||||
*************************************************************************
|
||||
** This header file defines the SQLite interface for use by
|
||||
** shared libraries that want to be imported as extensions into
|
||||
** an SQLite instance. Shared libraries that intend to be loaded
|
||||
** as extensions by SQLite should #include this file instead of
|
||||
** sqlite3.h.
|
||||
*/
|
||||
#ifndef SQLITE3EXT_H
|
||||
#define SQLITE3EXT_H
|
||||
#include "sqlite3.h"
|
||||
|
||||
/*
|
||||
** The following structure holds pointers to all of the SQLite API
|
||||
** routines.
|
||||
**
|
||||
** WARNING: In order to maintain backwards compatibility, add new
|
||||
** interfaces to the end of this structure only. If you insert new
|
||||
** interfaces in the middle of this structure, then older different
|
||||
** versions of SQLite will not be able to load each other's shared
|
||||
** libraries!
|
||||
*/
|
||||
struct sqlite3_api_routines {
|
||||
void * (*aggregate_context)(sqlite3_context*,int nBytes);
|
||||
int (*aggregate_count)(sqlite3_context*);
|
||||
int (*bind_blob)(sqlite3_stmt*,int,const void*,int n,void(*)(void*));
|
||||
int (*bind_double)(sqlite3_stmt*,int,double);
|
||||
int (*bind_int)(sqlite3_stmt*,int,int);
|
||||
int (*bind_int64)(sqlite3_stmt*,int,sqlite_int64);
|
||||
int (*bind_null)(sqlite3_stmt*,int);
|
||||
int (*bind_parameter_count)(sqlite3_stmt*);
|
||||
int (*bind_parameter_index)(sqlite3_stmt*,const char*zName);
|
||||
const char * (*bind_parameter_name)(sqlite3_stmt*,int);
|
||||
int (*bind_text)(sqlite3_stmt*,int,const char*,int n,void(*)(void*));
|
||||
int (*bind_text16)(sqlite3_stmt*,int,const void*,int,void(*)(void*));
|
||||
int (*bind_value)(sqlite3_stmt*,int,const sqlite3_value*);
|
||||
int (*busy_handler)(sqlite3*,int(*)(void*,int),void*);
|
||||
int (*busy_timeout)(sqlite3*,int ms);
|
||||
int (*changes)(sqlite3*);
|
||||
int (*close)(sqlite3*);
|
||||
int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*,
|
||||
int eTextRep,const char*));
|
||||
int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*,
|
||||
int eTextRep,const void*));
|
||||
const void * (*column_blob)(sqlite3_stmt*,int iCol);
|
||||
int (*column_bytes)(sqlite3_stmt*,int iCol);
|
||||
int (*column_bytes16)(sqlite3_stmt*,int iCol);
|
||||
int (*column_count)(sqlite3_stmt*pStmt);
|
||||
const char * (*column_database_name)(sqlite3_stmt*,int);
|
||||
const void * (*column_database_name16)(sqlite3_stmt*,int);
|
||||
const char * (*column_decltype)(sqlite3_stmt*,int i);
|
||||
const void * (*column_decltype16)(sqlite3_stmt*,int);
|
||||
double (*column_double)(sqlite3_stmt*,int iCol);
|
||||
int (*column_int)(sqlite3_stmt*,int iCol);
|
||||
sqlite_int64 (*column_int64)(sqlite3_stmt*,int iCol);
|
||||
const char * (*column_name)(sqlite3_stmt*,int);
|
||||
const void * (*column_name16)(sqlite3_stmt*,int);
|
||||
const char * (*column_origin_name)(sqlite3_stmt*,int);
|
||||
const void * (*column_origin_name16)(sqlite3_stmt*,int);
|
||||
const char * (*column_table_name)(sqlite3_stmt*,int);
|
||||
const void * (*column_table_name16)(sqlite3_stmt*,int);
|
||||
const unsigned char * (*column_text)(sqlite3_stmt*,int iCol);
|
||||
const void * (*column_text16)(sqlite3_stmt*,int iCol);
|
||||
int (*column_type)(sqlite3_stmt*,int iCol);
|
||||
sqlite3_value* (*column_value)(sqlite3_stmt*,int iCol);
|
||||
void * (*commit_hook)(sqlite3*,int(*)(void*),void*);
|
||||
int (*complete)(const char*sql);
|
||||
int (*complete16)(const void*sql);
|
||||
int (*create_collation)(sqlite3*,const char*,int,void*,
|
||||
int(*)(void*,int,const void*,int,const void*));
|
||||
int (*create_collation16)(sqlite3*,const void*,int,void*,
|
||||
int(*)(void*,int,const void*,int,const void*));
|
||||
int (*create_function)(sqlite3*,const char*,int,int,void*,
|
||||
void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xFinal)(sqlite3_context*));
|
||||
int (*create_function16)(sqlite3*,const void*,int,int,void*,
|
||||
void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xFinal)(sqlite3_context*));
|
||||
int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*);
|
||||
int (*data_count)(sqlite3_stmt*pStmt);
|
||||
sqlite3 * (*db_handle)(sqlite3_stmt*);
|
||||
int (*declare_vtab)(sqlite3*,const char*);
|
||||
int (*enable_shared_cache)(int);
|
||||
int (*errcode)(sqlite3*db);
|
||||
const char * (*errmsg)(sqlite3*);
|
||||
const void * (*errmsg16)(sqlite3*);
|
||||
int (*exec)(sqlite3*,const char*,sqlite3_callback,void*,char**);
|
||||
int (*expired)(sqlite3_stmt*);
|
||||
int (*finalize)(sqlite3_stmt*pStmt);
|
||||
void (*free)(void*);
|
||||
void (*free_table)(char**result);
|
||||
int (*get_autocommit)(sqlite3*);
|
||||
void * (*get_auxdata)(sqlite3_context*,int);
|
||||
int (*get_table)(sqlite3*,const char*,char***,int*,int*,char**);
|
||||
int (*global_recover)(void);
|
||||
void (*interruptx)(sqlite3*);
|
||||
sqlite_int64 (*last_insert_rowid)(sqlite3*);
|
||||
const char * (*libversion)(void);
|
||||
int (*libversion_number)(void);
|
||||
void *(*malloc)(int);
|
||||
char * (*mprintf)(const char*,...);
|
||||
int (*open)(const char*,sqlite3**);
|
||||
int (*open16)(const void*,sqlite3**);
|
||||
int (*prepare)(sqlite3*,const char*,int,sqlite3_stmt**,const char**);
|
||||
int (*prepare16)(sqlite3*,const void*,int,sqlite3_stmt**,const void**);
|
||||
void * (*profile)(sqlite3*,void(*)(void*,const char*,sqlite_uint64),void*);
|
||||
void (*progress_handler)(sqlite3*,int,int(*)(void*),void*);
|
||||
void *(*realloc)(void*,int);
|
||||
int (*reset)(sqlite3_stmt*pStmt);
|
||||
void (*result_blob)(sqlite3_context*,const void*,int,void(*)(void*));
|
||||
void (*result_double)(sqlite3_context*,double);
|
||||
void (*result_error)(sqlite3_context*,const char*,int);
|
||||
void (*result_error16)(sqlite3_context*,const void*,int);
|
||||
void (*result_int)(sqlite3_context*,int);
|
||||
void (*result_int64)(sqlite3_context*,sqlite_int64);
|
||||
void (*result_null)(sqlite3_context*);
|
||||
void (*result_text)(sqlite3_context*,const char*,int,void(*)(void*));
|
||||
void (*result_text16)(sqlite3_context*,const void*,int,void(*)(void*));
|
||||
void (*result_text16be)(sqlite3_context*,const void*,int,void(*)(void*));
|
||||
void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*));
|
||||
void (*result_value)(sqlite3_context*,sqlite3_value*);
|
||||
void * (*rollback_hook)(sqlite3*,void(*)(void*),void*);
|
||||
int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*,
|
||||
const char*,const char*),void*);
|
||||
void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*));
|
||||
char * (*xsnprintf)(int,char*,const char*,...);
|
||||
int (*step)(sqlite3_stmt*);
|
||||
int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*,
|
||||
char const**,char const**,int*,int*,int*);
|
||||
void (*thread_cleanup)(void);
|
||||
int (*total_changes)(sqlite3*);
|
||||
void * (*trace)(sqlite3*,void(*xTrace)(void*,const char*),void*);
|
||||
int (*transfer_bindings)(sqlite3_stmt*,sqlite3_stmt*);
|
||||
void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*,
|
||||
sqlite_int64),void*);
|
||||
void * (*user_data)(sqlite3_context*);
|
||||
const void * (*value_blob)(sqlite3_value*);
|
||||
int (*value_bytes)(sqlite3_value*);
|
||||
int (*value_bytes16)(sqlite3_value*);
|
||||
double (*value_double)(sqlite3_value*);
|
||||
int (*value_int)(sqlite3_value*);
|
||||
sqlite_int64 (*value_int64)(sqlite3_value*);
|
||||
int (*value_numeric_type)(sqlite3_value*);
|
||||
const unsigned char * (*value_text)(sqlite3_value*);
|
||||
const void * (*value_text16)(sqlite3_value*);
|
||||
const void * (*value_text16be)(sqlite3_value*);
|
||||
const void * (*value_text16le)(sqlite3_value*);
|
||||
int (*value_type)(sqlite3_value*);
|
||||
char *(*vmprintf)(const char*,va_list);
|
||||
/* Added ??? */
|
||||
int (*overload_function)(sqlite3*, const char *zFuncName, int nArg);
|
||||
/* Added by 3.3.13 */
|
||||
int (*prepare_v2)(sqlite3*,const char*,int,sqlite3_stmt**,const char**);
|
||||
int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**);
|
||||
int (*clear_bindings)(sqlite3_stmt*);
|
||||
/* Added by 3.4.1 */
|
||||
int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*,
|
||||
void (*xDestroy)(void *));
|
||||
/* Added by 3.5.0 */
|
||||
int (*bind_zeroblob)(sqlite3_stmt*,int,int);
|
||||
int (*blob_bytes)(sqlite3_blob*);
|
||||
int (*blob_close)(sqlite3_blob*);
|
||||
int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64,
|
||||
int,sqlite3_blob**);
|
||||
int (*blob_read)(sqlite3_blob*,void*,int,int);
|
||||
int (*blob_write)(sqlite3_blob*,const void*,int,int);
|
||||
int (*create_collation_v2)(sqlite3*,const char*,int,void*,
|
||||
int(*)(void*,int,const void*,int,const void*),
|
||||
void(*)(void*));
|
||||
int (*file_control)(sqlite3*,const char*,int,void*);
|
||||
sqlite3_int64 (*memory_highwater)(int);
|
||||
sqlite3_int64 (*memory_used)(void);
|
||||
sqlite3_mutex *(*mutex_alloc)(int);
|
||||
void (*mutex_enter)(sqlite3_mutex*);
|
||||
void (*mutex_free)(sqlite3_mutex*);
|
||||
void (*mutex_leave)(sqlite3_mutex*);
|
||||
int (*mutex_try)(sqlite3_mutex*);
|
||||
int (*open_v2)(const char*,sqlite3**,int,const char*);
|
||||
int (*release_memory)(int);
|
||||
void (*result_error_nomem)(sqlite3_context*);
|
||||
void (*result_error_toobig)(sqlite3_context*);
|
||||
int (*sleep)(int);
|
||||
void (*soft_heap_limit)(int);
|
||||
sqlite3_vfs *(*vfs_find)(const char*);
|
||||
int (*vfs_register)(sqlite3_vfs*,int);
|
||||
int (*vfs_unregister)(sqlite3_vfs*);
|
||||
int (*xthreadsafe)(void);
|
||||
void (*result_zeroblob)(sqlite3_context*,int);
|
||||
void (*result_error_code)(sqlite3_context*,int);
|
||||
int (*test_control)(int, ...);
|
||||
void (*randomness)(int,void*);
|
||||
sqlite3 *(*context_db_handle)(sqlite3_context*);
|
||||
int (*extended_result_codes)(sqlite3*,int);
|
||||
int (*limit)(sqlite3*,int,int);
|
||||
sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*);
|
||||
const char *(*sql)(sqlite3_stmt*);
|
||||
int (*status)(int,int*,int*,int);
|
||||
int (*backup_finish)(sqlite3_backup*);
|
||||
sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*);
|
||||
int (*backup_pagecount)(sqlite3_backup*);
|
||||
int (*backup_remaining)(sqlite3_backup*);
|
||||
int (*backup_step)(sqlite3_backup*,int);
|
||||
const char *(*compileoption_get)(int);
|
||||
int (*compileoption_used)(const char*);
|
||||
int (*create_function_v2)(sqlite3*,const char*,int,int,void*,
|
||||
void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xFinal)(sqlite3_context*),
|
||||
void(*xDestroy)(void*));
|
||||
int (*db_config)(sqlite3*,int,...);
|
||||
sqlite3_mutex *(*db_mutex)(sqlite3*);
|
||||
int (*db_status)(sqlite3*,int,int*,int*,int);
|
||||
int (*extended_errcode)(sqlite3*);
|
||||
void (*log)(int,const char*,...);
|
||||
sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64);
|
||||
const char *(*sourceid)(void);
|
||||
int (*stmt_status)(sqlite3_stmt*,int,int);
|
||||
int (*strnicmp)(const char*,const char*,int);
|
||||
int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*);
|
||||
int (*wal_autocheckpoint)(sqlite3*,int);
|
||||
int (*wal_checkpoint)(sqlite3*,const char*);
|
||||
void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*);
|
||||
int (*blob_reopen)(sqlite3_blob*,sqlite3_int64);
|
||||
int (*vtab_config)(sqlite3*,int op,...);
|
||||
int (*vtab_on_conflict)(sqlite3*);
|
||||
/* Version 3.7.16 and later */
|
||||
int (*close_v2)(sqlite3*);
|
||||
const char *(*db_filename)(sqlite3*,const char*);
|
||||
int (*db_readonly)(sqlite3*,const char*);
|
||||
int (*db_release_memory)(sqlite3*);
|
||||
const char *(*errstr)(int);
|
||||
int (*stmt_busy)(sqlite3_stmt*);
|
||||
int (*stmt_readonly)(sqlite3_stmt*);
|
||||
int (*stricmp)(const char*,const char*);
|
||||
int (*uri_boolean)(const char*,const char*,int);
|
||||
sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64);
|
||||
const char *(*uri_parameter)(const char*,const char*);
|
||||
char *(*xvsnprintf)(int,char*,const char*,va_list);
|
||||
int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*);
|
||||
/* Version 3.8.7 and later */
|
||||
int (*auto_extension)(void(*)(void));
|
||||
int (*bind_blob64)(sqlite3_stmt*,int,const void*,sqlite3_uint64,
|
||||
void(*)(void*));
|
||||
int (*bind_text64)(sqlite3_stmt*,int,const char*,sqlite3_uint64,
|
||||
void(*)(void*),unsigned char);
|
||||
int (*cancel_auto_extension)(void(*)(void));
|
||||
int (*load_extension)(sqlite3*,const char*,const char*,char**);
|
||||
void *(*malloc64)(sqlite3_uint64);
|
||||
sqlite3_uint64 (*msize)(void*);
|
||||
void *(*realloc64)(void*,sqlite3_uint64);
|
||||
void (*reset_auto_extension)(void);
|
||||
void (*result_blob64)(sqlite3_context*,const void*,sqlite3_uint64,
|
||||
void(*)(void*));
|
||||
void (*result_text64)(sqlite3_context*,const char*,sqlite3_uint64,
|
||||
void(*)(void*), unsigned char);
|
||||
int (*strglob)(const char*,const char*);
|
||||
/* Version 3.8.11 and later */
|
||||
sqlite3_value *(*value_dup)(const sqlite3_value*);
|
||||
void (*value_free)(sqlite3_value*);
|
||||
int (*result_zeroblob64)(sqlite3_context*,sqlite3_uint64);
|
||||
int (*bind_zeroblob64)(sqlite3_stmt*, int, sqlite3_uint64);
|
||||
/* Version 3.9.0 and later */
|
||||
unsigned int (*value_subtype)(sqlite3_value*);
|
||||
void (*result_subtype)(sqlite3_context*,unsigned int);
|
||||
/* Version 3.10.0 and later */
|
||||
int (*status64)(int,sqlite3_int64*,sqlite3_int64*,int);
|
||||
int (*strlike)(const char*,const char*,unsigned int);
|
||||
int (*db_cacheflush)(sqlite3*);
|
||||
/* Version 3.12.0 and later */
|
||||
int (*system_errno)(sqlite3*);
|
||||
/* Version 3.14.0 and later */
|
||||
int (*trace_v2)(sqlite3*,unsigned,int(*)(unsigned,void*,void*,void*),void*);
|
||||
char *(*expanded_sql)(sqlite3_stmt*);
|
||||
/* Version 3.18.0 and later */
|
||||
void (*set_last_insert_rowid)(sqlite3*,sqlite3_int64);
|
||||
/* Version 3.20.0 and later */
|
||||
int (*prepare_v3)(sqlite3*,const char*,int,unsigned int,
|
||||
sqlite3_stmt**,const char**);
|
||||
int (*prepare16_v3)(sqlite3*,const void*,int,unsigned int,
|
||||
sqlite3_stmt**,const void**);
|
||||
int (*bind_pointer)(sqlite3_stmt*,int,void*,const char*,void(*)(void*));
|
||||
void (*result_pointer)(sqlite3_context*,void*,const char*,void(*)(void*));
|
||||
void *(*value_pointer)(sqlite3_value*,const char*);
|
||||
int (*vtab_nochange)(sqlite3_context*);
|
||||
int (*value_nochange)(sqlite3_value*);
|
||||
const char *(*vtab_collation)(sqlite3_index_info*,int);
|
||||
/* Version 3.24.0 and later */
|
||||
int (*keyword_count)(void);
|
||||
int (*keyword_name)(int,const char**,int*);
|
||||
int (*keyword_check)(const char*,int);
|
||||
sqlite3_str *(*str_new)(sqlite3*);
|
||||
char *(*str_finish)(sqlite3_str*);
|
||||
void (*str_appendf)(sqlite3_str*, const char *zFormat, ...);
|
||||
void (*str_vappendf)(sqlite3_str*, const char *zFormat, va_list);
|
||||
void (*str_append)(sqlite3_str*, const char *zIn, int N);
|
||||
void (*str_appendall)(sqlite3_str*, const char *zIn);
|
||||
void (*str_appendchar)(sqlite3_str*, int N, char C);
|
||||
void (*str_reset)(sqlite3_str*);
|
||||
int (*str_errcode)(sqlite3_str*);
|
||||
int (*str_length)(sqlite3_str*);
|
||||
char *(*str_value)(sqlite3_str*);
|
||||
/* Version 3.25.0 and later */
|
||||
int (*create_window_function)(sqlite3*,const char*,int,int,void*,
|
||||
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
|
||||
void (*xFinal)(sqlite3_context*),
|
||||
void (*xValue)(sqlite3_context*),
|
||||
void (*xInv)(sqlite3_context*,int,sqlite3_value**),
|
||||
void(*xDestroy)(void*));
|
||||
/* Version 3.26.0 and later */
|
||||
const char *(*normalized_sql)(sqlite3_stmt*);
|
||||
/* Version 3.28.0 and later */
|
||||
int (*stmt_isexplain)(sqlite3_stmt*);
|
||||
int (*value_frombind)(sqlite3_value*);
|
||||
/* Version 3.30.0 and later */
|
||||
int (*drop_modules)(sqlite3*,const char**);
|
||||
/* Version 3.31.0 and later */
|
||||
sqlite3_int64 (*hard_heap_limit64)(sqlite3_int64);
|
||||
const char *(*uri_key)(const char*,int);
|
||||
const char *(*filename_database)(const char*);
|
||||
const char *(*filename_journal)(const char*);
|
||||
const char *(*filename_wal)(const char*);
|
||||
/* Version 3.32.0 and later */
|
||||
const char *(*create_filename)(const char*,const char*,const char*,
|
||||
int,const char**);
|
||||
void (*free_filename)(const char*);
|
||||
sqlite3_file *(*database_file_object)(const char*);
|
||||
/* Version 3.34.0 and later */
|
||||
int (*txn_state)(sqlite3*,const char*);
|
||||
/* Version 3.36.1 and later */
|
||||
sqlite3_int64 (*changes64)(sqlite3*);
|
||||
sqlite3_int64 (*total_changes64)(sqlite3*);
|
||||
/* Version 3.37.0 and later */
|
||||
int (*autovacuum_pages)(sqlite3*,
|
||||
unsigned int(*)(void*,const char*,unsigned int,unsigned int,unsigned int),
|
||||
void*, void(*)(void*));
|
||||
/* Version 3.38.0 and later */
|
||||
int (*error_offset)(sqlite3*);
|
||||
int (*vtab_rhs_value)(sqlite3_index_info*,int,sqlite3_value**);
|
||||
int (*vtab_distinct)(sqlite3_index_info*);
|
||||
int (*vtab_in)(sqlite3_index_info*,int,int);
|
||||
int (*vtab_in_first)(sqlite3_value*,sqlite3_value**);
|
||||
int (*vtab_in_next)(sqlite3_value*,sqlite3_value**);
|
||||
/* Version 3.39.0 and later */
|
||||
int (*deserialize)(sqlite3*,const char*,unsigned char*,
|
||||
sqlite3_int64,sqlite3_int64,unsigned);
|
||||
unsigned char *(*serialize)(sqlite3*,const char *,sqlite3_int64*,
|
||||
unsigned int);
|
||||
const char *(*db_name)(sqlite3*,int);
|
||||
/* Version 3.40.0 and later */
|
||||
int (*value_encoding)(sqlite3_value*);
|
||||
/* Version 3.41.0 and later */
|
||||
int (*is_interrupted)(sqlite3*);
|
||||
/* Version 3.43.0 and later */
|
||||
int (*stmt_explain)(sqlite3_stmt*,int);
|
||||
/* Version 3.44.0 and later */
|
||||
void *(*get_clientdata)(sqlite3*,const char*);
|
||||
int (*set_clientdata)(sqlite3*, const char*, void*, void(*)(void*));
|
||||
/* Version 3.50.0 and later */
|
||||
int (*setlk_timeout)(sqlite3*,int,int);
|
||||
};
|
||||
|
||||
/*
|
||||
** This is the function signature used for all extension entry points. It
|
||||
** is also defined in the file "loadext.c".
|
||||
*/
|
||||
typedef int (*sqlite3_loadext_entry)(
|
||||
sqlite3 *db, /* Handle to the database. */
|
||||
char **pzErrMsg, /* Used to set error string on failure. */
|
||||
const sqlite3_api_routines *pThunk /* Extension API function pointers. */
|
||||
);
|
||||
|
||||
/*
|
||||
** The following macros redefine the API routines so that they are
|
||||
** redirected through the global sqlite3_api structure.
|
||||
**
|
||||
** This header file is also used by the loadext.c source file
|
||||
** (part of the main SQLite library - not an extension) so that
|
||||
** it can get access to the sqlite3_api_routines structure
|
||||
** definition. But the main library does not want to redefine
|
||||
** the API. So the redefinition macros are only valid if the
|
||||
** SQLITE_CORE macros is undefined.
|
||||
*/
|
||||
#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
|
||||
#define sqlite3_aggregate_context sqlite3_api->aggregate_context
|
||||
#ifndef SQLITE_OMIT_DEPRECATED
|
||||
#define sqlite3_aggregate_count sqlite3_api->aggregate_count
|
||||
#endif
|
||||
#define sqlite3_bind_blob sqlite3_api->bind_blob
|
||||
#define sqlite3_bind_double sqlite3_api->bind_double
|
||||
#define sqlite3_bind_int sqlite3_api->bind_int
|
||||
#define sqlite3_bind_int64 sqlite3_api->bind_int64
|
||||
#define sqlite3_bind_null sqlite3_api->bind_null
|
||||
#define sqlite3_bind_parameter_count sqlite3_api->bind_parameter_count
|
||||
#define sqlite3_bind_parameter_index sqlite3_api->bind_parameter_index
|
||||
#define sqlite3_bind_parameter_name sqlite3_api->bind_parameter_name
|
||||
#define sqlite3_bind_text sqlite3_api->bind_text
|
||||
#define sqlite3_bind_text16 sqlite3_api->bind_text16
|
||||
#define sqlite3_bind_value sqlite3_api->bind_value
|
||||
#define sqlite3_busy_handler sqlite3_api->busy_handler
|
||||
#define sqlite3_busy_timeout sqlite3_api->busy_timeout
|
||||
#define sqlite3_changes sqlite3_api->changes
|
||||
#define sqlite3_close sqlite3_api->close
|
||||
#define sqlite3_collation_needed sqlite3_api->collation_needed
|
||||
#define sqlite3_collation_needed16 sqlite3_api->collation_needed16
|
||||
#define sqlite3_column_blob sqlite3_api->column_blob
|
||||
#define sqlite3_column_bytes sqlite3_api->column_bytes
|
||||
#define sqlite3_column_bytes16 sqlite3_api->column_bytes16
|
||||
#define sqlite3_column_count sqlite3_api->column_count
|
||||
#define sqlite3_column_database_name sqlite3_api->column_database_name
|
||||
#define sqlite3_column_database_name16 sqlite3_api->column_database_name16
|
||||
#define sqlite3_column_decltype sqlite3_api->column_decltype
|
||||
#define sqlite3_column_decltype16 sqlite3_api->column_decltype16
|
||||
#define sqlite3_column_double sqlite3_api->column_double
|
||||
#define sqlite3_column_int sqlite3_api->column_int
|
||||
#define sqlite3_column_int64 sqlite3_api->column_int64
|
||||
#define sqlite3_column_name sqlite3_api->column_name
|
||||
#define sqlite3_column_name16 sqlite3_api->column_name16
|
||||
#define sqlite3_column_origin_name sqlite3_api->column_origin_name
|
||||
#define sqlite3_column_origin_name16 sqlite3_api->column_origin_name16
|
||||
#define sqlite3_column_table_name sqlite3_api->column_table_name
|
||||
#define sqlite3_column_table_name16 sqlite3_api->column_table_name16
|
||||
#define sqlite3_column_text sqlite3_api->column_text
|
||||
#define sqlite3_column_text16 sqlite3_api->column_text16
|
||||
#define sqlite3_column_type sqlite3_api->column_type
|
||||
#define sqlite3_column_value sqlite3_api->column_value
|
||||
#define sqlite3_commit_hook sqlite3_api->commit_hook
|
||||
#define sqlite3_complete sqlite3_api->complete
|
||||
#define sqlite3_complete16 sqlite3_api->complete16
|
||||
#define sqlite3_create_collation sqlite3_api->create_collation
|
||||
#define sqlite3_create_collation16 sqlite3_api->create_collation16
|
||||
#define sqlite3_create_function sqlite3_api->create_function
|
||||
#define sqlite3_create_function16 sqlite3_api->create_function16
|
||||
#define sqlite3_create_module sqlite3_api->create_module
|
||||
#define sqlite3_create_module_v2 sqlite3_api->create_module_v2
|
||||
#define sqlite3_data_count sqlite3_api->data_count
|
||||
#define sqlite3_db_handle sqlite3_api->db_handle
|
||||
#define sqlite3_declare_vtab sqlite3_api->declare_vtab
|
||||
#define sqlite3_enable_shared_cache sqlite3_api->enable_shared_cache
|
||||
#define sqlite3_errcode sqlite3_api->errcode
|
||||
#define sqlite3_errmsg sqlite3_api->errmsg
|
||||
#define sqlite3_errmsg16 sqlite3_api->errmsg16
|
||||
#define sqlite3_exec sqlite3_api->exec
|
||||
#ifndef SQLITE_OMIT_DEPRECATED
|
||||
#define sqlite3_expired sqlite3_api->expired
|
||||
#endif
|
||||
#define sqlite3_finalize sqlite3_api->finalize
|
||||
#define sqlite3_free sqlite3_api->free
|
||||
#define sqlite3_free_table sqlite3_api->free_table
|
||||
#define sqlite3_get_autocommit sqlite3_api->get_autocommit
|
||||
#define sqlite3_get_auxdata sqlite3_api->get_auxdata
|
||||
#define sqlite3_get_table sqlite3_api->get_table
|
||||
#ifndef SQLITE_OMIT_DEPRECATED
|
||||
#define sqlite3_global_recover sqlite3_api->global_recover
|
||||
#endif
|
||||
#define sqlite3_interrupt sqlite3_api->interruptx
|
||||
#define sqlite3_last_insert_rowid sqlite3_api->last_insert_rowid
|
||||
#define sqlite3_libversion sqlite3_api->libversion
|
||||
#define sqlite3_libversion_number sqlite3_api->libversion_number
|
||||
#define sqlite3_malloc sqlite3_api->malloc
|
||||
#define sqlite3_mprintf sqlite3_api->mprintf
|
||||
#define sqlite3_open sqlite3_api->open
|
||||
#define sqlite3_open16 sqlite3_api->open16
|
||||
#define sqlite3_prepare sqlite3_api->prepare
|
||||
#define sqlite3_prepare16 sqlite3_api->prepare16
|
||||
#define sqlite3_prepare_v2 sqlite3_api->prepare_v2
|
||||
#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2
|
||||
#define sqlite3_profile sqlite3_api->profile
|
||||
#define sqlite3_progress_handler sqlite3_api->progress_handler
|
||||
#define sqlite3_realloc sqlite3_api->realloc
|
||||
#define sqlite3_reset sqlite3_api->reset
|
||||
#define sqlite3_result_blob sqlite3_api->result_blob
|
||||
#define sqlite3_result_double sqlite3_api->result_double
|
||||
#define sqlite3_result_error sqlite3_api->result_error
|
||||
#define sqlite3_result_error16 sqlite3_api->result_error16
|
||||
#define sqlite3_result_int sqlite3_api->result_int
|
||||
#define sqlite3_result_int64 sqlite3_api->result_int64
|
||||
#define sqlite3_result_null sqlite3_api->result_null
|
||||
#define sqlite3_result_text sqlite3_api->result_text
|
||||
#define sqlite3_result_text16 sqlite3_api->result_text16
|
||||
#define sqlite3_result_text16be sqlite3_api->result_text16be
|
||||
#define sqlite3_result_text16le sqlite3_api->result_text16le
|
||||
#define sqlite3_result_value sqlite3_api->result_value
|
||||
#define sqlite3_rollback_hook sqlite3_api->rollback_hook
|
||||
#define sqlite3_set_authorizer sqlite3_api->set_authorizer
|
||||
#define sqlite3_set_auxdata sqlite3_api->set_auxdata
|
||||
#define sqlite3_snprintf sqlite3_api->xsnprintf
|
||||
#define sqlite3_step sqlite3_api->step
|
||||
#define sqlite3_table_column_metadata sqlite3_api->table_column_metadata
|
||||
#define sqlite3_thread_cleanup sqlite3_api->thread_cleanup
|
||||
#define sqlite3_total_changes sqlite3_api->total_changes
|
||||
#define sqlite3_trace sqlite3_api->trace
|
||||
#ifndef SQLITE_OMIT_DEPRECATED
|
||||
#define sqlite3_transfer_bindings sqlite3_api->transfer_bindings
|
||||
#endif
|
||||
#define sqlite3_update_hook sqlite3_api->update_hook
|
||||
#define sqlite3_user_data sqlite3_api->user_data
|
||||
#define sqlite3_value_blob sqlite3_api->value_blob
|
||||
#define sqlite3_value_bytes sqlite3_api->value_bytes
|
||||
#define sqlite3_value_bytes16 sqlite3_api->value_bytes16
|
||||
#define sqlite3_value_double sqlite3_api->value_double
|
||||
#define sqlite3_value_int sqlite3_api->value_int
|
||||
#define sqlite3_value_int64 sqlite3_api->value_int64
|
||||
#define sqlite3_value_numeric_type sqlite3_api->value_numeric_type
|
||||
#define sqlite3_value_text sqlite3_api->value_text
|
||||
#define sqlite3_value_text16 sqlite3_api->value_text16
|
||||
#define sqlite3_value_text16be sqlite3_api->value_text16be
|
||||
#define sqlite3_value_text16le sqlite3_api->value_text16le
|
||||
#define sqlite3_value_type sqlite3_api->value_type
|
||||
#define sqlite3_vmprintf sqlite3_api->vmprintf
|
||||
#define sqlite3_vsnprintf sqlite3_api->xvsnprintf
|
||||
#define sqlite3_overload_function sqlite3_api->overload_function
|
||||
#define sqlite3_prepare_v2 sqlite3_api->prepare_v2
|
||||
#define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2
|
||||
#define sqlite3_clear_bindings sqlite3_api->clear_bindings
|
||||
#define sqlite3_bind_zeroblob sqlite3_api->bind_zeroblob
|
||||
#define sqlite3_blob_bytes sqlite3_api->blob_bytes
|
||||
#define sqlite3_blob_close sqlite3_api->blob_close
|
||||
#define sqlite3_blob_open sqlite3_api->blob_open
|
||||
#define sqlite3_blob_read sqlite3_api->blob_read
|
||||
#define sqlite3_blob_write sqlite3_api->blob_write
|
||||
#define sqlite3_create_collation_v2 sqlite3_api->create_collation_v2
|
||||
#define sqlite3_file_control sqlite3_api->file_control
|
||||
#define sqlite3_memory_highwater sqlite3_api->memory_highwater
|
||||
#define sqlite3_memory_used sqlite3_api->memory_used
|
||||
#define sqlite3_mutex_alloc sqlite3_api->mutex_alloc
|
||||
#define sqlite3_mutex_enter sqlite3_api->mutex_enter
|
||||
#define sqlite3_mutex_free sqlite3_api->mutex_free
|
||||
#define sqlite3_mutex_leave sqlite3_api->mutex_leave
|
||||
#define sqlite3_mutex_try sqlite3_api->mutex_try
|
||||
#define sqlite3_open_v2 sqlite3_api->open_v2
|
||||
#define sqlite3_release_memory sqlite3_api->release_memory
|
||||
#define sqlite3_result_error_nomem sqlite3_api->result_error_nomem
|
||||
#define sqlite3_result_error_toobig sqlite3_api->result_error_toobig
|
||||
#define sqlite3_sleep sqlite3_api->sleep
|
||||
#define sqlite3_soft_heap_limit sqlite3_api->soft_heap_limit
|
||||
#define sqlite3_vfs_find sqlite3_api->vfs_find
|
||||
#define sqlite3_vfs_register sqlite3_api->vfs_register
|
||||
#define sqlite3_vfs_unregister sqlite3_api->vfs_unregister
|
||||
#define sqlite3_threadsafe sqlite3_api->xthreadsafe
|
||||
#define sqlite3_result_zeroblob sqlite3_api->result_zeroblob
|
||||
#define sqlite3_result_error_code sqlite3_api->result_error_code
|
||||
#define sqlite3_test_control sqlite3_api->test_control
|
||||
#define sqlite3_randomness sqlite3_api->randomness
|
||||
#define sqlite3_context_db_handle sqlite3_api->context_db_handle
|
||||
#define sqlite3_extended_result_codes sqlite3_api->extended_result_codes
|
||||
#define sqlite3_limit sqlite3_api->limit
|
||||
#define sqlite3_next_stmt sqlite3_api->next_stmt
|
||||
#define sqlite3_sql sqlite3_api->sql
|
||||
#define sqlite3_status sqlite3_api->status
|
||||
#define sqlite3_backup_finish sqlite3_api->backup_finish
|
||||
#define sqlite3_backup_init sqlite3_api->backup_init
|
||||
#define sqlite3_backup_pagecount sqlite3_api->backup_pagecount
|
||||
#define sqlite3_backup_remaining sqlite3_api->backup_remaining
|
||||
#define sqlite3_backup_step sqlite3_api->backup_step
|
||||
#define sqlite3_compileoption_get sqlite3_api->compileoption_get
|
||||
#define sqlite3_compileoption_used sqlite3_api->compileoption_used
|
||||
#define sqlite3_create_function_v2 sqlite3_api->create_function_v2
|
||||
#define sqlite3_db_config sqlite3_api->db_config
|
||||
#define sqlite3_db_mutex sqlite3_api->db_mutex
|
||||
#define sqlite3_db_status sqlite3_api->db_status
|
||||
#define sqlite3_extended_errcode sqlite3_api->extended_errcode
|
||||
#define sqlite3_log sqlite3_api->log
|
||||
#define sqlite3_soft_heap_limit64 sqlite3_api->soft_heap_limit64
|
||||
#define sqlite3_sourceid sqlite3_api->sourceid
|
||||
#define sqlite3_stmt_status sqlite3_api->stmt_status
|
||||
#define sqlite3_strnicmp sqlite3_api->strnicmp
|
||||
#define sqlite3_unlock_notify sqlite3_api->unlock_notify
|
||||
#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint
|
||||
#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint
|
||||
#define sqlite3_wal_hook sqlite3_api->wal_hook
|
||||
#define sqlite3_blob_reopen sqlite3_api->blob_reopen
|
||||
#define sqlite3_vtab_config sqlite3_api->vtab_config
|
||||
#define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict
|
||||
/* Version 3.7.16 and later */
|
||||
#define sqlite3_close_v2 sqlite3_api->close_v2
|
||||
#define sqlite3_db_filename sqlite3_api->db_filename
|
||||
#define sqlite3_db_readonly sqlite3_api->db_readonly
|
||||
#define sqlite3_db_release_memory sqlite3_api->db_release_memory
|
||||
#define sqlite3_errstr sqlite3_api->errstr
|
||||
#define sqlite3_stmt_busy sqlite3_api->stmt_busy
|
||||
#define sqlite3_stmt_readonly sqlite3_api->stmt_readonly
|
||||
#define sqlite3_stricmp sqlite3_api->stricmp
|
||||
#define sqlite3_uri_boolean sqlite3_api->uri_boolean
|
||||
#define sqlite3_uri_int64 sqlite3_api->uri_int64
|
||||
#define sqlite3_uri_parameter sqlite3_api->uri_parameter
|
||||
#define sqlite3_uri_vsnprintf sqlite3_api->xvsnprintf
|
||||
#define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2
|
||||
/* Version 3.8.7 and later */
|
||||
#define sqlite3_auto_extension sqlite3_api->auto_extension
|
||||
#define sqlite3_bind_blob64 sqlite3_api->bind_blob64
|
||||
#define sqlite3_bind_text64 sqlite3_api->bind_text64
|
||||
#define sqlite3_cancel_auto_extension sqlite3_api->cancel_auto_extension
|
||||
#define sqlite3_load_extension sqlite3_api->load_extension
|
||||
#define sqlite3_malloc64 sqlite3_api->malloc64
|
||||
#define sqlite3_msize sqlite3_api->msize
|
||||
#define sqlite3_realloc64 sqlite3_api->realloc64
|
||||
#define sqlite3_reset_auto_extension sqlite3_api->reset_auto_extension
|
||||
#define sqlite3_result_blob64 sqlite3_api->result_blob64
|
||||
#define sqlite3_result_text64 sqlite3_api->result_text64
|
||||
#define sqlite3_strglob sqlite3_api->strglob
|
||||
/* Version 3.8.11 and later */
|
||||
#define sqlite3_value_dup sqlite3_api->value_dup
|
||||
#define sqlite3_value_free sqlite3_api->value_free
|
||||
#define sqlite3_result_zeroblob64 sqlite3_api->result_zeroblob64
|
||||
#define sqlite3_bind_zeroblob64 sqlite3_api->bind_zeroblob64
|
||||
/* Version 3.9.0 and later */
|
||||
#define sqlite3_value_subtype sqlite3_api->value_subtype
|
||||
#define sqlite3_result_subtype sqlite3_api->result_subtype
|
||||
/* Version 3.10.0 and later */
|
||||
#define sqlite3_status64 sqlite3_api->status64
|
||||
#define sqlite3_strlike sqlite3_api->strlike
|
||||
#define sqlite3_db_cacheflush sqlite3_api->db_cacheflush
|
||||
/* Version 3.12.0 and later */
|
||||
#define sqlite3_system_errno sqlite3_api->system_errno
|
||||
/* Version 3.14.0 and later */
|
||||
#define sqlite3_trace_v2 sqlite3_api->trace_v2
|
||||
#define sqlite3_expanded_sql sqlite3_api->expanded_sql
|
||||
/* Version 3.18.0 and later */
|
||||
#define sqlite3_set_last_insert_rowid sqlite3_api->set_last_insert_rowid
|
||||
/* Version 3.20.0 and later */
|
||||
#define sqlite3_prepare_v3 sqlite3_api->prepare_v3
|
||||
#define sqlite3_prepare16_v3 sqlite3_api->prepare16_v3
|
||||
#define sqlite3_bind_pointer sqlite3_api->bind_pointer
|
||||
#define sqlite3_result_pointer sqlite3_api->result_pointer
|
||||
#define sqlite3_value_pointer sqlite3_api->value_pointer
|
||||
/* Version 3.22.0 and later */
|
||||
#define sqlite3_vtab_nochange sqlite3_api->vtab_nochange
|
||||
#define sqlite3_value_nochange sqlite3_api->value_nochange
|
||||
#define sqlite3_vtab_collation sqlite3_api->vtab_collation
|
||||
/* Version 3.24.0 and later */
|
||||
#define sqlite3_keyword_count sqlite3_api->keyword_count
|
||||
#define sqlite3_keyword_name sqlite3_api->keyword_name
|
||||
#define sqlite3_keyword_check sqlite3_api->keyword_check
|
||||
#define sqlite3_str_new sqlite3_api->str_new
|
||||
#define sqlite3_str_finish sqlite3_api->str_finish
|
||||
#define sqlite3_str_appendf sqlite3_api->str_appendf
|
||||
#define sqlite3_str_vappendf sqlite3_api->str_vappendf
|
||||
#define sqlite3_str_append sqlite3_api->str_append
|
||||
#define sqlite3_str_appendall sqlite3_api->str_appendall
|
||||
#define sqlite3_str_appendchar sqlite3_api->str_appendchar
|
||||
#define sqlite3_str_reset sqlite3_api->str_reset
|
||||
#define sqlite3_str_errcode sqlite3_api->str_errcode
|
||||
#define sqlite3_str_length sqlite3_api->str_length
|
||||
#define sqlite3_str_value sqlite3_api->str_value
|
||||
/* Version 3.25.0 and later */
|
||||
#define sqlite3_create_window_function sqlite3_api->create_window_function
|
||||
/* Version 3.26.0 and later */
|
||||
#define sqlite3_normalized_sql sqlite3_api->normalized_sql
|
||||
/* Version 3.28.0 and later */
|
||||
#define sqlite3_stmt_isexplain sqlite3_api->stmt_isexplain
|
||||
#define sqlite3_value_frombind sqlite3_api->value_frombind
|
||||
/* Version 3.30.0 and later */
|
||||
#define sqlite3_drop_modules sqlite3_api->drop_modules
|
||||
/* Version 3.31.0 and later */
|
||||
#define sqlite3_hard_heap_limit64 sqlite3_api->hard_heap_limit64
|
||||
#define sqlite3_uri_key sqlite3_api->uri_key
|
||||
#define sqlite3_filename_database sqlite3_api->filename_database
|
||||
#define sqlite3_filename_journal sqlite3_api->filename_journal
|
||||
#define sqlite3_filename_wal sqlite3_api->filename_wal
|
||||
/* Version 3.32.0 and later */
|
||||
#define sqlite3_create_filename sqlite3_api->create_filename
|
||||
#define sqlite3_free_filename sqlite3_api->free_filename
|
||||
#define sqlite3_database_file_object sqlite3_api->database_file_object
|
||||
/* Version 3.34.0 and later */
|
||||
#define sqlite3_txn_state sqlite3_api->txn_state
|
||||
/* Version 3.36.1 and later */
|
||||
#define sqlite3_changes64 sqlite3_api->changes64
|
||||
#define sqlite3_total_changes64 sqlite3_api->total_changes64
|
||||
/* Version 3.37.0 and later */
|
||||
#define sqlite3_autovacuum_pages sqlite3_api->autovacuum_pages
|
||||
/* Version 3.38.0 and later */
|
||||
#define sqlite3_error_offset sqlite3_api->error_offset
|
||||
#define sqlite3_vtab_rhs_value sqlite3_api->vtab_rhs_value
|
||||
#define sqlite3_vtab_distinct sqlite3_api->vtab_distinct
|
||||
#define sqlite3_vtab_in sqlite3_api->vtab_in
|
||||
#define sqlite3_vtab_in_first sqlite3_api->vtab_in_first
|
||||
#define sqlite3_vtab_in_next sqlite3_api->vtab_in_next
|
||||
/* Version 3.39.0 and later */
|
||||
#ifndef SQLITE_OMIT_DESERIALIZE
|
||||
#define sqlite3_deserialize sqlite3_api->deserialize
|
||||
#define sqlite3_serialize sqlite3_api->serialize
|
||||
#endif
|
||||
#define sqlite3_db_name sqlite3_api->db_name
|
||||
/* Version 3.40.0 and later */
|
||||
#define sqlite3_value_encoding sqlite3_api->value_encoding
|
||||
/* Version 3.41.0 and later */
|
||||
#define sqlite3_is_interrupted sqlite3_api->is_interrupted
|
||||
/* Version 3.43.0 and later */
|
||||
#define sqlite3_stmt_explain sqlite3_api->stmt_explain
|
||||
/* Version 3.44.0 and later */
|
||||
#define sqlite3_get_clientdata sqlite3_api->get_clientdata
|
||||
#define sqlite3_set_clientdata sqlite3_api->set_clientdata
|
||||
/* Version 3.50.0 and later */
|
||||
#define sqlite3_setlk_timeout sqlite3_api->setlk_timeout
|
||||
#endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */
|
||||
|
||||
#if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION)
|
||||
/* This case when the file really is being compiled as a loadable
|
||||
** extension */
|
||||
# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0;
|
||||
# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v;
|
||||
# define SQLITE_EXTENSION_INIT3 \
|
||||
extern const sqlite3_api_routines *sqlite3_api;
|
||||
#else
|
||||
/* This case when the file is being statically linked into the
|
||||
** application */
|
||||
# define SQLITE_EXTENSION_INIT1 /*no-op*/
|
||||
# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */
|
||||
# define SQLITE_EXTENSION_INIT3 /*no-op*/
|
||||
#endif
|
||||
|
||||
#endif /* SQLITE3EXT_H */
|
||||
@@ -0,0 +1,70 @@
|
||||
"""config.yaml sessions.* bridges for the search-index knobs (config-authoritative).
|
||||
|
||||
Salvaged from PR #65544 (adapted: agent.fts_v2_read → sessions.cjk_fts).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
import gateway.run as gateway_run
|
||||
|
||||
|
||||
def _write_home(tmp_path: Path, sessions_cfg: dict, env_text: str = "") -> Path:
|
||||
hermes_home = tmp_path / ".hermes"
|
||||
hermes_home.mkdir()
|
||||
(hermes_home / "config.yaml").write_text(
|
||||
yaml.safe_dump({"sessions": sessions_cfg}), encoding="utf-8"
|
||||
)
|
||||
(hermes_home / ".env").write_text(env_text, encoding="utf-8")
|
||||
return hermes_home
|
||||
|
||||
|
||||
def test_cjk_fts_bridged_from_config(tmp_path, monkeypatch):
|
||||
home = _write_home(tmp_path, {"cjk_fts": False})
|
||||
monkeypatch.setattr(gateway_run, "_hermes_home", home)
|
||||
monkeypatch.setenv("HERMES_CJK_FTS", "1")
|
||||
gateway_run._reload_runtime_env_preserving_config_authority()
|
||||
assert os.environ["HERMES_CJK_FTS"] == "False"
|
||||
|
||||
|
||||
def test_search_slow_ms_bridged_from_config(tmp_path, monkeypatch):
|
||||
home = _write_home(tmp_path, {"search_slow_ms": 250})
|
||||
monkeypatch.setattr(gateway_run, "_hermes_home", home)
|
||||
monkeypatch.delenv("HERMES_SEARCH_SLOW_MS", raising=False)
|
||||
gateway_run._reload_runtime_env_preserving_config_authority()
|
||||
assert os.environ["HERMES_SEARCH_SLOW_MS"] == "250"
|
||||
|
||||
|
||||
def test_env_survives_when_config_omits_search_knobs(tmp_path, monkeypatch):
|
||||
home = _write_home(tmp_path, {"auto_prune": False})
|
||||
monkeypatch.setattr(gateway_run, "_hermes_home", home)
|
||||
monkeypatch.setenv("HERMES_CJK_FTS", "0")
|
||||
monkeypatch.setenv("HERMES_SEARCH_SLOW_MS", "700")
|
||||
gateway_run._reload_runtime_env_preserving_config_authority()
|
||||
assert os.environ["HERMES_CJK_FTS"] == "0"
|
||||
assert os.environ["HERMES_SEARCH_SLOW_MS"] == "700"
|
||||
|
||||
|
||||
def test_search_knobs_have_documented_defaults():
|
||||
"""The advertised config surface must exist in DEFAULT_CONFIG (no
|
||||
user-facing env switch): cjk index default ON, slow-search log at 1s."""
|
||||
from hermes_cli.config import DEFAULT_CONFIG
|
||||
|
||||
assert DEFAULT_CONFIG["sessions"]["cjk_fts"] is True
|
||||
assert DEFAULT_CONFIG["sessions"]["search_slow_ms"] == 1000
|
||||
|
||||
|
||||
def test_config_false_disables_cjk_semantics(tmp_path, monkeypatch):
|
||||
"""The bridged 'False' string must parse as OFF in hermes_state."""
|
||||
from hermes_state import _cjk_fts_config_enabled
|
||||
|
||||
monkeypatch.setenv("HERMES_CJK_FTS", "False")
|
||||
assert not _cjk_fts_config_enabled()
|
||||
monkeypatch.setenv("HERMES_CJK_FTS", "True")
|
||||
assert _cjk_fts_config_enabled()
|
||||
monkeypatch.delenv("HERMES_CJK_FTS", raising=False)
|
||||
assert _cjk_fts_config_enabled() # default on
|
||||
@@ -0,0 +1,325 @@
|
||||
"""Tests for the messages_fts_cjk CJK-bigram index (salvaged from PR #65544).
|
||||
|
||||
Builds the loadable tokenizer from native/fts5_cjk/fts5_cjk.c on the fly;
|
||||
skips when no C toolchain / extension loading is available.
|
||||
"""
|
||||
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_state import FTS_CJK_STALE_KEY, SessionDB
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
SRC = REPO / "native" / "fts5_cjk" / "fts5_cjk.c"
|
||||
VENDOR = REPO / "native" / "fts5_cjk" / "vendor"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def cjk_so(tmp_path_factory):
|
||||
if shutil.which("gcc") is None or not SRC.exists():
|
||||
pytest.skip("no C toolchain / tokenizer source")
|
||||
out = tmp_path_factory.mktemp("fts5cjk") / "libfts5_cjk.so"
|
||||
try:
|
||||
subprocess.run(
|
||||
["gcc", "-shared", "-fPIC", "-O2", f"-I{VENDOR}", str(SRC),
|
||||
"-o", str(out)],
|
||||
check=True, capture_output=True, text=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
pytest.skip(f"tokenizer build failed: {e.stderr[:200]}")
|
||||
# Loadability probe (extension loading may be disabled in this build).
|
||||
probe = sqlite3.connect(":memory:")
|
||||
try:
|
||||
probe.enable_load_extension(True)
|
||||
probe.load_extension(str(out))
|
||||
except Exception as e:
|
||||
pytest.skip(f"extension loading unavailable: {e}")
|
||||
finally:
|
||||
probe.close()
|
||||
return out
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def db(cjk_so, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
d = SessionDB(db_path=tmp_path / "state.db")
|
||||
assert d._fts_cjk_loaded, "tokenizer must load on the writer connection"
|
||||
assert d._fts_cjk_available, "fresh DB must be born with the cjk index"
|
||||
d.create_session(session_id="s1", source="cli", model="m")
|
||||
d.append_message("s1", role="user", content="웅기가 shared default 프로필을 요청했다")
|
||||
d.append_message("s1", role="assistant", content="일본 MCP 후보 우선순위 정리했습니다")
|
||||
d.append_message("s1", role="user", content="graphiti daemon looks healthy")
|
||||
d.append_message("s1", role="tool", content="일본 tool output blob", tool_name="terminal")
|
||||
yield d
|
||||
d.close()
|
||||
|
||||
|
||||
def test_two_char_korean_hits_cjk_index(db):
|
||||
rows = db.search_messages("웅기", limit=10)
|
||||
assert rows and "웅기" in rows[0]["snippet"]
|
||||
rows = db.search_messages("일본", limit=10)
|
||||
assert rows
|
||||
|
||||
|
||||
def test_mixed_and_ascii_queries(db):
|
||||
assert db.search_messages("graphiti", limit=10)
|
||||
assert db.search_messages('"shared default" AND 웅기', limit=10)
|
||||
assert db.search_messages("우선순위", limit=10)
|
||||
|
||||
|
||||
def test_no_false_positive_across_words(db):
|
||||
# 기가/했다 exist inside runs; a bigram crossing a word boundary must not.
|
||||
assert db.search_messages("다프로", limit=10) == []
|
||||
|
||||
|
||||
def test_lone_single_cjk_char_routes_like(db):
|
||||
# 1-char CJK terms keep LIKE substring semantics (bigram index only
|
||||
# holds unigrams for isolated chars). "가" appears inside 웅기가.
|
||||
assert db._describe_search_path("가") == "like_scan"
|
||||
rows = db.search_messages("가", limit=10)
|
||||
assert rows, "LIKE fallback must still find substring matches"
|
||||
|
||||
|
||||
def test_tool_role_filter_routes_like(db):
|
||||
# Tool rows are excluded from the cjk index; role_filter=['tool'] CJK
|
||||
# queries take the LIKE route and still find tool output.
|
||||
rows = db.search_messages("일본", role_filter=["tool"], limit=10)
|
||||
assert rows and all(r["role"] == "tool" for r in rows)
|
||||
|
||||
|
||||
def test_triggers_mirror_updates_and_deletes(db):
|
||||
db.append_message("s1", role="user", content="자바스크립트 리팩토링")
|
||||
assert db.search_messages("리팩토링", limit=10)
|
||||
with db._lock:
|
||||
db._conn.execute(
|
||||
"UPDATE messages SET content = '파이썬 리라이트' WHERE content LIKE '%리팩토링%'"
|
||||
)
|
||||
db._conn.commit()
|
||||
assert db.search_messages("리팩토링", limit=10) == []
|
||||
assert db.search_messages("리라이트", limit=10)
|
||||
with db._lock:
|
||||
db._conn.execute("DELETE FROM messages WHERE content = '파이썬 리라이트'")
|
||||
db._conn.commit()
|
||||
assert db.search_messages("리라이트", limit=10) == []
|
||||
|
||||
|
||||
def test_rewound_rows_hidden_from_cjk_search(db):
|
||||
db.append_message("s1", role="user", content="되돌리기 대상 메시지")
|
||||
assert db.search_messages("되돌리기", limit=10)
|
||||
with db._lock:
|
||||
db._conn.execute(
|
||||
"UPDATE messages SET active = 0, compacted = 0 "
|
||||
"WHERE content LIKE '%되돌리기%'"
|
||||
)
|
||||
db._conn.commit()
|
||||
assert db.search_messages("되돌리기", limit=10) == []
|
||||
assert db.search_messages("되돌리기", include_inactive=True, limit=10)
|
||||
|
||||
|
||||
def test_config_toggle_disables_cjk(cjk_so, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
monkeypatch.setenv("HERMES_CJK_FTS", "0")
|
||||
d = SessionDB(db_path=tmp_path / "state.db")
|
||||
try:
|
||||
assert not d._fts_cjk_loaded
|
||||
assert not d._fts_cjk_available
|
||||
# No cjk objects created at all.
|
||||
with d._lock:
|
||||
row = d._conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE name = 'messages_fts_cjk'"
|
||||
).fetchone()
|
||||
assert row is None
|
||||
finally:
|
||||
d.close()
|
||||
|
||||
|
||||
def test_no_extension_no_cjk_objects(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(tmp_path / "nonexistent.so"))
|
||||
d = SessionDB(db_path=tmp_path / "state.db")
|
||||
try:
|
||||
assert not d._fts_cjk_loaded
|
||||
assert not d._fts_cjk_available
|
||||
d.create_session(session_id="s1", source="cli", model="m")
|
||||
d.append_message("s1", role="user", content="일본 MCP 정리")
|
||||
# Trigram/LIKE routing still answers.
|
||||
assert d.search_messages("일본", limit=10)
|
||||
finally:
|
||||
d.close()
|
||||
|
||||
|
||||
def test_tokenizer_loss_self_heals_and_optimize_rebuilds(cjk_so, tmp_path, monkeypatch):
|
||||
"""Full stale lifecycle: capable open → tokenizer-less open (drops
|
||||
triggers, breadcrumbs) → rows written in the gap → capable open again
|
||||
(index NOT served) → optimize-storage rebuilds → search complete."""
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
db_path = tmp_path / "state.db"
|
||||
|
||||
d1 = SessionDB(db_path=db_path)
|
||||
assert d1._fts_cjk_available
|
||||
d1.create_session(session_id="s1", source="cli", model="m")
|
||||
d1.append_message("s1", role="user", content="첫번째 메시지")
|
||||
d1.close()
|
||||
|
||||
# Tokenizer-less open: triggers dropped, breadcrumb set, writes fine.
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(tmp_path / "gone.so"))
|
||||
d2 = SessionDB(db_path=db_path)
|
||||
assert not d2._fts_cjk_loaded
|
||||
assert not d2._fts_cjk_available
|
||||
d2.append_message("s1", role="user", content="틈새에 쓰인 메시지")
|
||||
assert d2.get_meta(FTS_CJK_STALE_KEY) == "1"
|
||||
with d2._lock:
|
||||
trigs = d2._conn.execute(
|
||||
"SELECT COUNT(*) FROM sqlite_master WHERE type='trigger' "
|
||||
"AND name LIKE 'messages_fts_cjk%'"
|
||||
).fetchone()[0]
|
||||
assert trigs == 0
|
||||
# Search still answers via trigram/LIKE.
|
||||
assert d2.search_messages("틈새", limit=10)
|
||||
d2.close()
|
||||
|
||||
# Capable open again: stale index must NOT be served.
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
d3 = SessionDB(db_path=db_path)
|
||||
assert d3._fts_cjk_loaded
|
||||
assert not d3._fts_cjk_available, "stale index must not serve reads"
|
||||
assert d3.fts_optimize_available(), "optimize must offer the rebuild"
|
||||
# Search still complete through the legacy routes meanwhile.
|
||||
assert d3.search_messages("틈새", limit=10)
|
||||
|
||||
result = d3.optimize_fts_storage(vacuum=False)
|
||||
assert result["ok"]
|
||||
assert d3._fts_cjk_available
|
||||
assert d3.get_meta(FTS_CJK_STALE_KEY) is None
|
||||
assert d3.fts_cjk_rebuild_status() is None
|
||||
# Both the pre-gap and in-gap rows are now searchable via the index.
|
||||
assert d3._describe_search_path("틈새") == "fts_cjk"
|
||||
assert d3.search_messages("첫번째", limit=10)
|
||||
assert d3.search_messages("틈새", limit=10)
|
||||
d3.close()
|
||||
|
||||
|
||||
def test_existing_v23_db_gains_cjk_via_optimize(cjk_so, tmp_path, monkeypatch):
|
||||
"""A v23 DB created BEFORE the extension existed: next capable open
|
||||
creates the index with backfill markers; optimize-storage backfills."""
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(tmp_path / "absent.so"))
|
||||
db_path = tmp_path / "state.db"
|
||||
d1 = SessionDB(db_path=db_path)
|
||||
d1.create_session(session_id="s1", source="cli", model="m")
|
||||
for i in range(10):
|
||||
d1.append_message("s1", role="user", content=f"기존 메시지 {i}")
|
||||
d1.close()
|
||||
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
d2 = SessionDB(db_path=db_path)
|
||||
assert d2._fts_cjk_loaded
|
||||
# Backfill pending — index not served yet, old rows not indexed.
|
||||
assert not d2._fts_cjk_available
|
||||
st = d2.fts_cjk_rebuild_status()
|
||||
assert st is not None and st["pending"]
|
||||
assert d2.fts_optimize_available()
|
||||
# NEW rows are indexed live by the id-gated triggers even mid-backfill.
|
||||
d2.append_message("s1", role="user", content="새로운 메시지")
|
||||
# Search answers via legacy routes meanwhile.
|
||||
assert d2.search_messages("기존", limit=10)
|
||||
|
||||
result = d2.optimize_fts_storage(vacuum=False)
|
||||
assert result["ok"]
|
||||
assert d2._fts_cjk_available
|
||||
assert d2.fts_cjk_rebuild_status() is None
|
||||
assert d2._describe_search_path("기존") == "fts_cjk"
|
||||
rows = d2.search_messages("기존", limit=20)
|
||||
assert len(rows) == 10
|
||||
assert d2.search_messages("새로운", limit=10)
|
||||
d2.close()
|
||||
|
||||
|
||||
def test_legacy_v22_optimize_lands_on_cjk(cjk_so, tmp_path, monkeypatch):
|
||||
"""A legacy inline-FTS (pre-v23) DB optimized on a tokenizer-capable
|
||||
host comes out with BOTH the v23 external-content layout AND a complete
|
||||
cjk index in the same run."""
|
||||
import time as _time
|
||||
|
||||
from hermes_state import SCHEMA_SQL
|
||||
|
||||
monkeypatch.setenv("HERMES_FTS5_CJK_SO", str(cjk_so))
|
||||
db_path = tmp_path / "state.db"
|
||||
|
||||
# Hand-build a genuine legacy inline DB (single-column messages_fts).
|
||||
conn = sqlite3.connect(str(db_path))
|
||||
conn.executescript(SCHEMA_SQL)
|
||||
conn.executescript("""
|
||||
DROP TABLE IF EXISTS messages_fts;
|
||||
DROP TABLE IF EXISTS messages_fts_trigram;
|
||||
DROP VIEW IF EXISTS messages_fts_trigram_src;
|
||||
CREATE VIRTUAL TABLE messages_fts USING fts5(content);
|
||||
CREATE TRIGGER messages_fts_insert AFTER INSERT ON messages BEGIN
|
||||
INSERT INTO messages_fts(rowid, content) VALUES (new.id, COALESCE(new.content,''));
|
||||
END;
|
||||
""")
|
||||
conn.execute("DELETE FROM schema_version")
|
||||
conn.execute("INSERT INTO schema_version (version) VALUES (10)")
|
||||
conn.execute(
|
||||
"INSERT INTO sessions (id, source, started_at) VALUES ('s1', 'cli', ?)",
|
||||
(_time.time(),),
|
||||
)
|
||||
for role, content in (
|
||||
("user", "레거시 일본 메시지"),
|
||||
("assistant", "legacy english reply"),
|
||||
("tool", "레거시 tool output"),
|
||||
):
|
||||
conn.execute(
|
||||
"INSERT INTO messages (session_id, timestamp, role, content) "
|
||||
"VALUES ('s1', ?, ?, ?)",
|
||||
(_time.time(), role, content),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
d = SessionDB(db_path=db_path)
|
||||
try:
|
||||
assert d.fts_optimize_available()
|
||||
# Legacy DB: cjk index deliberately not created at open (the legacy
|
||||
# branch of _init_schema doesn't touch v23 surfaces).
|
||||
assert not d._fts_cjk_available
|
||||
|
||||
result = d.optimize_fts_storage(vacuum=False)
|
||||
assert result["ok"]
|
||||
assert d._fts_cjk_available
|
||||
assert d.fts_cjk_rebuild_status() is None
|
||||
assert d._describe_search_path("일본") == "fts_cjk"
|
||||
assert d.search_messages("일본", limit=10)
|
||||
assert d.search_messages("legacy english", limit=10)
|
||||
with d._lock:
|
||||
idx = d._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages_fts_cjk"
|
||||
).fetchone()[0]
|
||||
non_tool = d._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages WHERE role <> 'tool'"
|
||||
).fetchone()[0]
|
||||
assert idx == non_tool
|
||||
finally:
|
||||
d.close()
|
||||
|
||||
|
||||
def test_fresh_db_index_counts_exclude_tool_rows(db):
|
||||
with db._lock:
|
||||
idx = db._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages_fts_cjk"
|
||||
).fetchone()[0]
|
||||
non_tool = db._conn.execute(
|
||||
"SELECT COUNT(*) FROM messages WHERE role <> 'tool'"
|
||||
).fetchone()[0]
|
||||
assert idx == non_tool
|
||||
|
||||
|
||||
def test_integrity_after_lifecycle(db):
|
||||
db.append_message("s1", role="user", content="무결성 검사")
|
||||
with db._lock:
|
||||
db._conn.execute(
|
||||
"INSERT INTO messages_fts_cjk(messages_fts_cjk) "
|
||||
"VALUES('integrity-check')"
|
||||
)
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Tests for the session-search slow-query log (patch search-slow-query-log)."""
|
||||
"""Tests for the session-search slow-query log (salvaged from PR #65544)."""
|
||||
|
||||
import logging
|
||||
|
||||
@@ -34,13 +34,26 @@ def test_no_log_under_threshold(db, monkeypatch, caplog):
|
||||
assert not [r for r in caplog.records if "slow session search" in r.getMessage()]
|
||||
|
||||
|
||||
def test_path_attribution(db, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_FTS_V2_READ", "0")
|
||||
def test_path_attribution(db):
|
||||
# Without the cjk tokenizer loaded, routing matches the pre-cjk shape.
|
||||
assert db._describe_search_path("graphiti OR neo4j") == "fts5"
|
||||
assert db._describe_search_path("우선순위 캘린더") == "trigram"
|
||||
assert db._describe_search_path("일본 MCP") == "like_scan"
|
||||
|
||||
|
||||
def test_path_attribution_cjk_available(db):
|
||||
# With the bigram index available, CJK queries (including 2-char terms)
|
||||
# route to fts_cjk; lone 1-char CJK runs keep the LIKE route.
|
||||
db._fts_cjk_available = True
|
||||
try:
|
||||
assert db._describe_search_path("일본 MCP") == "fts_cjk"
|
||||
assert db._describe_search_path("우선순위 캘린더") == "fts_cjk"
|
||||
assert db._describe_search_path("가 alone") == "like_scan"
|
||||
assert db._describe_search_path("graphiti OR neo4j") == "fts5"
|
||||
finally:
|
||||
db._fts_cjk_available = False
|
||||
|
||||
|
||||
def test_results_unchanged_by_wrapper(db, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_SEARCH_SLOW_MS", "0")
|
||||
rows = db.search_messages("graphiti", limit=5)
|
||||
|
||||
Reference in New Issue
Block a user