diff --git a/tests/tools/test_tool_search_multiquery.py b/tests/tools/test_tool_search_multiquery.py index f37eef2138..3f554dbdf8 100644 --- a/tests/tools/test_tool_search_multiquery.py +++ b/tests/tools/test_tool_search_multiquery.py @@ -7,6 +7,7 @@ and added Snowball stemming to the shared tokenizer. """ import json +from concurrent.futures import ThreadPoolExecutor import pytest @@ -85,6 +86,79 @@ class TestStemming: names = [h.name for h in search_catalog(catalog, "post_mess", limit=5)] assert names == ["mq_slack_post_message"] + def test_single_token_stems_are_cached(self): + from tools.tool_search import _stem, _tokenize + + _stem.cache_clear() + corpus = "issues creating issues creating" + _tokenize(corpus) + hits_before = _stem.cache_info().hits + _tokenize(corpus) + + assert _stem.cache_info().hits > hits_before + assert _stem.cache_info().hits > 0 + assert _tokenize("issues creating") == ["issu", "creat"] + + def test_parallel_tokenize_search_and_dispatch_are_deterministic(self, issue_defs): + from tools.tool_search import ( + ToolSearchConfig, + _tokenize, + build_catalog, + dispatch_tool_search, + search_catalog, + ) + + corpus = ( + "issues", + "issue", + "creating", + "create", + "meetings", + "meeting", + "post slack message", + "messages posted", + ) + catalog = build_catalog(issue_defs) + expected = { + text: ( + _tokenize(text), + [entry.name for entry in search_catalog(catalog, text, limit=3)], + ) + for text in corpus + } + + def tokenize_and_search(index): + text = corpus[index % len(corpus)] + return text, _tokenize(text), [ + entry.name for entry in search_catalog(catalog, text, limit=3) + ] + + with ThreadPoolExecutor(max_workers=8) as pool: + threaded = list(pool.map(tokenize_and_search, range(512))) + + for text, tokens, names in threaded: + assert (tokens, names) == expected[text] + + args = {"queries": ["issues", "post slack message", "meetings"]} + config = ToolSearchConfig.from_raw({}) + expected_json = dispatch_tool_search( + args, + current_tool_defs=issue_defs, + config=config, + ) + + def dispatch(_index): + return dispatch_tool_search( + args, + current_tool_defs=issue_defs, + config=config, + ) + + with ThreadPoolExecutor(max_workers=8) as pool: + dispatched = list(pool.map(dispatch, range(64))) + + assert dispatched == [expected_json] * 64 + # --------------------------------------------------------------------------- # Exact-name ranking and shared corpus statistics @@ -181,6 +255,39 @@ class TestMultiQuerySearch: for group in result["results"]: assert len(group["matches"]) <= 1 + def test_required_names_are_bounded(self): + from tools.tool_search import ToolSearchConfig, dispatch_tool_search + + required = [f"field_{index}_" + ("x" * 5000) for index in range(200)] + name = "mq_bounded_required_fields" + tool_def = _register(name, "mcp-mq-bounds", required=required) + result = json.loads(dispatch_tool_search( + {"queries": [name]}, + current_tool_defs=[tool_def], + config=ToolSearchConfig.from_raw({}), + )) + record = result["tools"][name] + + assert len(record["required"]) <= 32 + assert all(len(item) <= 64 for item in record["required"]) + + @pytest.mark.parametrize("schema", [ + {"function": "not an object"}, + {"function": {"parameters": ["not", "an", "object"]}}, + ]) + def test_shared_record_handles_non_object_schema_fields(self, schema): + from tools.tool_search import CatalogEntry, _shared_tool_record + + entry = CatalogEntry( + name="mq_malformed_schema", + description="Malformed schema fixture.", + schema=schema, + source="mcp", + source_name="mcp-mq-malformed", + ) + + assert _shared_tool_record(entry)["required"] == [] + def test_partial_miss_adds_fallback_to_empty_group(self, issue_defs): from tools.tool_search import ToolSearchConfig, dispatch_tool_search diff --git a/tools/tool_search.py b/tools/tool_search.py index bd6fa64a46..ed7a861bb3 100644 --- a/tools/tool_search.py +++ b/tools/tool_search.py @@ -42,6 +42,7 @@ for the full rationale): from __future__ import annotations +import functools import json import logging import math @@ -417,6 +418,12 @@ def _stemmer() -> Any: return st +@functools.lru_cache(maxsize=16384) +def _stem(token: str) -> str: + """Stem one token, memoized across stateless catalog rebuilds.""" + return _stemmer().stemWord(token) + + def _tokenize(text: str) -> List[str]: """Lowercase alphanumeric tokens, Snowball-stemmed (English). @@ -427,8 +434,7 @@ def _tokenize(text: str) -> List[str]: """ if not text: return [] - tokens = [t.lower() for t in _TOKEN_RE.findall(text)] - return list(_stemmer().stemWords(tokens)) + return [_stem(token.lower()) for token in _TOKEN_RE.findall(text)] def _entry_search_text(td: Dict[str, Any], source_label: str = "") -> str: @@ -994,8 +1000,13 @@ def _shared_tool_record(entry: CatalogEntry) -> Dict[str, Any]: required parameter names so the model can attempt a call without a ``tool_describe`` round-trip when the required surface is trivial. """ - fn = (entry.schema or {}).get("function") or {} - params = fn.get("parameters") or {} + schema = entry.schema if isinstance(entry.schema, dict) else {} + fn = schema.get("function") + if not isinstance(fn, dict): + fn = {} + params = fn.get("parameters") + if not isinstance(params, dict): + params = {} required = params.get("required") if not isinstance(required, list): required = [] @@ -1004,7 +1015,7 @@ def _shared_tool_record(entry: CatalogEntry) -> Dict[str, Any]: "source_name": entry.source_name, # Cap description so a chatty MCP server doesn't blow up the result. "description": (entry.description or "")[:400], - "required": [r for r in required if isinstance(r, str)], + "required": [r[:64] for r in required if isinstance(r, str)][:32], }