perf(plugins): scan each plugin's source for removed imports once per process, not once per profile

A multiplex gateway runs plugin discovery for every served profile, and each
discovery ran plugin_compat.scan_plugin (an ast.parse + two ast.walk passes per
file) over every external plugin: on an 11-profile host that was ~0.4s per
profile of identical work on the gateway boot path, ahead of adapter connect.
The scan result is now cached process-wide on the plugin dir's (relpath,
mtime_ns, size) signature whenever the loaded manifest is used; a changed file
rescans, a caller-supplied manifest bypasses the cache.

Measured over the 11-profile MCP discovery loop on the reporting host:
3.78s -> 3.01s; per empty profile 0.25-0.30s -> 0.11s.
This commit is contained in:
teknium1
2026-09-15 12:21:07 -07:00
committed by Teknium
parent f81c33cb6a
commit 416a8177c2
2 changed files with 78 additions and 2 deletions
+35 -2
View File
@@ -149,17 +149,50 @@ def scan_source(src: str, rel: str, manifest: Dict[str, Dict[str, str]]) -> List
return sorted(set(hits), key=lambda h: (h.file, h.line, h.old))
_scan_lock = threading.Lock()
_scan_cache: Dict[str, Tuple[Tuple[Tuple[str, int, int], ...], List[Hit]]] = {}
def _plugin_dir_signature(plugin_dir: Path) -> Tuple[Tuple[str, int, int], ...]:
"""``(relpath, mtime_ns, size)`` of every ``.py`` under the plugin — the cache key for a scan.
A stat walk costs ~1 ms; the AST scan it replaces costs ~10 ms per file."""
sig = []
for p in _iter_py(plugin_dir):
try:
st = p.stat()
except OSError:
continue
sig.append((str(p.relative_to(plugin_dir)), st.st_mtime_ns, st.st_size))
return tuple(sorted(sig))
def scan_plugin(plugin_dir: Optional[Path], manifest: Optional[Dict[str, Dict[str, str]]] = None) -> List[Hit]:
manifest = load_manifest() if manifest is None else manifest
"""Hits for one plugin directory. With the default manifest the result is cached process-wide
on the directory's file signature: a multiplex gateway discovers plugins once per served
profile, and re-parsing every plugin's source per profile was ~0.4s × profiles on the boot path."""
loaded = load_manifest()
manifest = loaded if manifest is None else manifest
cacheable = manifest is loaded
if not manifest or not plugin_dir or not Path(plugin_dir).is_dir():
return []
plugin_dir = Path(plugin_dir)
key = str(plugin_dir.resolve(strict=False))
signature = _plugin_dir_signature(plugin_dir) if cacheable else ()
if cacheable:
with _scan_lock:
cached = _scan_cache.get(key)
if cached is not None and cached[0] == signature:
return list(cached[1])
hits: List[Hit] = []
for p in _iter_py(Path(plugin_dir)):
for p in _iter_py(plugin_dir):
try:
src = p.read_text(encoding="utf-8", errors="replace")
except OSError:
continue
hits += scan_source(src, str(p.relative_to(plugin_dir)), manifest)
if cacheable:
with _scan_lock:
_scan_cache[key] = (signature, list(hits))
return hits