Merge branch 'simp/r3-32-A' into simp/r3-32

This commit is contained in:
Teknium
2026-09-02 23:46:48 -07:00
8 changed files with 960 additions and 1742 deletions
+5 -1
View File
@@ -30,7 +30,11 @@ def test_sibling_container_config_sites_carry_docker_network():
import tools.code_execution_tool as code_execution_tool
import tools.file_tools as file_tools
for module in (terminal_tool, file_tools, code_execution_tool):
# file_tools no longer builds its own container_config: it goes through
# the shared _create_configured_env, which is what carries docker_network.
assert "_create_configured_env(" in inspect.getsource(file_tools)
for module in (terminal_tool, code_execution_tool):
tree = ast.parse(inspect.getsource(module))
sites = 0
for node in ast.walk(tree):
+63 -118
View File
@@ -1,24 +1,11 @@
"""Cross-agent file state coordination.
Prevents mangled edits when concurrent subagents (same process, same
filesystem) touch the same file: subagent B writes a file that subagent A
already read, so A's next write would clobber B's changes with stale content.
Complements the single-agent path-overlap check in
``run_agent._should_parallelize_tool_batch``.
A process-wide ``FileStateRegistry`` tracks, per resolved path:
* per-agent read stamps {task_id: {path: (mtime, read_ts, partial)}}
* last writer globally {path: (task_id, write_ts)}
* a per-path ``threading.Lock`` for read->modify->write sections
Hooks used by the file tools: ``record_read`` (read_file), ``note_write``
(after write_file/patch), ``check_stale`` (BEFORE write_file/patch),
``lock_path`` (wrap the whole read->modify->write block) and ``writes_since``
(delegate_tool's subagent-completion reminder).
All methods are no-ops when ``HERMES_DISABLE_FILE_STATE_GUARD=1``. This is
separate from ``file_tools._read_tracker``, which handles per-task
consecutive-read loop detection.
filesystem) touch the same file: B writes a file A already read, so A's next
write would clobber B's changes. Complements the single-agent path-overlap
check in ``run_agent._should_parallelize_tool_batch``. A process-wide
``FileStateRegistry`` tracks per-agent read stamps, the global last writer and
a per-path lock; every method is a no-op under ``HERMES_DISABLE_FILE_STATE_GUARD=1``.
"""
from __future__ import annotations
@@ -35,12 +22,41 @@ from typing import Dict, Iterable, List, Optional, Tuple
# so the model re-reads in full.
ReadStamp = Tuple[float, float, bool]
# Bounded so long sessions don't accumulate unbounded state; oldest by
# insertion order are dropped on overflow.
# Bounded so long sessions don't accumulate unbounded state.
_MAX_PATHS_PER_AGENT = 4096
_MAX_GLOBAL_WRITERS = 4096
def _disabled() -> bool:
# Re-read each call so tests can toggle via monkeypatch.setenv.
return os.environ.get("HERMES_DISABLE_FILE_STATE_GUARD", "").strip() == "1"
def _mtime_or_none(resolved: str) -> Optional[float]:
try:
return os.path.getmtime(resolved)
except OSError:
return None
def _fmt_ts(ts: float) -> str:
# Short wall-clock for warnings; avoids datetime formatting on the hot path.
return time.strftime("%H:%M:%S", time.localtime(ts))
def _evict_oldest(container, cap: int) -> None:
"""Pop entries until *container* is within *cap* (sets: arbitrary; dicts: oldest
by insertion order). An eviction only costs one redundant re-send or staleness check."""
for _ in range(len(container) - cap):
try:
if isinstance(container, set):
container.pop()
else:
container.pop(next(iter(container)))
except (StopIteration, KeyError):
break
class FileStateRegistry:
"""Process-wide coordinator for cross-agent file edits."""
@@ -51,50 +67,31 @@ class FileStateRegistry:
self._meta_lock = threading.Lock() # guards _path_locks
self._state_lock = threading.Lock() # guards _reads + _last_writer
def _lock_for(self, resolved: str) -> threading.Lock:
with self._meta_lock:
lock = self._path_locks.get(resolved)
if lock is None:
lock = threading.Lock()
self._path_locks[resolved] = lock
return lock
@contextmanager
def lock_path(self, resolved: str):
"""Per-path lock: threads on the same path serialize, different paths proceed."""
lock = self._lock_for(resolved)
lock.acquire()
try:
with self._meta_lock:
lock = self._path_locks.setdefault(resolved, threading.Lock())
with lock:
yield
finally:
lock.release()
def record_read(
self,
task_id: str,
resolved: str,
*,
partial: bool = False,
mtime: Optional[float] = None,
) -> None:
def _stamp(self, task_id: str, resolved: str, mtime: float, now: float, partial: bool) -> None:
"""Caller holds ``_state_lock``."""
agent_reads = self._reads[task_id]
agent_reads[resolved] = (float(mtime), now, bool(partial))
_evict_oldest(agent_reads, _MAX_PATHS_PER_AGENT)
def record_read(self, task_id: str, resolved: str, *, partial: bool = False,
mtime: Optional[float] = None) -> None:
if _disabled():
return
mtime = _mtime_or_none(resolved) if mtime is None else mtime
if mtime is None:
return
now = time.time()
with self._state_lock:
agent_reads = self._reads[task_id]
agent_reads[resolved] = (float(mtime), now, bool(partial))
_cap_dict(agent_reads, _MAX_PATHS_PER_AGENT)
self._stamp(task_id, resolved, mtime, time.time(), partial)
def note_write(
self,
task_id: str,
resolved: str,
*,
mtime: Optional[float] = None,
) -> None:
def note_write(self, task_id: str, resolved: str, *, mtime: Optional[float] = None) -> None:
"""Record a successful write: global last-writer AND this agent's own
read stamp (a write is an implicit read of the current content)."""
if _disabled():
@@ -105,27 +102,19 @@ class FileStateRegistry:
now = time.time()
with self._state_lock:
self._last_writer[resolved] = (task_id, now)
_cap_dict(self._last_writer, _MAX_GLOBAL_WRITERS)
self._reads[task_id][resolved] = (float(mtime), now, False)
_cap_dict(self._reads[task_id], _MAX_PATHS_PER_AGENT)
_evict_oldest(self._last_writer, _MAX_GLOBAL_WRITERS)
self._stamp(task_id, resolved, mtime, now, False)
def check_stale(self, task_id: str, resolved: str) -> Optional[str]:
"""Model-facing warning if this write would be stale, else ``None``.
Checked in severity order: (1) a sibling subagent wrote after this
agent's last read; (2) mtime drifted since our read (external edit) or
the read was partial; (3) this agent never read the file. Never raises
— callers decide whether to block or warn.
"""
"""Model-facing warning if this write would be stale, else ``None``. Severity
order: sibling wrote after our read > mtime drift / partial read > never read."""
if _disabled():
return None
with self._state_lock:
stamp = self._reads.get(task_id, {}).get(resolved)
last_writer = self._last_writer.get(resolved)
# Never read and no write record: net-new file or first touch —
# existing sensitive-path / file-exists logic handles it.
if stamp is None and last_writer is None:
if stamp is None and last_writer is None: # net-new file / first touch
return None
current_mtime = _mtime_or_none(resolved)
if current_mtime is None:
@@ -139,16 +128,14 @@ class FileStateRegistry:
f"{resolved} was modified by sibling subagent "
f"{writer_tid!r} but this agent never read it. "
"Read the file before writing to avoid overwriting "
"the sibling's changes."
)
"the sibling's changes.")
read_ts = stamp[1]
if writer_ts > read_ts:
return (
f"{resolved} was modified by sibling subagent "
f"{writer_tid!r} at {_fmt_ts(writer_ts)} — after "
f"this agent's last read at {_fmt_ts(read_ts)}. "
"Re-read the file before writing."
)
"Re-read the file before writing.")
if stamp is not None:
read_mtime, _read_ts, partial = stamp
@@ -156,27 +143,20 @@ class FileStateRegistry:
return (
f"{resolved} was modified since you last read it "
"on disk (external edit or unrecorded writer). "
"Re-read the file before writing."
)
"Re-read the file before writing.")
if partial:
return (
f"{resolved} was last read with offset/limit pagination "
"(partial view). Re-read the whole file before "
"overwriting it."
)
"overwriting it.")
return None
return (
f"{resolved} was not read by this agent. "
"Read the file first so you can write an informed edit."
)
"Read the file first so you can write an informed edit.")
def writes_since(
self,
exclude_task_id: str,
since_ts: float,
paths: Iterable[str],
) -> Dict[str, List[str]]:
def writes_since(self, exclude_task_id: str, since_ts: float,
paths: Iterable[str]) -> Dict[str, List[str]]:
"""``{writer_task_id: [paths]}`` for writes after ``since_ts`` by agents
other than ``exclude_task_id`` (delegate_task's "subagent modified files
you previously read" reminder)."""
@@ -213,36 +193,6 @@ def get_registry() -> FileStateRegistry:
return _registry
def _disabled() -> bool:
# Re-read each call so tests can toggle via monkeypatch.setenv.
return os.environ.get("HERMES_DISABLE_FILE_STATE_GUARD", "").strip() == "1"
def _mtime_or_none(resolved: str) -> Optional[float]:
try:
return os.path.getmtime(resolved)
except OSError:
return None
def _fmt_ts(ts: float) -> str:
# Short wall-clock for warnings; avoids datetime formatting on the hot path.
return time.strftime("%H:%M:%S", time.localtime(ts))
def _cap_dict(d: dict, limit: int) -> None:
"""Trim ``d`` to ``limit`` entries by dropping the insertion-order oldest."""
over = len(d) - limit
if over <= 0:
return
it = iter(d)
for _ in range(over):
try:
d.pop(next(it))
except (StopIteration, KeyError):
break
# Convenience wrappers (short names used at call sites).
def record_read(task_id: str, resolved_or_path: str | Path, *, partial: bool = False) -> None:
_registry.record_read(task_id, str(resolved_or_path), partial=partial)
@@ -260,11 +210,7 @@ def lock_path(resolved_or_path: str | Path):
return _registry.lock_path(str(resolved_or_path))
def writes_since(
exclude_task_id: str,
since_ts: float,
paths: Iterable[str | Path],
) -> Dict[str, List[str]]:
def writes_since(exclude_task_id: str, since_ts: float, paths: Iterable[str | Path]) -> Dict[str, List[str]]:
return _registry.writes_since(exclude_task_id, since_ts, [str(p) for p in paths])
@@ -280,5 +226,4 @@ __all__ = [
"check_stale",
"lock_path",
"writes_since",
"known_reads",
]
"known_reads"]
+262 -553
View File
File diff suppressed because it is too large Load Diff
+72 -140
View File
@@ -1,11 +1,9 @@
"""Path resolution for the file tools: task-aware base dir, ``~`` expansion, workspace-divergence warning.
Companion to ``tools.file_tools`` (which re-imports every name here). The core
invariant: the base directory used to anchor relative paths is ALWAYS absolute
Core invariant: the base directory anchoring relative paths is ALWAYS absolute
and derived from the task's terminal cwd, never from the process cwd unless no
other anchor exists. A relative or sentinel ``TERMINAL_CWD`` would otherwise
silently anchor edits to the agent process cwd (e.g. the main repo while a
worktree session is active).
other anchor exists (a relative/sentinel ``TERMINAL_CWD`` would silently anchor
edits to the agent process cwd, e.g. the main repo during a worktree session).
"""
import os
@@ -13,9 +11,8 @@ import posixpath
import sys
from pathlib import Path, PurePosixPath
# ``TERMINAL_CWD`` values that mean "not configured", not a directory to resolve
# against ("." from a stale config; "auto"/"cwd" are setup-wizard placeholders).
# The gateway sanitizes the same set at import time (gateway/run.py).
# ``TERMINAL_CWD`` values that mean "not configured" ("." from a stale config;
# "auto"/"cwd" are wizard placeholders). gateway/run.py sanitizes the same set.
_TERMINAL_CWD_SENTINELS = frozenset({"", ".", "./", "auto", "cwd"})
_CONTAINER_PATH_BACKENDS_FALLBACK = frozenset({"docker", "singularity", "modal", "daytona", "vercel_sandbox"})
# Backend name inferred from the live environment's class name (first match wins).
@@ -23,13 +20,8 @@ _ENV_CLASS_NAME_HINTS = ("local", "ssh", "docker", "singularity", "modal", "dayt
def _expand_tilde(path: str) -> str:
"""Expand ``~`` using the effective profile home when available.
In-process file tools share the gateway process's HOME, which may differ
from the profile-specific HOME interactive CLI sessions use; mirroring
``hermes_constants.get_subprocess_home()`` keeps ``~`` consistent across
interactive and gateway-driven (cron) runs.
"""
"""Expand ``~`` using the effective profile home (``get_subprocess_home``) so
gateway/cron runs, whose process HOME may differ, agree with interactive CLI sessions."""
if not path or "~" not in path:
return path
try:
@@ -47,11 +39,7 @@ def _terminal_env_type_for_task(task_id: str = "default") -> str:
"""Best-effort terminal backend type for path-resolution decisions."""
try:
from tools.terminal_tool import (
_active_environments,
_env_lock,
_get_env_config,
_resolve_container_task_id,
)
_active_environments, _env_lock, _get_env_config, _resolve_container_task_id)
try:
container_key = _resolve_container_task_id(task_id)
@@ -61,14 +49,11 @@ def _terminal_env_type_for_task(task_id: str = "default") -> str:
env = _active_environments.get(container_key) or _active_environments.get(task_id)
if env is not None:
name = env.__class__.__name__.lower()
for hint in _ENV_CLASS_NAME_HINTS:
if hint in name:
return hint
hint = next((h for h in _ENV_CLASS_NAME_HINTS if h in name), None)
stamped = getattr(env, "_hermes_backend_name", None)
if isinstance(stamped, str) and stamped:
return stamped
cfg = _get_env_config()
return str(cfg.get("env_type") or os.getenv("TERMINAL_ENV") or "local").lower()
if hint or (isinstance(stamped, str) and stamped):
return hint or stamped
return str(_get_env_config().get("env_type") or os.getenv("TERMINAL_ENV") or "local").lower()
except Exception:
return str(os.getenv("TERMINAL_ENV") or "local").lower()
@@ -84,36 +69,24 @@ def _uses_container_paths(task_id: str = "default") -> bool:
def _normalize_without_host_deref(path: str | Path | PurePosixPath) -> PurePosixPath:
"""Normalize path syntax without following host symlinks.
Container paths are meaningful inside the sandbox; ``Path.resolve()`` on the
host could dereference a host-side symlink (e.g. ``/workspace``) and rewrite
the path before Docker sees it.
"""
"""Normalize path syntax without following host symlinks: container paths are
meaningful inside the sandbox, and a host-side ``/workspace`` symlink must not rewrite them."""
return PurePosixPath(posixpath.normpath(str(path)))
def _sentinel_free_abs_cwd(raw: str | None) -> str | None:
"""Return *raw* expanded when it is a non-sentinel ABSOLUTE anchor, else ``None``.
A relative anchor is meaningless without knowing which cwd it is relative
to — exactly the ambiguity that misroutes worktree edits.
"""
"""Return *raw* expanded when it is a non-sentinel ABSOLUTE anchor, else ``None``
(a relative anchor is exactly the ambiguity that misroutes worktree edits)."""
raw = str(raw or "").strip()
if raw.lower() in _TERMINAL_CWD_SENTINELS:
return None
expanded = _expand_tilde(raw)
if not os.path.isabs(expanded):
return None
return expanded
return expanded if os.path.isabs(expanded) else None
def _configured_terminal_cwd() -> str | None:
"""Return ``$TERMINAL_CWD`` only when it names a real (absolute, non-sentinel) anchor.
Scope-aware: under gateway multiplexing the routed profile's cwd lives in
the per-turn terminal scope, not the process env.
"""
Scope-aware: under gateway multiplexing the routed profile's cwd lives in the per-turn scope."""
from agent.runtime_cwd import scope_terminal_cwd
return _sentinel_free_abs_cwd(scope_terminal_cwd() or None)
@@ -122,10 +95,8 @@ def _configured_terminal_cwd() -> str | None:
def _registered_task_cwd_override(task_id: str = "default") -> str | None:
"""Return a registered cwd override keyed by the RAW task id, when available.
``terminal_tool`` collapses CWD-only task overrides to the shared
``"default"`` environment (TUI/dashboard/ACP sessions share one sandbox),
but the cwd value itself stays keyed by the raw session id — so read the
raw override before falling back to the collapsed container key.
``terminal_tool`` collapses CWD-only overrides to the shared ``"default"``
env, but the cwd value stays keyed by the raw session id.
"""
try:
from tools.terminal_tool import resolve_task_overrides
@@ -140,10 +111,9 @@ def _registered_task_cwd_override(task_id: str = "default") -> str | None:
def _authoritative_workspace_root(task_id: str = "default") -> str | None:
"""Best-effort absolute workspace root, or ``None`` when no reliable anchor exists.
Order: (1) the session's own cwd record (written on every completed terminal
command; per-session, so one session's ``cd`` never leaks into another);
(2) a registered raw-keyed task/session cwd override (TUI/Desktop/ACP);
(3) a sentinel-free absolute ``$TERMINAL_CWD`` (``-w`` sessions).
Order: (1) the session's own cwd record (per-session, so one session's
``cd`` never leaks into another); (2) a registered raw-keyed cwd override
(TUI/Desktop/ACP); (3) a sentinel-free absolute ``$TERMINAL_CWD``.
"""
try:
from tools.terminal_tool import get_session_cwd
@@ -151,86 +121,55 @@ def _authoritative_workspace_root(task_id: str = "default") -> str | None:
recorded = get_session_cwd(task_id)
except Exception:
recorded = None
if recorded:
return recorded
registered = _registered_task_cwd_override(task_id)
if registered:
return registered
return _configured_terminal_cwd()
return recorded or _registered_task_cwd_override(task_id) or _configured_terminal_cwd()
def _host_text(text: str, container_paths: bool) -> str:
"""Expand ``~``; on host backends also translate Git Bash ``/c/Users/...`` drive
paths before Path sees them. Container/WSL Linux paths are never rewritten."""
if not container_paths:
from tools.environments.local import _msys_to_windows_path
text = _msys_to_windows_path(text)
return _expand_tilde(text)
def _anchor(text: str, base, container_paths: bool) -> Path | PurePosixPath:
"""Return *text* as an absolute, normalized path, joining it onto ``base()`` when
relative. Container: pure-posix, no host deref. Host: resolve() (win32: ntpath normpath)."""
if container_paths:
if not posixpath.isabs(text):
text = posixpath.join(str(base()), text)
return _normalize_without_host_deref(text)
if sys.platform == "win32":
import ntpath
if not ntpath.isabs(text):
text = ntpath.join(str(base()), text)
return Path(ntpath.normpath(text))
p = Path(text)
if not p.is_absolute():
p = Path(base()) / p
return p.resolve()
def _resolve_base_dir(
task_id: str = "default",
*,
container_paths: bool | None = None,
) -> Path | PurePosixPath:
"""Return the ABSOLUTE base directory for resolving relative paths.
Uses ``_authoritative_workspace_root`` (live cwd → registered override →
``$TERMINAL_CWD``), falling back to the process cwd only as a last resort.
Sentinel/relative ``TERMINAL_CWD`` values are rejected outright rather than
anchored to the process cwd, so the result never depends on where the
agent process happens to run.
"""
task_id: str = "default", *, container_paths: bool | None = None) -> Path | PurePosixPath:
"""Return the ABSOLUTE base directory for resolving relative paths:
``_authoritative_workspace_root``, else the process cwd as a last resort."""
root = _authoritative_workspace_root(task_id)
if container_paths is None:
container_paths = _uses_container_paths(task_id)
base_text = _expand_tilde(root) if root else os.getcwd()
if container_paths:
if not posixpath.isabs(base_text):
base_text = posixpath.join(os.getcwd(), base_text)
return _normalize_without_host_deref(base_text)
# Git Bash ``pwd -P`` reports ``/c/Users/...``; translate before Path so
# relative file-tool paths don't anchor under a nonexistent ``\\c\\Users``.
from tools.environments.local import _msys_to_windows_path
base_text = _msys_to_windows_path(base_text)
if sys.platform == "win32":
import ntpath
if not ntpath.isabs(base_text):
base_text = ntpath.join(os.getcwd(), base_text)
return Path(ntpath.normpath(base_text))
base = Path(base_text)
if not base.is_absolute():
# A backend reporting a relative cwd is anchored to the process cwd
# once, here, so the result no longer depends on cwd at resolve().
base = Path(os.getcwd()) / base
return base.resolve()
# A backend's relative cwd is anchored to the process cwd once, here.
return _anchor(_host_text(root or os.getcwd(), container_paths), os.getcwd, container_paths)
def _resolve_path_for_task(filepath: str, task_id: str = "default") -> Path | PurePosixPath:
"""Resolve *filepath* against the task's absolute base directory.
Absolute inputs are returned resolved-but-unanchored. On native Windows,
Git Bash / MSYS drive paths (``/c/Users/...``) are translated first so
they aren't treated as relative ``\\c\\Users\\...`` under the process cwd;
container/WSL Linux paths are never rewritten.
"""
"""Resolve *filepath* against the task's absolute base directory
(absolute inputs are returned resolved-but-unanchored)."""
container_paths = _uses_container_paths(task_id)
if container_paths:
expanded = _expand_tilde(filepath)
if posixpath.isabs(expanded):
return _normalize_without_host_deref(expanded)
resolved = _resolve_base_dir(task_id, container_paths=True) / expanded
return _normalize_without_host_deref(resolved)
from tools.environments.local import _msys_to_windows_path
expanded = _expand_tilde(_msys_to_windows_path(filepath))
if sys.platform == "win32":
import ntpath
if ntpath.isabs(expanded):
return Path(ntpath.normpath(expanded))
joined = ntpath.join(str(_resolve_base_dir(task_id, container_paths=False)), expanded)
return Path(ntpath.normpath(joined))
p = Path(expanded)
if p.is_absolute():
return p.resolve()
resolved = _resolve_base_dir(task_id, container_paths=False) / p
return resolved.resolve()
return _anchor(_host_text(filepath, container_paths),
lambda: _resolve_base_dir(task_id, container_paths=container_paths), container_paths)
# Back-compat alias (imported by agent.context_references and tests).
@@ -238,13 +177,9 @@ _resolve_path = _resolve_path_for_task
def _path_resolution_warning(filepath: str, resolved: Path, task_id: str = "default") -> str | None:
"""Warn when a RELATIVE path resolved OUTSIDE the task's workspace root.
Surfaces the worktree-cwd divergence the moment it matters — the edit is
about to land in a different checkout than the terminal's cwd. ``None`` for
absolute paths, an unknown root, or a path correctly under the root. Fires
on the very first write even before any ``cd`` populated the cwd registry.
"""
"""Warn when a RELATIVE path resolved OUTSIDE the task's workspace root (the
edit is about to land in a different checkout than the terminal's cwd).
``None`` for absolute paths, an unknown root, or a path under the root."""
try:
if Path(_expand_tilde(filepath)).is_absolute():
return None
@@ -255,16 +190,13 @@ def _path_resolution_warning(filepath: str, resolved: Path, task_id: str = "defa
root = _normalize_without_host_deref(Path(_expand_tilde(workspace_root)))
else:
root = Path(_expand_tilde(workspace_root)).resolve()
try:
resolved.relative_to(root)
if resolved.is_relative_to(root):
return None
except ValueError:
return (
f"Relative path {filepath!r} resolved to {str(resolved)!r}, which is "
f"OUTSIDE the active workspace ({str(root)!r}). The edit will land in "
f"a different directory than the terminal's cwd. If this is not "
f"intended (e.g. a git-worktree session writing into the main "
f"checkout), pass an absolute path under the workspace instead."
)
return (
f"Relative path {filepath!r} resolved to {str(resolved)!r}, which is "
f"OUTSIDE the active workspace ({str(root)!r}). The edit will land in "
f"a different directory than the terminal's cwd. If this is not "
f"intended (e.g. a git-worktree session writing into the main "
f"checkout), pass an absolute path under the workspace instead.")
except Exception:
return None
+73 -151
View File
@@ -1,23 +1,12 @@
"""Per-task read/search bookkeeping for the file tools.
Process-lifetime state behind ``read_file`` / ``search_files`` / ``write_file`` /
``patch``; ``tools.file_tools`` re-imports every name here. Per task_id
``_read_tracker`` stores:
last_key / consecutive most recent read-or-search key and its repeat count
(loop detection; reset by any OTHER tool call).
read_history set of (path, offset, limit) — diagnostic summaries only.
dedup (resolved_path, offset, limit) -> mtime; skip identical
re-reads of unchanged files. Cleared on context
compression (the original content was summarised away).
dedup_hits per-key count of stub returns, to break stub loops.
read_timestamps resolved_path -> mtime at last read/write by this task;
write/patch warn when the file changed underneath.
not_found (op, resolved_path) -> (monotonic, cached error JSON);
short-TTL negative cache for retried missing paths.
Every container is hard-capped (``_cap_read_tracker_data``) so a long CLI
session accretes a few hundred KB at most instead of ~1.5MB per 10k reads.
Process-lifetime state behind read_file/search_files/write_file/patch;
``tools.file_tools`` re-imports every name here. Per task_id ``_read_tracker``
stores: ``last_key``/``consecutive`` (loop detection; reset by any OTHER tool
call), ``read_history`` (diagnostics), ``dedup`` (key -> mtime; cleared on
context compression), ``dedup_hits`` (stub-loop breaker), ``read_timestamps``
(staleness warnings) and ``not_found`` (short-TTL negative cache). Every
container is hard-capped (``_cap_read_tracker_data``) so long sessions stay small.
"""
import logging
@@ -25,6 +14,7 @@ import os
import threading
import time
from tools.file_state import _evict_oldest
from tools.file_tools_paths import _authoritative_workspace_root, _resolve_path_for_task
logger = logging.getLogger("tools.file_tools")
@@ -33,8 +23,7 @@ _read_tracker_lock = threading.Lock()
_read_tracker: dict = {}
# Consecutive patch failures per (task_id, resolved_path); escalates the hint
# when the model keeps failing the same file (stale view, ambiguous old_string).
# Reset on a successful patch to that path.
# when the model keeps failing the same file. Reset on a successful patch.
_patch_failure_lock = threading.Lock()
_patch_failure_tracker: dict = {} # {task_id: {resolved_path: count}}
_PATCH_FAILURE_PATHS_CAP = 64
@@ -48,24 +37,13 @@ _NOT_FOUND_CAP = 500
_NOT_FOUND_TTL_SECONDS = 60.0 # a path that didn't exist may be created soon
def _new_task_data() -> dict:
return {
"last_key": None, "consecutive": 0,
"read_history": set(), "dedup": {},
"dedup_hits": {}, "read_timestamps": {},
}
def _task_data(task_id: str) -> dict:
"""Get-or-create the tracker entry for *task_id*, back-filling any missing keys.
Must be called with ``_read_tracker_lock`` held. Entries created by older
code paths (or injected by tests) may lack the newer containers.
"""
task_data = _read_tracker.setdefault(task_id, _new_task_data())
for key, factory in (("dedup", dict), ("dedup_hits", dict), ("read_timestamps", dict)):
if key not in task_data:
task_data[key] = factory()
"""Get-or-create the tracker entry for *task_id*, back-filling missing containers
(search_tool / tests create partial entries). Lock must be held."""
task_data = _read_tracker.setdefault(task_id, {
"last_key": None, "consecutive": 0, "read_history": set()})
for key in ("dedup", "dedup_hits", "read_timestamps"):
task_data.setdefault(key, {})
return task_data
@@ -74,11 +52,8 @@ def _record_patch_failure(task_id: str, resolved_path: str) -> int:
with _patch_failure_lock:
task_failures = _patch_failure_tracker.setdefault(task_id, {})
# Evict the oldest entry once a task has failed on many distinct files.
if len(task_failures) >= _PATCH_FAILURE_PATHS_CAP and resolved_path not in task_failures:
try:
del task_failures[next(iter(task_failures))]
except StopIteration:
pass
if resolved_path not in task_failures:
_evict_oldest(task_failures, _PATCH_FAILURE_PATHS_CAP - 1)
task_failures[resolved_path] = task_failures.get(resolved_path, 0) + 1
return task_failures[resolved_path]
@@ -89,29 +64,10 @@ def _reset_patch_failures(task_id: str, resolved_paths: list) -> None:
return
with _patch_failure_lock:
task_failures = _patch_failure_tracker.get(task_id)
if not task_failures:
return
for rp in resolved_paths:
for rp in resolved_paths if task_failures else ():
task_failures.pop(rp, None)
def _evict_oldest(container, cap: int) -> None:
"""Pop entries until *container* is within *cap*.
Sets pop arbitrary entries (they only feed diagnostic summaries); dicts pop
oldest by insertion order. An evicted entry costs one redundant re-send
(dedup) or one non-mtime staleness check — graceful degradation, not a bug.
"""
for _ in range(len(container) - cap):
try:
if isinstance(container, set):
container.pop()
else:
container.pop(next(iter(container)))
except (StopIteration, KeyError):
break
def _cap_read_tracker_data(task_data: dict) -> None:
"""Enforce size caps on the per-task sub-containers. Call with ``_read_tracker_lock`` held."""
# Caps are read at call time so tests can monkeypatch the module constants.
@@ -120,44 +76,48 @@ def _cap_read_tracker_data(task_data: dict) -> None:
("dedup", _DEDUP_CAP),
("dedup_hits", _DEDUP_CAP),
("read_timestamps", _READ_TIMESTAMPS_CAP),
("not_found", _NOT_FOUND_CAP),
):
("not_found", _NOT_FOUND_CAP)):
container = task_data.get(key)
if container is not None and len(container) > cap:
_evict_oldest(container, cap)
def _resolved_or_none(filepath: str, task_id: str) -> str | None:
try:
return str(_resolve_path_for_task(filepath, task_id))
except (OSError, ValueError):
return None
def _pop_not_found(op: str, resolved_str: str, task_id: str) -> None:
"""Drop the negative-cache entry for *(op, resolved_str)*. Lock must be held."""
task_data = _read_tracker.get(task_id)
nf = task_data.get("not_found") if task_data else None
if nf:
nf.pop((op, resolved_str), None)
def _check_not_found_cache(op: str, resolved_str: str, task_id: str) -> str | None:
"""Return cached not-found JSON for *(op, resolved_str)* if still fresh.
Skips the subprocess + similar-name walk when the model retries the same
missing path. *op* is "read" or "search" (different error JSON shapes).
Evicted by TTL, by write_file/patch on the path, or by any other tool call.
*op* is "read" or "search" (different error JSON shapes). Evicted by TTL,
by write_file/patch on the path, or by any other tool call.
"""
with _read_tracker_lock:
task_data = _read_tracker.get(task_id)
if not task_data:
return None
nf = task_data.get("not_found")
if not nf:
return None
entry = nf.get((op, resolved_str))
entry = (task_data.get("not_found") or {}).get((op, resolved_str)) if task_data else None
if entry is None:
return None
ts, cached_json = entry
if time.monotonic() - ts > _NOT_FOUND_TTL_SECONDS:
nf.pop((op, resolved_str), None)
_pop_not_found(op, resolved_str, task_id)
return None
# The path may have been created since the miss was cached (terminal,
# another agent, ...) — the "check → create → read" pattern is common, so
# serving a stale miss breaks it. The stat runs OUTSIDE the global tracker
# lock: a hung stat on a dead network mount must not stall every task.
# "check → create → read" is common, so never serve a stale miss for a path
# that now exists. The stat runs OUTSIDE the tracker lock: a hung stat on a
# dead network mount must not stall every task.
if os.path.exists(resolved_str):
with _read_tracker_lock:
task_data = _read_tracker.get(task_id)
nf = task_data.get("not_found") if task_data else None
if nf:
nf.pop((op, resolved_str), None)
_pop_not_found(op, resolved_str, task_id)
return None
return cached_json
@@ -166,8 +126,7 @@ def _record_not_found(op: str, resolved_str: str, task_id: str, error_json: str)
"""Cache a not-found error so the next *op* call for *resolved_str* skips I/O."""
with _read_tracker_lock:
task_data = _task_data(task_id)
nf = task_data.setdefault("not_found", {})
nf[(op, resolved_str)] = (time.monotonic(), error_json)
task_data.setdefault("not_found", {})[(op, resolved_str)] = (time.monotonic(), error_json)
_cap_read_tracker_data(task_data)
@@ -182,11 +141,8 @@ def _bump_consecutive(task_data: dict, key: tuple) -> int:
def reset_file_dedup(task_id: str = None):
"""Clear the read-dedup cache (one task, or all when ``task_id`` is None).
Called after context compression: the original read content was summarised
away, so a "file unchanged" stub would point at content no longer in context.
"""
"""Clear the read-dedup cache (one task, or all when ``task_id`` is None). Called
after context compression: a "file unchanged" stub would point at summarised-away content."""
with _read_tracker_lock:
if task_id:
targets = [_read_tracker[task_id]] if _read_tracker.get(task_id) else []
@@ -201,33 +157,25 @@ def reset_file_dedup(task_id: str = None):
def notify_other_tool_call(task_id: str = "default"):
"""Reset the consecutive read/search counter for a task.
Called by the dispatcher for every tool OTHER than read_file/search_files,
so loop detection only fires on truly consecutive repeats. Also clears the
stub-hit counters and the not-found cache: any other tool may have created
a previously-missing path (the serve-side stat covers most cases; clearing
covers the rest, e.g. permission flips).
Called by the dispatcher for every tool OTHER than read_file/search_files.
Also clears stub-hit counters and the not-found cache: any other tool may
have created a previously-missing path (or flipped its permissions).
"""
with _read_tracker_lock:
task_data = _read_tracker.get(task_id)
if task_data:
task_data["last_key"] = None
task_data["consecutive"] = 0
if "dedup_hits" in task_data:
task_data["dedup_hits"].clear()
nf = task_data.get("not_found")
if nf:
nf.clear()
for key in ("dedup_hits", "not_found"):
if task_data.get(key):
task_data[key].clear()
def _invalidate_dedup_for_path(filepath: str, task_id: str) -> None:
"""Evict every dedup entry (all offset/limit ranges) and not-found entry for *filepath*.
Called after write_file/patch so the next read returns fresh content
instead of a stale "unchanged" stub. Acquires ``_read_tracker_lock`` itself.
"""
try:
resolved = str(_resolve_path_for_task(filepath, task_id))
except (OSError, ValueError):
"""Evict every dedup entry (all offset/limit ranges) and not-found entry for *filepath*
after a write, so the next read returns fresh content. Acquires the lock itself."""
resolved = _resolved_or_none(filepath, task_id)
if resolved is None:
return
with _read_tracker_lock:
task_data = _read_tracker.get(task_id)
@@ -237,20 +185,20 @@ def _invalidate_dedup_for_path(filepath: str, task_id: str) -> None:
if dedup:
for k in [k for k in dedup if k[0] == resolved]:
del dedup[k]
nf = task_data.get("not_found")
if nf:
nf.pop(("read", resolved), None)
nf.pop(("search", resolved), None)
_pop_not_found("read", resolved, task_id)
_pop_not_found("search", resolved, task_id)
def _update_read_timestamp(filepath: str, task_id: str) -> None:
"""After a successful write: invalidate dedup and refresh the stored mtime so
consecutive edits by the same task don't trigger false staleness warnings."""
_invalidate_dedup_for_path(filepath, task_id)
resolved = _resolved_or_none(filepath, task_id)
if resolved is None:
return
try:
resolved = str(_resolve_path_for_task(filepath, task_id))
current_mtime = os.path.getmtime(resolved)
except (OSError, ValueError):
except OSError:
return
with _read_tracker_lock:
task_data = _read_tracker.get(task_id)
@@ -261,19 +209,13 @@ def _update_read_timestamp(filepath: str, task_id: str) -> None:
def _check_file_staleness(filepath: str, task_id: str) -> str | None:
"""Warn (don't block) when the file's mtime changed since this task last read it.
``None`` when never read, fresh, or unstattable (a deleted file is the
write's problem to report).
"""
try:
resolved = str(_resolve_path_for_task(filepath, task_id))
except (OSError, ValueError):
``None`` when never read, fresh, or unstattable (a deleted file is the write's problem)."""
resolved = _resolved_or_none(filepath, task_id)
if resolved is None:
return None
with _read_tracker_lock:
task_data = _read_tracker.get(task_id)
if not task_data:
return None
read_mtime = task_data.get("read_timestamps", {}).get(resolved)
read_mtime = task_data.get("read_timestamps", {}).get(resolved) if task_data else None
if read_mtime is None:
return None
try:
@@ -284,21 +226,14 @@ def _check_file_staleness(filepath: str, task_id: str) -> str | None:
return (
f"Warning: {filepath} was modified since you last read it "
"(external edit or concurrent agent). The content you read may be "
"stale. Consider re-reading the file to verify before writing."
)
"stale. Consider re-reading the file to verify before writing.")
return None
def _mark_verification_stale(
task_id: str,
resolved_paths: list[str],
session_id: str | None = None,
) -> None:
"""Best-effort note that successful edits made prior verification stale.
The workspace cwd is the first edited path's project root when one is
recognised, else the task's workspace root, else the first path's parent.
"""
def _mark_verification_stale(task_id: str, resolved_paths: list[str],
session_id: str | None = None) -> None:
"""Best-effort note that successful edits made prior verification stale. cwd: the
first edited path's recognised project root, else the workspace root, else the first parent."""
from pathlib import Path
paths = [p for p in resolved_paths if p]
@@ -308,22 +243,9 @@ def _mark_verification_stale(
from agent.coding_context import project_facts_for
from agent.verification_evidence import mark_workspace_edited
cwd = None
for path in paths:
try:
candidate = str(Path(path).parent)
except Exception:
continue
if project_facts_for(candidate):
cwd = candidate
break
if cwd is None:
cwd = _authoritative_workspace_root(task_id)
if cwd is None:
try:
cwd = str(Path(paths[0]).parent)
except Exception:
cwd = None
parents = [str(Path(p).parent) for p in paths]
cwd = (next((c for c in parents if project_facts_for(c)), None)
or _authoritative_workspace_root(task_id) or parents[0])
mark_workspace_edited(session_id=session_id or task_id, cwd=cwd, paths=paths)
except Exception:
logger.debug("verification stale marker failed", exc_info=True)
+133 -267
View File
@@ -2,14 +2,10 @@
Every guard returns ``None`` when the write may proceed, else an error string
the tool returns verbatim. ``tools.file_tools`` re-imports every name here.
Guards (in the order the tools apply them):
* ``_check_sensitive_path`` — system paths + the Hermes config.yaml (hard deny).
* ``_check_binary_document_write`` — text write would corrupt an Office/PDF container.
* ``_check_protected_instruction_write`` — AGENTS.md-style files: ALWAYS ask, no yolo bypass.
* ``_check_approval_required_write`` — ~/.ssh/config-style files: normal approval gate.
* ``_check_cross_profile_path`` — sandbox-mirror writes the host never reads (lost-work guard).
* ``_is_internal_file_tool_content`` — refuse to persist read_file display text as a file.
Guards, in the order the tools apply them: ``_check_sensitive_path`` (hard
deny), ``_check_binary_document_write``, ``_check_protected_instruction_write``
(ALWAYS ask), ``_check_approval_required_write`` (normal gate),
``_check_cross_profile_path`` (sandbox-mirror lost-work), ``_is_internal_file_tool_content``.
"""
import fnmatch
@@ -25,29 +21,51 @@ from tools.file_tools_paths import _expand_tilde, _resolve_path_for_task
_SENSITIVE_PATH_PREFIXES = (
"/etc/", "/boot/", "/usr/lib/systemd/",
"/private/etc/",
"/private/var/db/", "/private/var/root/",
)
"/private/var/db/", "/private/var/root/")
_SENSITIVE_EXACT_PATHS = {"/var/run/docker.sock", "/run/docker.sock"}
_hermes_config_resolved: str | None = None
_hermes_config_resolved_loaded = False
_real_hermes_home_cached: str | None = None
_real_hermes_home_loaded = False
def _cached_lookup(slot: str, flag: str, primary, fallback) -> str | None:
"""Fill module global *slot* once (guarded by *flag*) from ``primary()``, else
``fallback()``, else None. Module globals so tests can monkeypatch the slots."""
g = globals()
if not g[flag]:
g[flag] = True
try:
g[slot] = primary()
except Exception:
try:
g[slot] = fallback()
except Exception:
g[slot] = None
return g[slot]
def _config_path_resolved() -> str:
from hermes_cli.config import get_config_path
return str(get_config_path().resolve())
def _hermes_home_real() -> str:
from hermes_constants import get_hermes_home
return os.path.realpath(str(get_hermes_home()))
def _get_hermes_config_resolved() -> str | None:
"""Return the resolved absolute path of the Hermes config file (cached)."""
global _hermes_config_resolved, _hermes_config_resolved_loaded
if _hermes_config_resolved_loaded:
return _hermes_config_resolved
_hermes_config_resolved_loaded = True
try:
from hermes_cli.config import get_config_path
_hermes_config_resolved = str(get_config_path().resolve())
except Exception:
try:
_hermes_config_resolved = str(Path(_expand_tilde("~/.hermes/config.yaml")).resolve())
except Exception:
_hermes_config_resolved = None
return _hermes_config_resolved
return _cached_lookup("_hermes_config_resolved", "_hermes_config_resolved_loaded", _config_path_resolved,
lambda: str(Path(_expand_tilde("~/.hermes/config.yaml")).resolve()))
def _get_real_hermes_home() -> str | None:
"""Return the realpath of the authoritative Hermes home (cached)."""
return _cached_lookup("_real_hermes_home_cached", "_real_hermes_home_loaded", _hermes_home_real,
lambda: os.path.realpath(_expand_tilde("~/.hermes")))
def _resolved_or_raw(filepath: str, task_id: str) -> str:
@@ -60,79 +78,40 @@ def _resolved_or_raw(filepath: str, task_id: str) -> str:
def _check_sensitive_path(filepath: str, task_id: str = "default") -> str | None:
"""Return an error message if the path targets a sensitive system location."""
resolved = _resolved_or_raw(filepath, task_id)
normalized = os.path.normpath(_expand_tilde(filepath))
_err = (
f"Refusing to write to sensitive system path: {filepath}\n"
"Use the terminal tool with sudo if you need to modify system files."
)
for prefix in _SENSITIVE_PATH_PREFIXES:
if resolved.startswith(prefix) or normalized.startswith(prefix):
return _err
if resolved in _SENSITIVE_EXACT_PATHS or normalized in _SENSITIVE_EXACT_PATHS:
return _err
candidates = (_resolved_or_raw(filepath, task_id), os.path.normpath(_expand_tilde(filepath)))
if any(c.startswith(_SENSITIVE_PATH_PREFIXES) or c in _SENSITIVE_EXACT_PATHS for c in candidates):
return (
f"Refusing to write to sensitive system path: {filepath}\n"
"Use the terminal tool with sudo if you need to modify system files.")
# approvals.mode and other security settings live in config.yaml; a
# prompt-injected agent could silently disable exec approval by editing it.
hermes_config = _get_hermes_config_resolved()
if hermes_config and (resolved == hermes_config or normalized == hermes_config):
if hermes_config and hermes_config in candidates:
return (
f"Refusing to write to Hermes config file: {filepath}\n"
"Agent cannot modify security-sensitive configuration. "
"Edit ~/.hermes/config.yaml directly or use 'hermes config' instead."
)
"Edit ~/.hermes/config.yaml directly or use 'hermes config' instead.")
return None
# ---------------------------------------------------------------------------
# Protected agent-instruction files (always-ask approval gate)
# ---------------------------------------------------------------------------
# ── Protected agent-instruction files (always-ask approval gate) ─────────
# Files that steer FUTURE agent behavior are a prompt-injection persistence
# vector: an injected edit to AGENTS.md / CLAUDE.md / SOUL.md / .cursorrules
# (or a project-local .hermes tree) outlives the turn and poisons every later
# session. Writes ALWAYS require human approval — even under --yolo — and fail
# closed when no human channel exists. (Ported from Roo-Code's
# RooProtectedController; the terminal-tool vector is gated separately.)
#
# Basenames match in ANY directory (instruction files load from cwd trees) and
# case-insensitively (case-insensitive filesystems; loaders probe variants).
# vector (AGENTS.md / CLAUDE.md / SOUL.md / .cursorrules / project .hermes tree).
# Writes ALWAYS require human approval — even under --yolo — and fail closed
# without a human channel. Basenames match in ANY directory, case-insensitively.
_PROTECTED_INSTRUCTION_BASENAMES = frozenset({
"agents.md", "claude.md", "soul.md", ".cursorrules",
})
_real_hermes_home_cached: str | None = None
_real_hermes_home_loaded = False
def _get_real_hermes_home() -> str | None:
"""Return the realpath of the authoritative Hermes home (cached)."""
global _real_hermes_home_cached, _real_hermes_home_loaded
if _real_hermes_home_loaded:
return _real_hermes_home_cached
_real_hermes_home_loaded = True
try:
from hermes_constants import get_hermes_home
_real_hermes_home_cached = os.path.realpath(str(get_hermes_home()))
except Exception:
try:
_real_hermes_home_cached = os.path.realpath(_expand_tilde("~/.hermes"))
except Exception:
_real_hermes_home_cached = None
return _real_hermes_home_cached
"agents.md", "claude.md", "soul.md", ".cursorrules"})
def _protected_instruction_config() -> tuple[bool, list[str]]:
"""Return ``(enabled, extra_patterns)`` from ``security.protected_instruction_files`` /
``security.protected_instruction_extra_patterns`` (fnmatch on basename).
Config read failures keep the gate ON — fail-safe for a security boundary.
"""
``security.protected_instruction_extra_patterns`` (fnmatch on basename). Config read
failures keep the gate ON — fail-safe for a security boundary."""
try:
from hermes_cli.config import load_config, cfg_get
cfg = load_config()
enabled = cfg_get(cfg, "security", "protected_instruction_files",
default=True)
extra = cfg_get(cfg, "security", "protected_instruction_extra_patterns",
default=[])
enabled = cfg_get(cfg, "security", "protected_instruction_files", default=True)
extra = cfg_get(cfg, "security", "protected_instruction_extra_patterns", default=[])
except Exception:
return True, []
if not isinstance(enabled, bool):
@@ -146,11 +125,7 @@ def _protected_instruction_reason(filepath: str, task_id: str = "default",
*, enabled: bool | None = None,
extra_patterns: list[str] | None = None) -> str | None:
"""Return a short label when ``filepath`` targets a protected instruction file, else ``None``.
Matches BOTH the normalized input and its realpath so neither a symlink
pointing AT a protected file nor a protected name that is itself a symlink
escapes; ``..`` traversal is neutralized by normpath/realpath first.
"""
Matches BOTH the normalized input and its realpath so no symlink direction escapes."""
if enabled is None or extra_patterns is None:
enabled, extra_patterns = _protected_instruction_config()
if not enabled:
@@ -166,18 +141,15 @@ def _protected_instruction_reason(filepath: str, task_id: str = "default",
# mirror guard, write_approval); this gate targets PROJECT-LOCAL files only.
# Must run before the ``.hermes`` component rule, which would match the home.
real_home = _get_real_hermes_home()
if real_home and (resolved == real_home
or resolved.startswith(real_home + os.sep)):
if real_home and (resolved == real_home or resolved.startswith(real_home + os.sep)):
return None
for candidate in (normalized, resolved):
base = os.path.basename(candidate)
base_lower = base.lower()
if base_lower in _PROTECTED_INSTRUCTION_BASENAMES:
if base_lower in _PROTECTED_INSTRUCTION_BASENAMES or any(
fnmatch.fnmatch(base_lower, pattern.lower()) for pattern in extra_patterns):
return base
for pattern in extra_patterns:
if fnmatch.fnmatch(base_lower, pattern.lower()):
return base
# Project-local .hermes config dirs (<repo>/.hermes/config.yaml) steer
# behavior too. Only the IMMEDIATE parent counts — matching any ancestor
# would gate every write inside a checkout living under ~/.hermes.
@@ -187,43 +159,39 @@ def _protected_instruction_reason(filepath: str, task_id: str = "default",
return None
def _request_protected_instruction_approval(
reasons: list[str], task_id: str = "default") -> str | None:
"""Ask the human to approve a write to protected instruction file(s).
_APPROVAL_UNAVAILABLE = "requires approval but the approval subsystem is unavailable."
_NO_HUMAN = "requires approval but no interactive user or gateway is present to approve it."
Returns ``None`` when approved, else a BLOCKED error string. Deliberately
NOT routed through ``_run_approval_gate``: that honors --yolo and
session/permanent allowlists, and this gate is one-operation approval EVERY
time with no persisted scope. Fail-closed when no human channel exists.
def _request_protected_instruction_approval(reasons: list[str], task_id: str = "default") -> str | None:
"""Ask the human to approve a write to protected instruction file(s); ``None`` when approved.
Deliberately NOT routed through ``_run_approval_gate`` (honors --yolo and
allowlists): this gate is one-operation approval EVERY time, no persisted
scope, fail-closed without a human channel.
"""
targets = ", ".join(dict.fromkeys(reasons))
description = (
f"Write to protected agent-instruction file(s): {targets}. "
"These files steer future agent behavior; approval is always "
"required (not bypassed by auto-approve)."
)
"required (not bypassed by auto-approve).")
display = f"<write to {targets}>"
blocked = (
f"BLOCKED: write to protected agent-instruction file(s) ({targets}) "
"{why} The user has NOT consented to this write. Do NOT retry it or "
"attempt the same edit via another path (terminal, execute_code, "
"etc.)."
)
timed_out = blocked.format(
why="approval prompt timed out without a user response. "
"Silence is not consent.")
"etc.).")
timed_out = blocked.format(why="approval prompt timed out without a user response. Silence is not consent.")
denied = blocked.format(why="was denied by the user.")
try:
import tools.approval as _approval
except Exception:
return blocked.format(why="requires approval but the approval "
"subsystem is unavailable.")
return blocked.format(why=_APPROVAL_UNAVAILABLE)
# Gateway surface: block on the button round-trip when a notify callback
# is registered for this session. One-operation only — no scope buttons.
session_key = _approval.get_current_session_key()
notify_cb = None
try:
with _approval._lock:
notify_cb = _approval._gateway_notify_cbs.get(session_key)
@@ -237,82 +205,48 @@ def _request_protected_instruction_approval(
"pattern_keys": ["protected_instruction_file"],
"description": description,
"allow_permanent": False,
"allow_session": False,
}
decision = _approval._await_gateway_decision(
session_key, notify_cb, approval_data, surface="gateway",
)
"allow_session": False}
decision = _approval._await_gateway_decision(session_key, notify_cb, approval_data, surface="gateway")
if decision.get("notify_failed"):
return blocked.format(
why="requires approval but the approval request could not "
"be delivered.")
choice = decision.get("choice")
# Any tapped scope is a one-operation grant; nothing is persisted.
if decision.get("resolved") and choice in {"once", "session", "always"}:
return None
if not decision.get("resolved"):
return timed_out
return denied
# CLI surface: per-thread approval callback (prompt_toolkit panel).
callback = None
try:
from tools.terminal_tool import _get_approval_callback
callback = _get_approval_callback()
except Exception:
callback = None
if callback is not None:
return blocked.format(why="requires approval but the approval request could not be delivered.")
choice, timed = decision.get("choice"), not decision.get("resolved")
else:
# CLI surface: per-thread approval callback (prompt_toolkit panel).
try:
from tools.terminal_tool import _get_approval_callback
callback = _get_approval_callback()
except Exception:
callback = None
if callback is None:
# No human channel (script, cron, background thread): fail closed —
# auto-approving here would recreate the persistence vector.
return blocked.format(why=_NO_HUMAN)
choice = _approval.prompt_dangerous_approval(
display, description,
allow_permanent=False,
allow_session=False,
approval_callback=callback,
)
if choice in {"once", "session", "always"}:
return None
if choice == "timeout":
return timed_out
return denied
# No human channel (script, cron, background thread): fail closed —
# auto-approving here would recreate the persistence vector.
return blocked.format(
why="requires approval but no interactive user or gateway is "
"present to approve it.")
display, description, allow_permanent=False, allow_session=False, approval_callback=callback)
timed = choice == "timeout"
# Any tapped scope is a one-operation grant; nothing is persisted.
if not timed and choice in {"once", "session", "always"}:
return None
return timed_out if timed else denied
def _check_protected_instruction_write(paths: list[str],
task_id: str = "default") -> str | None:
"""Gate a write/patch touching protected instruction files.
ONE protected file gates the ENTIRE multi-file patch: a single prompt lists
every protected target and a deny applies nothing (atomic all-or-nothing
beats a partially-applied patch).
"""
def _check_protected_instruction_write(paths: list[str], task_id: str = "default") -> str | None:
"""Gate a write/patch touching protected instruction files. ONE protected file gates
the ENTIRE multi-file patch (one prompt, all-or-nothing)."""
enabled, extra = _protected_instruction_config()
if not enabled:
return None
reasons = [
r for r in (
_protected_instruction_reason(p, task_id, enabled=enabled, extra_patterns=extra)
for p in paths
) if r
]
reasons = [r for r in (_protected_instruction_reason(p, task_id, enabled=enabled, extra_patterns=extra)
for p in paths) if r]
if not reasons:
return None
return _request_protected_instruction_approval(reasons, task_id)
def _check_approval_required_write(paths: list[str],
task_id: str = "default") -> str | None:
"""Gate a write/patch touching an approval-required path (``~/.ssh/config``).
Not credentials and not hard-denied, but they can steer process execution
(SSH ``ProxyCommand`` / ``Match exec``). Unlike the protected-instruction
gate this is a routine user edit: the prompt offers once/session/always and
honors --yolo. Fail-closed when no interactive/gateway channel exists.
"""
def _check_approval_required_write(paths: list[str], task_id: str = "default") -> str | None:
"""Gate a write/patch touching an approval-required path (``~/.ssh/config`` can steer
execution via ``ProxyCommand``). Routine gate: once/session/always, honors --yolo,
fail-closed without an interactive/gateway channel."""
try:
from agent.file_safety import is_write_approval_required
except Exception:
@@ -326,26 +260,22 @@ def _check_approval_required_write(paths: list[str],
description = (
f"Write to SSH client config file(s): {display_targets}. "
"The SSH config can carry ProxyCommand / Match exec directives that "
"run commands, so writes require your approval."
)
"run commands, so writes require your approval.")
blocked = (
f"BLOCKED: write to SSH config file(s) ({display_targets}) "
"{why} Do NOT retry it via another path (terminal, execute_code) "
"without the user's explicit consent."
)
"without the user's explicit consent.")
try:
import tools.approval as _approval
except Exception:
return blocked.format(why="requires approval but the approval "
"subsystem is unavailable.")
return blocked.format(why=_APPROVAL_UNAVAILABLE)
result = _approval._run_approval_gate(
pattern_key="ssh_config_write",
description=description,
display_target=f"<write to {display_targets}>",
cron_deny_message=blocked.format(
why="requires approval but this cron session denies it."),
cron_deny_message=blocked.format(why="requires approval but this cron session denies it."),
single_query_deny_message=blocked.format(
why="requires approval but single-query (-q) sessions run "
"without a user present to approve it. To allow flagged "
@@ -353,10 +283,7 @@ def _check_approval_required_write(paths: list[str],
"approve in config.yaml."),
autoapprove_log_prefix="ssh_config_write",
fail_closed_when_no_human=True,
no_human_block_message=blocked.format(
why="requires approval but no interactive user or gateway is "
"present to approve it."),
)
no_human_block_message=blocked.format(why=_NO_HUMAN))
if result.get("approved"):
return None
return result.get("message") or blocked.format(why="was denied.")
@@ -366,74 +293,41 @@ def _get_container_mirror_prefix_for_task(task_id: str = "default") -> str | Non
"""Return the container-side Hermes mirror prefix for persistent Docker file tools."""
try:
from tools.terminal_tool import (
_active_environments,
_env_lock,
_get_env_config,
_resolve_container_task_id,
)
_active_environments, _env_lock, _get_env_config, _resolve_container_task_id)
container_key = _resolve_container_task_id(task_id)
except Exception:
return None
try:
with _env_lock:
env = _active_environments.get(container_key) or _active_environments.get(task_id)
if env is not None:
if env.__class__.__name__ == "DockerEnvironment" and bool(
getattr(env, "_persistent", False)
):
return "/root/.hermes"
return None
persistent_docker = (env.__class__.__name__ == "DockerEnvironment"
and bool(getattr(env, "_persistent", False)))
return "/root/.hermes" if persistent_docker else None
config = _get_env_config()
except Exception:
return None
if config.get("env_type") == "docker" and config.get("container_persistent", True):
return "/root/.hermes"
return None
def _check_cross_profile_path(filepath: str, task_id: str = "default") -> str | None:
"""Soft-guard: warn when ``filepath`` lands on a host-side or Docker sandbox
MIRROR of Hermes state — a write the host process never reads (lost work).
Not profile isolation: the former cross-PROFILE guard was removed by
maintainer decision (profiles were never isolated). ``cross_profile=True``
on the tools still bypasses these mirror guards (name kept for replay compat).
Fails open on import error — the sensitive-path guard and denylist still apply.
"""
"""Soft-guard: warn when ``filepath`` lands on a host-side or Docker sandbox MIRROR of
Hermes state (a write the host never reads). Not profile isolation — that guard was
removed; ``cross_profile=True`` keeps bypassing this one for replay compat. Fails open."""
try:
from agent.file_safety import (
get_container_mirror_warning,
get_sandbox_mirror_warning,
)
from agent.file_safety import get_container_mirror_warning, get_sandbox_mirror_warning
except Exception:
return None
resolved = _resolved_or_raw(filepath, task_id)
warning = get_sandbox_mirror_warning(resolved)
if warning is not None:
return warning
return get_container_mirror_warning(
resolved,
mirror_prefix=_get_container_mirror_prefix_for_task(task_id),
)
return get_container_mirror_warning(resolved, mirror_prefix=_get_container_mirror_prefix_for_task(task_id))
def _check_binary_document_write(filepath: str, task_id: str = "default") -> str | None:
"""Reject text-tool writes that would corrupt a binary document.
``read_file`` auto-extracts Office/PDF to text, so the model plausibly
believes it holds the file's bytes and writes edited text back — which can
never form a valid container. Opaque formats (.docx/.xlsx/.pptx/.odt/...)
are always rejected; .pdf only when OVERWRITING an existing regular file
(raw PDF syntax is text-authorable, so new-file creation stays allowed).
"""
"""Reject text-tool writes that would corrupt a binary document (read_file showed
EXTRACTED text, so the model may write it back). Opaque formats are always rejected;
.pdf only when OVERWRITING an existing file (raw PDF syntax is text-authorable)."""
if has_opaque_document_extension(filepath):
ext = filepath[filepath.rfind("."):].lower()
return (
@@ -442,8 +336,7 @@ def _check_binary_document_write(filepath: str, task_id: str = "default") -> str
"corrupt the file (read_file showed you EXTRACTED text, not the real "
"bytes). Use the docx/xlsx/powerpoint skills or a library like "
"python-docx/openpyxl/python-pptx via the terminal to create or edit "
"this document."
)
"this document.")
if is_pdf_path(filepath):
try:
resolved = Path(_resolve_path_for_task(filepath, task_id))
@@ -456,75 +349,48 @@ def _check_binary_document_write(filepath: str, task_id: str = "default") -> str
"read_file showed you EXTRACTED text, not the real bytes — writing "
"text back would destroy the document. Use the pdf skill or a PDF "
"library via the terminal to modify it. (Creating a NEW .pdf file "
"is allowed.)"
)
"is allowed.)")
except OSError:
pass
return None
# ---------------------------------------------------------------------------
# Internal display text must never be persisted as file content
# ---------------------------------------------------------------------------
# ── Internal display text must never be persisted as file content ────────
_READ_DEDUP_STATUS_MESSAGE = (
"File unchanged since last read. The content from "
"the earlier read_file result in this conversation is "
"still current — refer to that instead of re-reading."
)
"still current — refer to that instead of re-reading.")
def _is_internal_file_status_text(content: str) -> bool:
"""True when content is the read_file dedup status message (verbatim or lightly framed).
Models echo the message verbatim OR wrap it with short framing ("Note:",
a trailing comment). Any write whose stripped body contains the full
message and is <=2x its length is status-dominated — a real file quoting
this message would be dramatically longer.
"""
"""True when content is the read_file dedup status message, verbatim or lightly framed
(contains the full message and is <=2x its length — a real file quoting it would be longer)."""
if not isinstance(content, str):
return False
stripped = content.strip()
if not stripped:
return False
if stripped == _READ_DEDUP_STATUS_MESSAGE:
return True
return (_READ_DEDUP_STATUS_MESSAGE in stripped
and len(stripped) <= 2 * len(_READ_DEDUP_STATUS_MESSAGE))
return bool(stripped) and _READ_DEDUP_STATUS_MESSAGE in stripped and (
len(stripped) <= 2 * len(_READ_DEDUP_STATUS_MESSAGE))
def _looks_like_read_file_line_numbered_content(content: str) -> bool:
"""True for content dominated by read_file's ``LINE_NUM|CONTENT`` display.
Rejects writes whose non-empty lines are mostly (>=60%) consecutive
numbered lines, while allowing sparse literal pipe content such as a
single ``1|value`` line.
"""
"""True for content dominated by read_file's ``LINE_NUM|CONTENT`` display (>=60% of
non-empty lines are consecutive numbered lines; a lone ``1|value`` is allowed)."""
if not isinstance(content, str):
return False
lines = [line for line in content.splitlines() if line.strip()]
if len(lines) < 2:
return False
numbered: list[int] = []
for line in lines:
prefix, sep, _rest = line.lstrip().partition("|")
if sep and prefix.isdigit():
numbered.append(int(prefix))
if len(numbered) < 2 or len(numbered) / len(lines) < 0.6:
return False
consecutive_pairs = sum(
1 for prev, current in zip(numbered, numbered[1:])
if current == prev + 1
)
consecutive_pairs = sum(1 for prev, current in zip(numbered, numbered[1:]) if current == prev + 1)
return consecutive_pairs >= len(numbered) - 1
def _is_internal_file_tool_content(content: str) -> bool:
"""Return True when content is file-tool display text, not intended file bytes."""
return (
_is_internal_file_status_text(content)
or _looks_like_read_file_line_numbered_content(content)
)
return _is_internal_file_status_text(content) or _looks_like_read_file_line_numbered_content(content)
+352 -178
View File
@@ -1,67 +1,321 @@
#!/usr/bin/env python3
"""Fuzzy find-and-replace for LLM-generated edits.
Tries an ordered chain of increasingly permissive matching strategies (see
:mod:`tools.fuzzy_match_strategies`) so whitespace, indentation, escaping and
Unicode drift in tool-call arguments still land on the intended region.
An ordered chain of increasingly permissive strategies (``STRATEGIES``) lets
whitespace, indentation, escaping and Unicode drift in tool-call arguments
still land on the intended region::
new_content, match_count, strategy, error = fuzzy_find_and_replace(
content, old_string, new_string, replace_all=False)
"""
import re
from difflib import SequenceMatcher
from typing import List, Optional, Tuple
from typing import Callable, Optional
from tools.fuzzy_match_strategies import ( # noqa: F401 — re-exported names
SIMILARITY_STRATEGIES,
STRATEGIES,
UNICODE_MAP,
_build_orig_to_norm_map,
_calculate_line_positions,
_invert_norm_map,
_map_normalized_positions,
_map_positions_norm_to_orig,
_norm_end_to_orig,
_strategy_block_anchor,
_strategy_context_aware,
_strategy_escape_normalized,
_strategy_exact,
_strategy_indentation_flexible,
_strategy_line_trimmed,
_strategy_trimmed_boundary,
_strategy_unicode_normalized,
_strategy_whitespace_normalized,
_unicode_normalize,
)
Span = tuple[int, int]
IDENTICAL_STRINGS_ERROR = (
"No edit was applied because old_string and new_string are identical. "
"Provide the existing text to replace in old_string and the changed "
"replacement text in new_string."
)
"replacement text in new_string.")
UNICODE_MAP = {
"\u201c": '"', "\u201d": '"', # smart double quotes
"\u2018": "'", "\u2019": "'", # smart single quotes
"\u2014": "--", "\u2013": "-", # em/en dashes
"\u2026": "...", "\u00a0": " ", # ellipsis and non-breaking space
"\u2212": "-", # typographic minus (math/scientific docs)
# Space-separator family (Zs): otherwise such files fall to the similarity fallback.
"\u2000": " ", "\u2001": " ", "\u2002": " ", "\u2003": " ",
"\u2004": " ", "\u2005": " ", "\u2006": " ", "\u2007": " ",
"\u2008": " ", "\u2009": " ", "\u200a": " ", "\u202f": " ",
"\u205f": " ", "\u3000": " "}
def _unicode_normalize(text: str) -> str:
"""Map typographic Unicode variants to ASCII equivalents."""
for char, repl in UNICODE_MAP.items():
text = text.replace(char, repl)
return text
# ── Position helpers ─────────────────────────────────────────────────────
def _calculate_line_positions(content_lines: list[str], start_line: int,
end_line: int, content_length: int) -> Span:
"""Character span covering ``content_lines[start_line:end_line]`` (end exclusive)."""
start_pos = sum(len(line) + 1 for line in content_lines[:start_line])
end_pos = sum(len(line) + 1 for line in content_lines[:end_line]) - 1
return start_pos, min(content_length, end_pos)
def _window_spans(content: str, content_lines: list[str], n: int,
accept: Callable[[int], bool]) -> list[Span]:
"""Spans of every ``n``-line window starting at ``i`` for which ``accept(i)``."""
return [
_calculate_line_positions(content_lines, i, i + n, len(content))
for i in range(len(content_lines) - n + 1) if accept(i)]
def _match_transformed_lines(content: str, pattern: str,
transform: Callable[[list[str]], list[str]]) -> list[Span]:
"""Match ``pattern`` against ``content`` after applying ``transform`` to each line block."""
content_lines = content.split('\n')
pattern_norm = transform(pattern.split('\n'))
n = len(pattern_norm)
return _window_spans(content, content_lines, n,
lambda i: transform(content_lines[i:i + n]) == pattern_norm)
def _strip_boundary(lines: list[str]) -> list[str]:
"""Strip whitespace on the first and last lines only."""
lines = list(lines)
lines[0] = lines[0].strip()
if len(lines) > 1:
lines[-1] = lines[-1].strip()
return lines
def _build_orig_to_norm_map(original: str) -> list[int]:
"""Map each original index to its index in ``_unicode_normalize(original)``
(replacements can expand one char into several). Length ``len(original)+1``;
the last entry is a sentinel one past the final character."""
result: list[int] = []
norm_pos = 0
for char in original:
result.append(norm_pos)
repl = UNICODE_MAP.get(char)
norm_pos += len(repl) if repl is not None else 1
result.append(norm_pos)
return result
def _invert_norm_map(orig_to_norm: list[int]) -> dict[int, int]:
"""norm_pos -> first original position mapping to it."""
inverted: dict[int, int] = {}
for orig_pos, norm_pos in enumerate(orig_to_norm[:-1]):
inverted.setdefault(norm_pos, orig_pos)
return inverted
def _norm_end_to_orig(orig_to_norm: list[int], orig_start: int, norm_end: int) -> int:
"""Walk from ``orig_start`` until the mapped position reaches ``norm_end``."""
orig_len = len(orig_to_norm) - 1
orig_end = orig_start
while orig_end < orig_len and orig_to_norm[orig_end] < norm_end:
orig_end += 1
return orig_end
def _map_positions_norm_to_orig(orig_to_norm: list[int], norm_matches: list[Span]) -> list[Span]:
"""Convert spans in the normalised string to original-string spans."""
norm_to_orig_start = _invert_norm_map(orig_to_norm)
results: list[Span] = []
for norm_start, norm_end in norm_matches:
if norm_start in norm_to_orig_start:
orig_start = norm_to_orig_start[norm_start]
results.append((orig_start, _norm_end_to_orig(orig_to_norm, orig_start, norm_end)))
return results
def _map_normalized_positions(original: str, normalized: str,
normalized_matches: list[Span]) -> list[Span]:
"""Best-effort span mapping for ``[ \\t]+`` -> ``' '`` whitespace collapsing."""
orig_to_norm = [] # orig_to_norm[i] = position in normalized
orig_idx = norm_idx = 0
while orig_idx < len(original) and norm_idx < len(normalized):
if original[orig_idx] == normalized[norm_idx]:
orig_to_norm.append(norm_idx)
orig_idx += 1
norm_idx += 1
elif original[orig_idx] in ' \t' and normalized[norm_idx] == ' ':
# Collapsed run: advance norm_idx only once the run is consumed.
orig_to_norm.append(norm_idx)
orig_idx += 1
if orig_idx < len(original) and original[orig_idx] not in ' \t':
norm_idx += 1
else:
# Extra whitespace in original, or a mismatch normalization should
# never produce — either way, pin to the current norm_idx.
orig_to_norm.append(norm_idx)
orig_idx += 1
orig_to_norm.extend([len(normalized)] * (len(original) - orig_idx))
norm_to_orig_start = {}
norm_to_orig_end = {}
for orig_pos, norm_pos in enumerate(orig_to_norm):
norm_to_orig_start.setdefault(norm_pos, orig_pos)
norm_to_orig_end[norm_pos] = orig_pos
original_matches = []
for norm_start, norm_end in normalized_matches:
if norm_start in norm_to_orig_start:
orig_start = norm_to_orig_start[norm_start]
else:
orig_start = min(i for i, n in enumerate(orig_to_norm) if n >= norm_start)
if norm_end - 1 in norm_to_orig_end:
orig_end = norm_to_orig_end[norm_end - 1] + 1
else:
orig_end = orig_start + (norm_end - norm_start)
# Absorb trailing collapsed whitespace only when the normalized match
# itself ended in a space; otherwise the first whitespace after the
# match is a word boundary that must survive.
if norm_end < len(normalized) and normalized[norm_end - 1] == ' ':
while orig_end < len(original) and original[orig_end] in ' \t':
orig_end += 1
original_matches.append((orig_start, min(orig_end, len(original))))
return original_matches
# ── Strategies ───────────────────────────────────────────────────────────
# Each takes ``(content, pattern)`` and returns ``(start, end)`` spans in the
# ORIGINAL content.
def _strategy_exact(content: str, pattern: str) -> list[Span]:
"""Strategy 1: exact, non-overlapping occurrences (str.replace semantics —
overlapping spans would corrupt the file under replace_all)."""
return [m.span() for m in re.finditer(re.escape(pattern), content)]
def _strategy_line_trimmed(content: str, pattern: str) -> list[Span]:
"""Strategy 2: strip each line before comparing."""
return _match_transformed_lines(content, pattern, lambda ls: [l.strip() for l in ls])
def _strategy_whitespace_normalized(content: str, pattern: str) -> list[Span]:
"""Strategy 3: collapse runs of spaces/tabs to a single space."""
def normalize(s):
return re.sub(r'[ \t]+', ' ', s)
content_normalized = normalize(content)
matches_in_normalized = _strategy_exact(content_normalized, normalize(pattern))
if not matches_in_normalized:
return []
return _map_normalized_positions(content, content_normalized, matches_in_normalized)
def _strategy_indentation_flexible(content: str, pattern: str) -> list[Span]:
"""Strategy 4: ignore leading indentation entirely."""
return _match_transformed_lines(content, pattern, lambda ls: [l.lstrip() for l in ls])
def _strategy_escape_normalized(content: str, pattern: str) -> list[Span]:
"""Strategy 5: treat literal ``\\n``/``\\t``/``\\r`` in the pattern as control chars."""
pattern_unescaped = pattern.replace('\\n', '\n').replace('\\t', '\t').replace('\\r', '\r')
if pattern_unescaped == pattern:
return []
return _strategy_exact(content, pattern_unescaped)
def _strategy_trimmed_boundary(content: str, pattern: str) -> list[Span]:
"""Strategy 6: strip whitespace on the first and last lines only."""
return _match_transformed_lines(content, pattern, _strip_boundary)
def _strategy_unicode_normalized(content: str, pattern: str) -> list[Span]:
"""Strategy 7: exact/line-trimmed match after Unicode->ASCII normalisation of both sides."""
norm_pattern = _unicode_normalize(pattern)
norm_content = _unicode_normalize(content)
if norm_content == content and norm_pattern == pattern:
return []
norm_matches = (_strategy_exact(norm_content, norm_pattern)
or _strategy_line_trimmed(norm_content, norm_pattern))
if not norm_matches:
return []
return _map_positions_norm_to_orig(_build_orig_to_norm_map(content), norm_matches)
def _strategy_block_anchor(content: str, pattern: str) -> list[Span]:
"""Strategy 8: anchor on first+last lines, similarity-score the middle."""
pattern_lines = _unicode_normalize(pattern).split('\n')
if len(pattern_lines) < 2:
return []
first_line = pattern_lines[0].strip()
last_line = pattern_lines[-1].strip()
n = len(pattern_lines)
# Match on normalized lines; compute offsets from the ORIGINAL lines so
# multi-char expansions (em-dash -> '--') don't shift positions.
norm_content_lines = _unicode_normalize(content).split('\n')
potential_matches = {
i for i in range(len(norm_content_lines) - n + 1)
if norm_content_lines[i].strip() == first_line
and norm_content_lines[i + n - 1].strip() == last_line}
# Looser thresholds (0.10/0.30) matched unrelated blocks; these are the safe floor.
threshold = 0.50 if len(potential_matches) == 1 else 0.70
pattern_middle = '\n'.join(pattern_lines[1:-1])
def similar(i: int) -> bool:
if i not in potential_matches:
return False
if n <= 2:
return True
content_middle = '\n'.join(norm_content_lines[i + 1:i + n - 1])
return SequenceMatcher(None, content_middle, pattern_middle).ratio() >= threshold
return _window_spans(content, content.split('\n'), n, similar)
def _strategy_context_aware(content: str, pattern: str) -> list[Span]:
"""Strategy 9 (last resort): anchored per-line similarity, every non-blank line >= 0.80.
The anchor pre-filter bounds the scan; the all-lines rule stops coincidental matches."""
pattern_lines = pattern.split('\n')
content_lines = content.split('\n')
n = len(pattern_lines)
if n > len(content_lines):
return []
first_pat = pattern_lines[0].strip()
last_pat = pattern_lines[-1].strip()
def _sim(a: str, b: str) -> float:
return 1.0 if a == b else SequenceMatcher(None, a, b).ratio()
def accept(i: int) -> bool:
block_lines = content_lines[i:i + n]
if _sim(first_pat, block_lines[0].strip()) < 0.80:
return False
if _sim(last_pat, block_lines[-1].strip()) < 0.80:
return False
return all(
not p_line.strip() or _sim(p_line.strip(), c_line.strip()) >= 0.80
for p_line, c_line in zip(pattern_lines, block_lines))
return _window_spans(content, content_lines, n, accept)
# Ordered chain: precise strategies first, similarity-based last.
STRATEGIES: list[tuple[str, Callable[[str, str], list[Span]]]] = [
("exact", _strategy_exact),
("line_trimmed", _strategy_line_trimmed),
("whitespace_normalized", _strategy_whitespace_normalized),
("indentation_flexible", _strategy_indentation_flexible),
("escape_normalized", _strategy_escape_normalized),
("trimmed_boundary", _strategy_trimmed_boundary),
("unicode_normalized", _strategy_unicode_normalized),
("block_anchor", _strategy_block_anchor),
("context_aware", _strategy_context_aware)]
# Matches from these only *approximately* resemble old_string — fine for one
# unique replacement, never safe under replace_all.
SIMILARITY_STRATEGIES = frozenset({"block_anchor", "context_aware"})
# ── Orchestrator ─────────────────────────────────────────────────────────
def is_already_applied(content: str, old_string: str, new_string: str) -> bool:
"""True when the requested edit is already present (re-sent edit -> success-shaped no-op).
Conservative: new_string must be non-trivial (>= 8 chars stripped) and
appear EXACTLY; when it differs from old_string, old_string must be gone.
"""
if not new_string or len(new_string.strip()) < 8:
"""True when the edit is already present (re-sent edit -> success-shaped no-op).
Conservative: new_string non-trivial (>= 8 chars) and present EXACTLY; old_string gone."""
if not new_string or len(new_string.strip()) < 8 or new_string not in content:
return False
if new_string not in content:
return False
if old_string == new_string:
return True
return old_string not in content
return old_string == new_string or old_string not in content
def _matched_regions(content: str, matches: List[Tuple[int, int]]) -> str:
def _matched_regions(content: str, matches: list[Span]) -> str:
return "".join(content[start:end] for start, end in matches)
def _format_match_locations(content: str, matches: List[Tuple[int, int]],
cap: int = 5) -> str:
def _format_match_locations(content: str, matches: list[Span], cap: int = 5) -> str:
"""Render up to ``cap`` match positions as 'L<line>: <snippet>' rows."""
rows = []
for start, _end in matches[:cap]:
@@ -81,7 +335,7 @@ def _format_match_locations(content: str, matches: List[Tuple[int, int]],
def fuzzy_find_and_replace(content: str, old_string: str, new_string: str,
replace_all: bool = False) -> Tuple[str, int, Optional[str], Optional[str]]:
replace_all: bool = False) -> tuple[str, int, Optional[str], Optional[str]]:
"""Find and replace via the strategy chain.
Returns ``(new_content, match_count, strategy_name, error)``; on failure
@@ -106,15 +360,13 @@ def fuzzy_find_and_replace(content: str, old_string: str, new_string: str,
return content, 0, None, (
f"Found {len(matches)} matches for old_string. "
f"Provide more context to make it unique, or use replace_all=True. "
f"Matches:\n{locations}"
)
f"Matches:\n{locations}")
if replace_all and len(matches) > 1 and strategy_name in SIMILARITY_STRATEGIES:
return content, 0, None, (
f"Found {len(matches)} approximate matches via the "
f"'{strategy_name}' strategy; replace_all only applies to exact "
f"matches. Provide the precise text (whitespace included) so an "
f"exact/line-trimmed match can be made."
)
f"exact/line-trimmed match can be made.")
# Non-exact matches came through some normalization, so new_string may
# carry serialization drift the file doesn't have.
@@ -125,13 +377,10 @@ def fuzzy_find_and_replace(content: str, old_string: str, new_string: str,
effective_new = _maybe_unescape_new_string(new_string, content, matches)
if strategy_name == "unicode_normalized":
effective_new = _preserve_unicode_in_replacement(
content, matches, old_string, effective_new,
)
effective_new = _preserve_unicode_in_replacement(content, matches, old_string, effective_new)
new_content = _apply_replacements(
content, matches, effective_new,
old_string=old_string if strategy_name != "exact" else None,
)
old_string=old_string if strategy_name != "exact" else None)
return new_content, len(matches), strategy_name, None
return content, 0, None, "Could not find a match for old_string in the file"
@@ -139,14 +388,10 @@ def fuzzy_find_and_replace(content: str, old_string: str, new_string: str,
# ── Escape-drift guards ──────────────────────────────────────────────────
def _detect_escape_drift(content: str, matches: List[Tuple[int, int]],
def _detect_escape_drift(content: str, matches: list[Span],
old_string: str, new_string: str) -> Optional[str]:
"""Error string when new_string carries tool-call escape artifacts, else None.
Fires on ``\\'``/``\\"`` present in both old_string and new_string but
absent from the matched region (spurious shell-style escaping), and on
JSON double-escaped backslash runs (see ``_detect_backslash_doubling``).
"""
"""Error string when new_string carries tool-call escape artifacts, else None:
``\\'``/``\\"`` in both strings but not the matched region, or doubled backslash runs."""
has_quote_suspects = "\\'" in new_string or '\\"' in new_string
if not has_quote_suspects and "\\" not in old_string:
return None
@@ -163,47 +408,27 @@ def _detect_escape_drift(content: str, matches: List[Tuple[int, int]],
f"serialization artifact where an apostrophe or quote got "
f"prefixed with a spurious backslash. Re-read the file with "
f"read_file and pass old_string/new_string without "
f"backslash-escaping {plain!r} characters."
)
f"backslash-escaping {plain!r} characters.")
return _detect_backslash_doubling(matched_regions, old_string, new_string)
def _backslash_runs(s: str) -> List[int]:
def _backslash_runs(s: str) -> list[int]:
"""Lengths of maximal backslash runs in ``s``, in order."""
runs: List[int] = []
n = 0
for ch in s:
if ch == "\\":
n += 1
elif n:
runs.append(n)
n = 0
if n:
runs.append(n)
return runs
return [len(run) for run in re.findall(r"\\+", s)]
def _detect_backslash_doubling(matched_regions: str, old_string: str,
new_string: str) -> Optional[str]:
"""Detect old_string whose every backslash run is exactly 2x the file's.
That pattern means the arguments were JSON-escaped one extra time; a
similarity strategy still matches, and writing new_string verbatim would
double every backslash in the file. Requires the same run count, a
non-trivial signal (a run >= 2 or 2+ runs), and new_string not already
matching the file's counts.
"""
"""Detect old_string whose every backslash run is exactly 2x the file's (arguments
JSON-escaped one extra time). Requires the same run count, a non-trivial signal
(a run >= 2 or 2+ runs), and new_string not already matching the file's counts."""
old_runs = _backslash_runs(old_string)
file_runs = _backslash_runs(matched_regions)
if not old_runs or not file_runs or len(old_runs) != len(file_runs):
return None
if old_runs == file_runs:
return None
if any(o != f * 2 for o, f in zip(old_runs, file_runs)):
return None
if not (any(f >= 2 for f in file_runs) or len(file_runs) >= 2):
return None
if _backslash_runs(new_string) == file_runs:
if (not old_runs or not file_runs or len(old_runs) != len(file_runs)
or old_runs == file_runs
or any(o != f * 2 for o, f in zip(old_runs, file_runs))
or not (any(f >= 2 for f in file_runs) or len(file_runs) >= 2)
or _backslash_runs(new_string) == file_runs):
return None
return (
"Escape-drift detected: every backslash run in old_string is exactly "
@@ -212,29 +437,20 @@ def _detect_backslash_doubling(matched_regions: str, old_string: str,
"were JSON-escaped one extra time; applying new_string verbatim would "
"double every backslash in the file. Re-read the file with read_file "
"and resend old_string/new_string with the backslash counts exactly "
"as they appear in the file."
)
"as they appear in the file.")
def _maybe_unescape_new_string(new_string: str, content: str,
matches: List[Tuple[int, int]]) -> str:
"""Convert literal ``\\t``/``\\r`` in new_string to control chars, per sequence,
only when the matched file region already contains the real control char.
Files that legitimately contain the two-char string (e.g. ``sep = "\\t"``)
have a backslash+t in the region, not a tab, so they're left alone.
``\\n`` is deliberately excluded: newlines serialize correctly through
JSON and rewriting them would mangle escape sequences in source literals.
"""
def _maybe_unescape_new_string(new_string: str, content: str, matches: list[Span]) -> str:
"""Convert literal ``\\t``/``\\r`` in new_string to control chars, per sequence, only
when the matched region already contains the real control char (so ``sep = "\\t"`` files
are left alone). ``\\n`` is excluded: rewriting it would mangle source escape literals."""
if "\\t" not in new_string and "\\r" not in new_string:
return new_string
matched_regions = _matched_regions(content, matches)
out = new_string
if "\\t" in out and "\t" in matched_regions:
out = out.replace("\\t", "\t")
if "\\r" in out and "\r" in matched_regions:
out = out.replace("\\r", "\r")
return out
for literal, control in (("\\t", "\t"), ("\\r", "\r")):
if literal in new_string and control in matched_regions:
new_string = new_string.replace(literal, control)
return new_string
# ── Replacement shaping ──────────────────────────────────────────────────
@@ -244,20 +460,13 @@ def _leading_whitespace(line: str) -> str:
def _first_meaningful_line(text: str) -> Optional[str]:
for line in text.split("\n"):
if line.strip():
return line
return None
return next((line for line in text.split("\n") if line.strip()), None)
def _reindent_replacement(file_region: str, old_string: str, new_string: str) -> str:
"""Re-anchor ``new_string``'s indentation onto the file's actual base indent.
After a non-exact match the LLM's base indent (first non-blank line of
old_string) may differ from the file's. Each non-blank new_string line
swaps the LLM base prefix for the file's, preserving relative nesting;
lines shallower than the LLM base are anchored to the file base.
"""
"""Re-anchor ``new_string``'s indentation onto the file's actual base indent after a
non-exact match: swap the LLM base prefix (first non-blank old_string line) for the
file's, preserving relative nesting; shallower lines anchor to the file base."""
if not new_string:
return new_string
old_first = _first_meaningful_line(old_string)
@@ -269,7 +478,7 @@ def _reindent_replacement(file_region: str, old_string: str, new_string: str) ->
if old_indent == file_indent:
return new_string
out_lines: List[str] = []
out_lines: list[str] = []
for line in new_string.split("\n"):
if not line.strip():
out_lines.append(line)
@@ -280,17 +489,10 @@ def _reindent_replacement(file_region: str, old_string: str, new_string: str) ->
return "\n".join(out_lines)
def _preserve_unicode_in_replacement(
content: str, matches: List[Tuple[int, int]],
old_string: str, new_string: str,
) -> str:
"""Apply only the old->new edits onto the file's original (Unicode) text.
After a unicode_normalized match, writing the LLM's ASCII new_string
verbatim would flatten the file's em-dashes/smart quotes. Diff the
normalized old_string against new_string and keep the file's original
characters for every ``equal`` span.
"""
def _preserve_unicode_in_replacement(content: str, matches: list[Span],
old_string: str, new_string: str) -> str:
"""Apply only the old->new edits onto the file's original (Unicode) text, so a
unicode_normalized match doesn't flatten the file's em-dashes/smart quotes."""
file_region = _matched_regions(content, matches)
norm_old = _unicode_normalize(old_string)
if norm_old != _unicode_normalize(file_region):
@@ -299,7 +501,7 @@ def _preserve_unicode_in_replacement(
file_orig_to_norm = _build_orig_to_norm_map(file_region)
file_norm_to_orig = _invert_norm_map(file_orig_to_norm)
result_parts: List[str] = []
result_parts: list[str] = []
for tag, i1, i2, j1, j2 in SequenceMatcher(None, norm_old, new_string).get_opcodes():
if tag == "equal":
orig_start = file_norm_to_orig.get(i1, 0)
@@ -310,13 +512,10 @@ def _preserve_unicode_in_replacement(
return "".join(result_parts)
def _apply_replacements(content: str, matches: List[Tuple[int, int]],
def _apply_replacements(content: str, matches: list[Span],
new_string: str, old_string: Optional[str] = None) -> str:
"""Splice ``new_string`` over each span (end-to-start so offsets stay valid).
``old_string`` non-None signals a non-exact match: new_string is
re-indented per region to the file's actual indentation.
"""
"""Splice ``new_string`` over each span (end-to-start so offsets stay valid);
``old_string`` non-None (non-exact match) re-indents it per region."""
result = content
for start, end in sorted(matches, key=lambda x: x[0], reverse=True):
adjusted = new_string
@@ -330,12 +529,9 @@ def _apply_replacements(content: str, matches: List[Tuple[int, int]],
def _visualize_whitespace(line: str) -> str:
"""Render the leading whitespace run visibly (→ = tab, · = space)."""
i = 0
prefix = []
while i < len(line) and line[i] in (" ", "\t"):
prefix.append("→" if line[i] == "\t" else "·")
i += 1
return "".join(prefix) + line[i:]
stripped = line.lstrip(" \t")
prefix = line[:len(line) - len(stripped)]
return prefix.replace("\t", "→").replace(" ", "·") + stripped
def find_closest_lines(old_string: str, content: str, context_lines: int = 2, max_results: int = 3) -> str:
@@ -347,25 +543,15 @@ def find_closest_lines(old_string: str, content: str, context_lines: int = 2, ma
if not old_lines or not content_lines:
return ""
anchor = old_lines[0].strip()
anchor = old_lines[0].strip() or next((l.strip() for l in old_lines if l.strip()), "")
if not anchor:
candidates = [l.strip() for l in old_lines if l.strip()]
if not candidates:
return ""
anchor = candidates[0]
scored = []
for i, line in enumerate(content_lines):
stripped = line.strip()
if not stripped:
continue
ratio = SequenceMatcher(None, anchor, stripped).ratio()
if ratio > 0.3:
scored.append((ratio, i))
if not scored:
return ""
scored.sort(key=lambda x: -x[0])
top = scored[:max_results]
scored = sorted(((SequenceMatcher(None, anchor, line.strip()).ratio(), i)
for i, line in enumerate(content_lines) if line.strip()), key=lambda x: -x[0])
top = [s for s in scored if s[0] > 0.3][:max_results]
if not top:
return ""
parts = []
seen_ranges = set()
@@ -376,11 +562,7 @@ def find_closest_lines(old_string: str, content: str, context_lines: int = 2, ma
continue
seen_ranges.add((start, end))
parts.append("\n".join(
f"{start + j + 1:4d}| {content_lines[start + j]}"
for j in range(end - start)
))
if not parts:
return ""
f"{start + j + 1:4d}| {content_lines[start + j]}" for j in range(end - start)))
result = "\n---\n".join(parts)
# Whitespace-shaped miss: best line equals the anchor once stripped. Show
@@ -391,23 +573,15 @@ def find_closest_lines(old_string: str, content: str, context_lines: int = 2, ma
"\n\nWhitespace difference detected (→ = tab, · = space):\n"
f" file has: {_visualize_whitespace(best_line)}\n"
f" you sent: {_visualize_whitespace(old_lines[0])}\n"
"Use the exact whitespace shown in 'file has'."
)
"Use the exact whitespace shown in 'file has'.")
return result
def format_no_match_hint(error: Optional[str], match_count: int,
old_string: str, content: str) -> str:
"""'\\n\\nDid you mean...' snippet for plain no-match errors only, else ''.
Ambiguous-match, escape-drift and identical-strings errors also have
``match_count == 0`` but a hint would mislead there.
"""
if match_count != 0:
return ""
if not error or not error.startswith("Could not find"):
"""'\\n\\nDid you mean...' snippet for plain no-match errors only, else '' (ambiguous /
escape-drift / identical errors also have ``match_count == 0`` but a hint would mislead)."""
if match_count != 0 or not error or not error.startswith("Could not find"):
return ""
hint = find_closest_lines(old_string, content)
if not hint:
return ""
return "\n\nDid you mean one of these sections?\n" + hint
return "\n\nDid you mean one of these sections?\n" + hint if hint else ""
-334
View File
@@ -1,334 +0,0 @@
"""Matching strategies for :mod:`tools.fuzzy_match`.
Each strategy takes ``(content, pattern)`` and returns a list of
``(start, end)`` spans in the ORIGINAL content. ``STRATEGIES`` is the ordered
chain the orchestrator tries; ``SIMILARITY_STRATEGIES`` names the ones whose
matches are approximate and therefore unsafe under ``replace_all``.
"""
import re
from difflib import SequenceMatcher
from typing import Callable, Dict, List, Tuple
UNICODE_MAP = {
"\u201c": '"', "\u201d": '"', # smart double quotes
"\u2018": "'", "\u2019": "'", # smart single quotes
"\u2014": "--", "\u2013": "-", # em/en dashes
"\u2026": "...", "\u00a0": " ", # ellipsis and non-breaking space
"\u2212": "-", # typographic minus (math/scientific docs)
# Space-separator family (Zs) beyond NBSP: files with typographic spacing
# otherwise miss every precise strategy and fall through to the
# similarity fallback, which can pick the wrong region.
"\u2000": " ", "\u2001": " ", "\u2002": " ", "\u2003": " ",
"\u2004": " ", "\u2005": " ", "\u2006": " ", "\u2007": " ",
"\u2008": " ", "\u2009": " ", "\u200a": " ", "\u202f": " ",
"\u205f": " ", "\u3000": " ",
}
Span = Tuple[int, int]
def _unicode_normalize(text: str) -> str:
"""Map typographic Unicode variants to ASCII equivalents."""
for char, repl in UNICODE_MAP.items():
text = text.replace(char, repl)
return text
# ── Position helpers ─────────────────────────────────────────────────────
def _calculate_line_positions(content_lines: List[str], start_line: int,
end_line: int, content_length: int) -> Span:
"""Character span covering ``content_lines[start_line:end_line]`` (end exclusive)."""
start_pos = sum(len(line) + 1 for line in content_lines[:start_line])
end_pos = sum(len(line) + 1 for line in content_lines[:end_line]) - 1
return start_pos, min(content_length, end_pos)
def _match_transformed_lines(content: str, pattern: str,
transform: Callable[[str], str]) -> List[Span]:
"""Match ``pattern`` against ``content`` after applying ``transform`` per line."""
content_lines = content.split('\n')
norm_lines = [transform(line) for line in content_lines]
pattern_norm = '\n'.join(transform(line) for line in pattern.split('\n'))
n = pattern_norm.count('\n') + 1
matches = []
for i in range(len(norm_lines) - n + 1):
if '\n'.join(norm_lines[i:i + n]) == pattern_norm:
matches.append(_calculate_line_positions(content_lines, i, i + n, len(content)))
return matches
def _build_orig_to_norm_map(original: str) -> List[int]:
"""Map each original index to its index in ``_unicode_normalize(original)``.
UNICODE_MAP replacements can expand one char into several, so the map is
needed to translate normalised spans back. Length is ``len(original)+1``;
the last entry is a sentinel one past the final character.
"""
result: List[int] = []
norm_pos = 0
for char in original:
result.append(norm_pos)
repl = UNICODE_MAP.get(char)
norm_pos += len(repl) if repl is not None else 1
result.append(norm_pos)
return result
def _invert_norm_map(orig_to_norm: List[int]) -> Dict[int, int]:
"""norm_pos -> first original position mapping to it."""
inverted: Dict[int, int] = {}
for orig_pos, norm_pos in enumerate(orig_to_norm[:-1]):
if norm_pos not in inverted:
inverted[norm_pos] = orig_pos
return inverted
def _norm_end_to_orig(orig_to_norm: List[int], orig_start: int, norm_end: int) -> int:
"""Walk from ``orig_start`` until the mapped position reaches ``norm_end``."""
orig_len = len(orig_to_norm) - 1
orig_end = orig_start
while orig_end < orig_len and orig_to_norm[orig_end] < norm_end:
orig_end += 1
return orig_end
def _map_positions_norm_to_orig(orig_to_norm: List[int],
norm_matches: List[Span]) -> List[Span]:
"""Convert spans in the normalised string to original-string spans."""
norm_to_orig_start = _invert_norm_map(orig_to_norm)
results: List[Span] = []
for norm_start, norm_end in norm_matches:
if norm_start not in norm_to_orig_start:
continue
orig_start = norm_to_orig_start[norm_start]
results.append((orig_start, _norm_end_to_orig(orig_to_norm, orig_start, norm_end)))
return results
def _map_normalized_positions(original: str, normalized: str,
normalized_matches: List[Span]) -> List[Span]:
"""Best-effort span mapping for ``[ \\t]+`` -> ``' '`` whitespace collapsing."""
orig_to_norm = [] # orig_to_norm[i] = position in normalized
orig_idx = norm_idx = 0
while orig_idx < len(original) and norm_idx < len(normalized):
if original[orig_idx] == normalized[norm_idx]:
orig_to_norm.append(norm_idx)
orig_idx += 1
norm_idx += 1
elif original[orig_idx] in ' \t' and normalized[norm_idx] == ' ':
# Collapsed run: advance norm_idx only once the run is consumed.
orig_to_norm.append(norm_idx)
orig_idx += 1
if orig_idx < len(original) and original[orig_idx] not in ' \t':
norm_idx += 1
else:
# Extra whitespace in original, or a mismatch that normalization
# should never produce — either way, pin to the current norm_idx.
orig_to_norm.append(norm_idx)
orig_idx += 1
while orig_idx < len(original):
orig_to_norm.append(len(normalized))
orig_idx += 1
norm_to_orig_start = {}
norm_to_orig_end = {}
for orig_pos, norm_pos in enumerate(orig_to_norm):
if norm_pos not in norm_to_orig_start:
norm_to_orig_start[norm_pos] = orig_pos
norm_to_orig_end[norm_pos] = orig_pos
original_matches = []
for norm_start, norm_end in normalized_matches:
if norm_start in norm_to_orig_start:
orig_start = norm_to_orig_start[norm_start]
else:
orig_start = min(i for i, n in enumerate(orig_to_norm) if n >= norm_start)
if norm_end - 1 in norm_to_orig_end:
orig_end = norm_to_orig_end[norm_end - 1] + 1
else:
orig_end = orig_start + (norm_end - norm_start)
# Absorb trailing collapsed whitespace only when the normalized match
# itself ended in a space; otherwise the first whitespace after the
# match is a word boundary that must survive (#52491).
if norm_end < len(normalized) and normalized[norm_end - 1] == ' ':
while orig_end < len(original) and original[orig_end] in ' \t':
orig_end += 1
original_matches.append((orig_start, min(orig_end, len(original))))
return original_matches
# ── Strategies ───────────────────────────────────────────────────────────
def _strategy_exact(content: str, pattern: str) -> List[Span]:
"""Strategy 1: exact, non-overlapping occurrences (str.replace semantics)."""
matches = []
start = 0
while True:
pos = content.find(pattern, start)
if pos == -1:
break
matches.append((pos, pos + len(pattern)))
# Advance past the whole match: overlapping spans would corrupt the
# file under replace_all (reverse-order apply on stale offsets).
start = pos + len(pattern)
return matches
def _strategy_line_trimmed(content: str, pattern: str) -> List[Span]:
"""Strategy 2: strip each line before comparing."""
return _match_transformed_lines(content, pattern, str.strip)
def _strategy_whitespace_normalized(content: str, pattern: str) -> List[Span]:
"""Strategy 3: collapse runs of spaces/tabs to a single space."""
def normalize(s):
return re.sub(r'[ \t]+', ' ', s)
content_normalized = normalize(content)
matches_in_normalized = _strategy_exact(content_normalized, normalize(pattern))
if not matches_in_normalized:
return []
return _map_normalized_positions(content, content_normalized, matches_in_normalized)
def _strategy_indentation_flexible(content: str, pattern: str) -> List[Span]:
"""Strategy 4: ignore leading indentation entirely."""
return _match_transformed_lines(content, pattern, str.lstrip)
def _strategy_escape_normalized(content: str, pattern: str) -> List[Span]:
"""Strategy 5: treat literal ``\\n``/``\\t``/``\\r`` in the pattern as control chars."""
pattern_unescaped = pattern.replace('\\n', '\n').replace('\\t', '\t').replace('\\r', '\r')
if pattern_unescaped == pattern:
return []
return _strategy_exact(content, pattern_unescaped)
def _strategy_trimmed_boundary(content: str, pattern: str) -> List[Span]:
"""Strategy 6: strip whitespace on the first and last lines only."""
pattern_lines = pattern.split('\n')
pattern_lines[0] = pattern_lines[0].strip()
if len(pattern_lines) > 1:
pattern_lines[-1] = pattern_lines[-1].strip()
modified_pattern = '\n'.join(pattern_lines)
n = len(pattern_lines)
content_lines = content.split('\n')
matches = []
for i in range(len(content_lines) - n + 1):
check_lines = content_lines[i:i + n]
check_lines[0] = check_lines[0].strip()
if n > 1:
check_lines[-1] = check_lines[-1].strip()
if '\n'.join(check_lines) == modified_pattern:
matches.append(_calculate_line_positions(content_lines, i, i + n, len(content)))
return matches
def _strategy_unicode_normalized(content: str, pattern: str) -> List[Span]:
"""Strategy 7: exact/line-trimmed match after Unicode->ASCII normalisation of both sides."""
norm_pattern = _unicode_normalize(pattern)
norm_content = _unicode_normalize(content)
if norm_content == content and norm_pattern == pattern:
return []
norm_matches = _strategy_exact(norm_content, norm_pattern)
if not norm_matches:
norm_matches = _strategy_line_trimmed(norm_content, norm_pattern)
if not norm_matches:
return []
return _map_positions_norm_to_orig(_build_orig_to_norm_map(content), norm_matches)
def _strategy_block_anchor(content: str, pattern: str) -> List[Span]:
"""Strategy 8: anchor on first+last lines, similarity-score the middle."""
pattern_lines = _unicode_normalize(pattern).split('\n')
if len(pattern_lines) < 2:
return []
first_line = pattern_lines[0].strip()
last_line = pattern_lines[-1].strip()
n = len(pattern_lines)
# Match on normalized lines; compute offsets from the ORIGINAL lines so
# multi-char expansions (em-dash -> '--') don't shift positions.
norm_content_lines = _unicode_normalize(content).split('\n')
orig_content_lines = content.split('\n')
potential_matches = [
i for i in range(len(norm_content_lines) - n + 1)
if norm_content_lines[i].strip() == first_line
and norm_content_lines[i + n - 1].strip() == last_line
]
# Looser thresholds (0.10/0.30) matched unrelated blocks; these are the safe floor.
threshold = 0.50 if len(potential_matches) == 1 else 0.70
matches = []
for i in potential_matches:
if n <= 2:
similarity = 1.0
else:
content_middle = '\n'.join(norm_content_lines[i + 1:i + n - 1])
pattern_middle = '\n'.join(pattern_lines[1:-1])
similarity = SequenceMatcher(None, content_middle, pattern_middle).ratio()
if similarity >= threshold:
matches.append(_calculate_line_positions(orig_content_lines, i, i + n, len(content)))
return matches
def _strategy_context_aware(content: str, pattern: str) -> List[Span]:
"""Strategy 9 (last resort): anchored per-line similarity, every non-blank line >= 0.80.
The first/last-line anchor pre-filter keeps a miss from being an
O(file x pattern) scan; the all-lines requirement stops one coincidental
line match from replacing an unrelated block.
"""
pattern_lines = pattern.split('\n')
content_lines = content.split('\n')
n = len(pattern_lines)
if n > len(content_lines):
return []
first_pat = pattern_lines[0].strip()
last_pat = pattern_lines[-1].strip()
ANCHOR_THRESHOLD = 0.80
def _sim(a: str, b: str) -> float:
if a == b:
return 1.0
return SequenceMatcher(None, a, b).ratio()
matches = []
for i in range(len(content_lines) - n + 1):
block_lines = content_lines[i:i + n]
if _sim(first_pat, block_lines[0].strip()) < ANCHOR_THRESHOLD:
continue
if _sim(last_pat, block_lines[-1].strip()) < ANCHOR_THRESHOLD:
continue
all_match = True
for p_line, c_line in zip(pattern_lines, block_lines):
p_stripped = p_line.strip()
if p_stripped and _sim(p_stripped, c_line.strip()) < 0.80:
all_match = False
break
if all_match:
matches.append(_calculate_line_positions(content_lines, i, i + n, len(content)))
return matches
# Ordered chain: precise strategies first, similarity-based last.
STRATEGIES: List[Tuple[str, Callable[[str, str], List[Span]]]] = [
("exact", _strategy_exact),
("line_trimmed", _strategy_line_trimmed),
("whitespace_normalized", _strategy_whitespace_normalized),
("indentation_flexible", _strategy_indentation_flexible),
("escape_normalized", _strategy_escape_normalized),
("trimmed_boundary", _strategy_trimmed_boundary),
("unicode_normalized", _strategy_unicode_normalized),
("block_anchor", _strategy_block_anchor),
("context_aware", _strategy_context_aware),
]
# Matches from these only *approximately* resemble old_string — fine for one
# unique replacement, never safe under replace_all.
SIMILARITY_STRATEGIES = frozenset({"block_anchor", "context_aware"})