fix: Windows path handling in search_files rg calls and patch escape drift (#84378)

* fix: Windows path handling in search_files rg calls and patch escape drift

Two related Windows failures from a live session (Windows 10, git-bash
terminal backend, winget-installed native ripgrep):

1. search_files was unusable on drive-letter paths. _escape_shell_arg
   rewrites C:\... to the MSYS form /c/... so bash builtins resolve it,
   but rg is a native Windows binary and Hermes disables MSYS argument
   conversion for its bash subprocesses (MSYS_NO_PATHCONV=1 /
   MSYS2_ARG_CONV_EXCL=*, see _apply_windows_msys_bash_env_defaults) —
   so nothing ever translated /c/... back and every search failed with
   'The system cannot find the path specified. (os error 3)'.

   Fix: new _escape_native_tool_arg emits the forward-slash NATIVE form
   (C:/Users/...), which native binaries accept, bash passes through
   untouched, and MSYS builds also handle. Applied to the six rg call
   sites (content search, --files search x2, zero-match probe x3); the
   grep fallback keeps the MSYS form since MSYS grep wants it.

2. The patch tool silently doubled backslash runs when tool-call args
   arrived JSON-escaped one extra time (file had \ where old_string
   had \\). Similarity strategies (context_aware) matched the region
   anyway and wrote new_string verbatim, corrupting every backslash run
   (reproduced: 6 backslashes on the line became 12). _detect_escape_drift
   now also blocks when every backslash run in old_string is exactly twice
   its counterpart in the matched region and new_string repeats the
   doubling — with guardrails so exact matches, intentional backslash
   edits, model-corrected new_strings, and single weak-signal runs all
   still apply. Blocking returns the standard escape-drift guidance so
   the model re-reads and retries with correct counts.

Tests: TestEscapeNativeToolArg (5 cases, including an end-to-end
_search_with_rg command capture) and TestBackslashDoublingDrift (6
cases). The 8 pre-existing failures in tests/tools/test_file_operations.py
on a Windows host (umask/symlink POSIX assumptions) are identical on
unmodified main and unrelated.

* fix: shell linters get native Windows paths too (node C:\c\... double-prefix)

Same class as the rg fix: LINTERS commands (python -m py_compile,
node --check, npx tsc, go vet, rustfmt) invoke native Windows binaries,
but _check_lint interpolated the MSYS /c/... form. node resolves that
as C:\c\Users\... (double-prefixed), so on Windows hosts every .js
write reported a phantom ENOENT lint failure that could mask real
syntax errors (issue #84303). Route the {file} arg through
_escape_native_tool_arg like the rg call sites.

Regression test asserts node --check receives 'C:/...' and never
'/c/...'.
This commit is contained in:
Teknium
2026-08-12 01:16:34 -07:00
committed by GitHub
parent 14692ec917
commit 07ee4a2ec8
4 changed files with 309 additions and 23 deletions
+94
View File
@@ -804,3 +804,97 @@ class TestByteLayerBinaryDetection:
result = ops.read_file("/tmp/a.out")
assert result.is_binary is True
class TestEscapeNativeToolArg:
"""Regression tests for _escape_native_tool_arg (Windows native-binary paths).
Live failure (Windows, Aug 2026): search_files passed rg the MSYS form
(/c/Users/...) that _escape_shell_arg produces, but Hermes sets
MSYS_NO_PATHCONV=1 / MSYS2_ARG_CONV_EXCL=* for its bash subprocesses,
so nothing converted the path back for the native (winget) ripgrep
binary — every search on a drive-letter path failed with
"The system cannot find the path specified. (os error 3)". Native
Windows binaries need C:/... (forward-slash native), which bash also
passes through untouched.
"""
def _ops(self, mock_env):
return ShellFileOperations(mock_env)
def test_windows_native_path_kept_native(self, mock_env, monkeypatch):
import tools.environments.local as local_mod
monkeypatch.setattr(local_mod, "_IS_WINDOWS", True)
ops = self._ops(mock_env)
out = ops._escape_native_tool_arg(r"C:\Users\alice\project")
assert out == "'C:/Users/alice/project'"
def test_msys_path_translated_back_to_native(self, mock_env, monkeypatch):
import tools.environments.local as local_mod
monkeypatch.setattr(local_mod, "_IS_WINDOWS", True)
ops = self._ops(mock_env)
out = ops._escape_native_tool_arg("/c/Users/alice/project")
assert out == "'C:/Users/alice/project'"
def test_posix_path_untouched_on_windows(self, mock_env, monkeypatch):
"""Multi-segment POSIX paths (/home/x, /tmp/y) are not drive paths."""
import tools.environments.local as local_mod
monkeypatch.setattr(local_mod, "_IS_WINDOWS", True)
ops = self._ops(mock_env)
assert ops._escape_native_tool_arg("/tmp/workdir") == "'/tmp/workdir'"
def test_non_windows_behaves_like_escape_shell_arg(self, mock_env, monkeypatch):
import tools.environments.local as local_mod
monkeypatch.setattr(local_mod, "_IS_WINDOWS", False)
ops = self._ops(mock_env)
assert ops._escape_native_tool_arg("/home/u/it's here") == (
ops._escape_shell_arg("/home/u/it's here")
)
def test_rg_content_search_uses_native_form(self, mock_env, monkeypatch):
"""_search_with_rg must pass the path in native C:/ form to rg."""
import tools.environments.local as local_mod
monkeypatch.setattr(local_mod, "_IS_WINDOWS", True)
commands = []
def side_effect(command, **kwargs):
commands.append(command)
if "test -e" in command:
return {"output": "exists", "returncode": 0}
if "command -v" in command:
return {"output": "yes", "returncode": 0}
return {"output": "", "returncode": 0}
mock_env.execute.side_effect = side_effect
ops = self._ops(mock_env)
ops.search("needle", path=r"C:\Users\alice\project")
rg_cmds = [c for c in commands if "rg " in c or c.startswith("rg")]
assert rg_cmds, f"no rg command captured in: {commands}"
assert any("'C:/Users/alice/project'" in c for c in rg_cmds), rg_cmds
assert all("/c/Users" not in c for c in rg_cmds), rg_cmds
def test_shell_linter_uses_native_form(self, mock_env, monkeypatch):
"""_check_lint must hand node/python/etc. the native C:/ path.
Regression for the double-prefix failure (#84303): node given the
MSYS /c/Users/... form resolves it as C:\\c\\Users\\... and every
.js write reports a phantom ENOENT lint error.
"""
import tools.environments.local as local_mod
monkeypatch.setattr(local_mod, "_IS_WINDOWS", True)
commands = []
def side_effect(command, **kwargs):
commands.append(command)
if "command -v" in command:
return {"output": "yes", "returncode": 0}
return {"output": "", "returncode": 0}
mock_env.execute.side_effect = side_effect
ops = self._ops(mock_env)
result = ops._check_lint(r"C:\Users\alice\app\main.js")
assert result.skipped is False
node_cmds = [c for c in commands if "node --check" in c]
assert node_cmds, f"no node command captured in: {commands}"
assert "'C:/Users/alice/app/main.js'" in node_cmds[0]
assert "/c/Users" not in node_cmds[0]
+85
View File
@@ -608,3 +608,88 @@ class TestContextAwareCorrectness:
# Was ~5.5s before anchoring; generous ceiling to avoid CI flake.
assert elapsed < 2.0, f"context_aware no-match took {elapsed:.2f}s"
class TestBackslashDoublingDrift:
"""Regression tests for the backslash-run doubling guard.
Live failure (Windows, Aug 2026): the model sent old_string/new_string
whose backslash runs were JSON-escaped one extra time (file had ``\``
where the args had ``\\``). The context_aware strategy matched the
region anyway and wrote new_string verbatim, doubling every backslash
in a Windows path inside a Python string literal. The guard must block
that case while never firing on intentional backslash edits.
"""
def setup_method(self):
from tools.fuzzy_match import fuzzy_find_and_replace
self.replace = fuzzy_find_and_replace
def _make(self, n_file: int, n_args: int):
b = "\\"
content = (
' "native `C:' + b * n_file + 'Users' + b * n_file
+ '<user>' + b * n_file + '...` paths. X "\n "next line"\n'
)
old = (
' "native `C:' + b * n_args + 'Users' + b * n_args
+ '<user>' + b * n_args + '...` paths. X "\n "next line"'
)
return content, old
def test_doubled_backslashes_blocked(self):
"""old/new with 2x the file's backslash runs must be rejected."""
content, old = self._make(n_file=2, n_args=4)
new = old + ' # touched'
result, count, strategy, err = self.replace(content, old, new)
assert count == 0
assert err is not None and "twice as long" in err
assert result == content # untouched
def test_matching_backslashes_apply(self):
"""Same edit with correct backslash counts applies exactly."""
content, old = self._make(n_file=2, n_args=2)
new = old.replace("next line", "next line edited")
result, count, strategy, err = self.replace(content, old, new)
assert count == 1 and err is None
assert "next line edited" in result
def test_intentional_backslash_reduction_exact_match_allowed(self):
"""Deliberately halving backslashes via an exact match is a real edit."""
b = "\\"
content = 'x = "a' + b * 4 + 'b"\ny = 1\n'
old = 'x = "a' + b * 4 + 'b"'
new = 'x = "a' + b * 2 + 'b"'
result, count, strategy, err = self.replace(content, old, new)
assert count == 1 and err is None
assert strategy == "exact"
def test_model_corrected_new_string_allowed(self):
"""Doubled old_string but corrected new_string writes the right bytes."""
b = "\\"
content, old = self._make(n_file=2, n_args=4)
new = old.replace(b * 4, b * 2).replace("next line", "corrected")
result, count, strategy, err = self.replace(content, old, new)
assert count == 1 and err is None
assert "corrected" in result
# No doubling in the output
assert b * 4 not in result
def test_single_prose_backslash_not_blocked(self):
"""A lone ``\`` vs ``\\`` in prose is too weak a signal to block."""
b = "\\"
content = "text with one " + b + " backslash here\nanother line\n"
old = "text with one " + b * 2 + " backslash here\nanother line"
new = old + " more"
result, count, strategy, err = self.replace(content, old, new)
assert err is None or "twice" not in err
def test_quote_drift_guard_still_fires(self):
"""The pre-existing apostrophe escape-drift guard must keep working."""
b = "\\"
content = "print('hello world')\nrest = 1\n"
old = "print(" + b + "'hello world" + b + "')\nrest = 1"
new = "print(" + b + "'hello there" + b + "')\nrest = 1"
result, count, strategy, err = self.replace(content, old, new)
assert count == 0
assert err is not None and "apostrophe" in err
+43 -8
View File
@@ -1144,6 +1144,34 @@ class ShellFileOperations(FileOperations):
# Use single quotes and escape any single quotes in the string
return "'" + arg.replace("'", "'\"'\"'") + "'"
def _escape_native_tool_arg(self, arg: str) -> str:
"""Escape a path argument destined for a NATIVE Windows binary.
``_escape_shell_arg`` rewrites Windows paths to the Git Bash MSYS
form (``/c/Users/x``) so bash builtins resolve them. But native
Windows binaries invoked from that bash (ripgrep installed via
winget/cargo/choco, native git, etc.) do not understand ``/c/...``
paths — and Hermes disables MSYS argument conversion for its bash
subprocesses (``MSYS_NO_PATHCONV=1`` / ``MSYS2_ARG_CONV_EXCL=*``,
see ``_apply_windows_msys_bash_env_defaults``), so nothing ever
translates the MSYS form back. The native tool then fails with
``The system cannot find the path specified. (os error 3)``.
The forward-slash native form (``C:/Users/x``) is the one spelling
every layer accepts: bash passes it through untouched (it is not an
absolute POSIX path, so no conversion applies even without the
opt-outs), and Windows APIs treat ``/`` and ``\\`` as equivalent
separators. MSYS builds of the same tools accept it too, so this is
safe regardless of which flavor of the binary is installed.
On non-Windows hosts this is exactly ``_escape_shell_arg``.
"""
from tools.environments.local import _IS_WINDOWS, _msys_to_windows_path
if _IS_WINDOWS and arg:
arg = _msys_to_windows_path(arg).replace("\\", "/")
return "'" + arg.replace("'", "'\"'\"'") + "'"
def _atomic_write(self, path: str, content: str) -> "ExecuteResult":
"""Write ``content`` to ``path`` atomically via temp-file + rename.
@@ -2279,8 +2307,12 @@ class ShellFileOperations(FileOperations):
if not self._has_command(base_cmd):
return LintResult(skipped=True, message=f"{base_cmd} not available")
# Run linter
cmd = linter_cmd.replace("{file}", self._escape_shell_arg(path))
# Run linter. Linters (python, node, tsc, go, rustfmt) are native
# Windows binaries on Windows hosts: they need the C:/... path form.
# The MSYS /c/... form makes node resolve the file as C:\c\Users\...
# (double-prefixed) — every .js write then reports a phantom ENOENT
# lint failure. Native form works for MSYS builds too.
cmd = linter_cmd.replace("{file}", self._escape_native_tool_arg(path))
result = self._exec(cmd, timeout=30)
if result.exit_code != 0 and _looks_like_linter_unusable(base_cmd, result.stdout):
@@ -2698,7 +2730,7 @@ class ShellFileOperations(FileOperations):
glob_expr = f" --glob {self._escape_shell_arg(file_glob)}" if file_glob else ""
probe = self._exec(
f"rg -i --count-matches{glob_expr} "
f"{self._escape_shell_arg(pattern)} {self._escape_shell_arg(path)} "
f"{self._escape_shell_arg(pattern)} {self._escape_native_tool_arg(path)} "
f"2>/dev/null | head -50",
timeout=30,
)
@@ -2720,7 +2752,7 @@ class ShellFileOperations(FileOperations):
# missing from results).
hidden = self._exec(
f"rg --hidden --no-ignore --count-matches{glob_expr} "
f"{self._escape_shell_arg(pattern)} {self._escape_shell_arg(path)} "
f"{self._escape_shell_arg(pattern)} {self._escape_native_tool_arg(path)} "
f"2>/dev/null | head -50",
timeout=30,
)
@@ -2740,7 +2772,7 @@ class ShellFileOperations(FileOperations):
if re.search(r"[.\[\](){}?*+^$\\|]", pattern):
fixed = self._exec(
f"rg -F --count-matches{glob_expr} "
f"{self._escape_shell_arg(pattern)} {self._escape_shell_arg(path)} "
f"{self._escape_shell_arg(pattern)} {self._escape_native_tool_arg(path)} "
f"2>/dev/null | head -50",
timeout=30,
)
@@ -2862,7 +2894,7 @@ class ShellFileOperations(FileOperations):
# Try mtime-sorted first (rg 13+); fall back to unsorted if not supported.
cmd_sorted = (
f"rg --files --sortr=modified -g {self._escape_shell_arg(glob_pattern)} "
f"{self._escape_shell_arg(path)} 2>/dev/null "
f"{self._escape_native_tool_arg(path)} 2>/dev/null "
f"| head -n {fetch_limit}"
)
result = self._exec(cmd_sorted, timeout=60)
@@ -2873,7 +2905,7 @@ class ShellFileOperations(FileOperations):
# --sortr may have failed on older rg; retry without it.
cmd_plain = (
f"rg --files -g {self._escape_shell_arg(glob_pattern)} "
f"{self._escape_shell_arg(path)} 2>/dev/null "
f"{self._escape_native_tool_arg(path)} 2>/dev/null "
f"| head -n {fetch_limit}"
)
result = self._exec(cmd_plain, timeout=60)
@@ -2957,7 +2989,10 @@ class ShellFileOperations(FileOperations):
# Add pattern and path
cmd_parts.append(self._escape_shell_arg(pattern))
cmd_parts.append(self._escape_shell_arg(path))
# rg is a native Windows binary when installed via winget/cargo/choco:
# it needs the C:/... path form, not the MSYS /c/... form (which
# nothing converts back — Hermes sets MSYS_NO_PATHCONV for its bash).
cmd_parts.append(self._escape_native_tool_arg(path))
# Fetch extra rows so we can report the true total before slicing.
# For context mode, rg emits separator lines ("--") between groups,
+87 -15
View File
@@ -267,9 +267,11 @@ def _detect_escape_drift(content: str, matches: List[Tuple[int, int]],
Returns an error string if drift is detected, None otherwise.
"""
# Cheap pre-check: bail out unless new_string actually contains a
# suspect escape sequence. This keeps the guard free for all the
# common, correct cases.
if "\\'" not in new_string and '\\"' not in new_string:
# suspect escape sequence or any backslash at all (the doubling guard
# below only matters when backslashes are present). This keeps the
# guard free for all the common, correct cases.
has_quote_suspects = "\\'" in new_string or '\\"' in new_string
if not has_quote_suspects and "\\" not in old_string:
return None
# Aggregate matched regions of the file — that's what new_string will
@@ -278,21 +280,91 @@ def _detect_escape_drift(content: str, matches: List[Tuple[int, int]],
# escaped strings); accept the patch.
matched_regions = "".join(content[start:end] for start, end in matches)
for suspect in ("\\'", '\\"'):
if suspect in new_string and suspect in old_string and suspect not in matched_regions:
plain = suspect[1] # "'" or '"'
return (
f"Escape-drift detected: old_string and new_string contain "
f"the literal sequence {suspect!r} but the matched region of "
f"the file does not. This is almost always a tool-call "
f"serialization artifact where an apostrophe or quote got "
f"prefixed with a spurious backslash. Re-read the file with "
f"read_file and pass old_string/new_string without "
f"backslash-escaping {plain!r} characters."
)
if has_quote_suspects:
for suspect in ("\\'", '\\"'):
if suspect in new_string and suspect in old_string and suspect not in matched_regions:
plain = suspect[1] # "'" or '"'
return (
f"Escape-drift detected: old_string and new_string contain "
f"the literal sequence {suspect!r} but the matched region of "
f"the file does not. This is almost always a tool-call "
f"serialization artifact where an apostrophe or quote got "
f"prefixed with a spurious backslash. Re-read the file with "
f"read_file and pass old_string/new_string without "
f"backslash-escaping {plain!r} characters."
)
# Backslash-run doubling: the model sent old_string with 2x the
# backslashes the file actually has (JSON string double-escaping —
# source text `\\` arrives as `\\\\`). A similarity strategy can match
# the region anyway, and writing new_string verbatim then doubles every
# backslash run in the file (`C:\\Users` becomes `C:\\\\Users`).
# Detect the halving relationship between the shared (unchanged) text
# of old_string and the matched region and block with guidance rather
# than silently corrupting escape sequences.
drift = _detect_backslash_doubling(matched_regions, old_string, new_string)
if drift:
return drift
return None
def _backslash_runs(s: str) -> List[int]:
"""Return the lengths of maximal backslash runs in ``s``, in order."""
runs: List[int] = []
n = 0
for ch in s:
if ch == "\\":
n += 1
elif n:
runs.append(n)
n = 0
if n:
runs.append(n)
return runs
def _detect_backslash_doubling(matched_regions: str, old_string: str,
new_string: str) -> Optional[str]:
"""Detect JSON double-escaped backslashes in old_string/new_string.
Fires when every backslash run in old_string is exactly twice the
length of the corresponding run in the matched file region (with the
same number of runs, and at least one run of length >= 2 so a single
doubled backslash in prose can't trigger it). That pattern means the
tool-call arguments went through an extra JSON-escaping pass; writing
new_string verbatim would double every backslash in the file.
"""
old_runs = _backslash_runs(old_string)
file_runs = _backslash_runs(matched_regions)
if not old_runs or not file_runs or len(old_runs) != len(file_runs):
return None
if old_runs == file_runs:
return None
# Every old run must be exactly double its file counterpart, and the
# doubling must be non-trivial (>= 2 backslashes in the file) for at
# least one run — a lone `\` vs `\\` is too weak a signal on its own
# unless it is consistent across 2+ runs.
if any(o != f * 2 for o, f in zip(old_runs, file_runs)):
return None
if not (any(f >= 2 for f in file_runs) or len(file_runs) >= 2):
return None
# new_string must exhibit the same doubling (the model copy-pasted the
# doubled form); if it already matches the file's counts, writing it is
# harmless and we let the edit through.
new_runs = _backslash_runs(new_string)
if new_runs == file_runs:
return None
return (
"Escape-drift detected: every backslash run in old_string is exactly "
"twice as long as in the matched region of the file (e.g. the file "
"has `\\\\` where old_string has `\\\\\\\\`). The tool-call arguments "
"were JSON-escaped one extra time; applying new_string verbatim would "
"double every backslash in the file. Re-read the file with read_file "
"and resend old_string/new_string with the backslash counts exactly "
"as they appear in the file."
)
def _leading_whitespace(line: str) -> str:
"""Return the leading whitespace prefix of a line (spaces/tabs)."""
i = 0