4f5ddd5473
Follow-up to the salvaged #95160. The test wrote the NFC spelling of café.txt and read the NFD spelling. APFS resolves NFD lookups to the NFC file directly, so on macOS the compound probe simply found the file, the retry path never ran and r.hint was None -> TypeError (Linux CI passed because ext4 does not alias normalisation forms). Use a curly-vs-ASCII apostrophe instead: still visually identical, still mapped by _unicode_variant_match, and never aliased by any filesystem, so the retry path is exercised everywhere.
475 lines
19 KiB
Python
475 lines
19 KiB
Python
"""``read_file`` / ``write_file`` cost one shell round-trip, not four.
|
||
|
||
Real ``LocalEnvironment`` against ``tmp_path`` (no mocks), with a spy on
|
||
``env.execute`` counting round-trips. The cases below are exactly the ones
|
||
that used to need their own probe (existence, size, binary sample, page,
|
||
line count, trailing newline), so each proves the compound reply carries
|
||
that answer.
|
||
"""
|
||
|
||
import logging
|
||
import os
|
||
import sys
|
||
import threading
|
||
from unittest.mock import patch
|
||
|
||
import pytest
|
||
|
||
from tools.environments.local import LocalEnvironment
|
||
from tools.file_operations import ExecuteResult, ShellFileOperations
|
||
|
||
pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX shell probes")
|
||
|
||
READ_PROBE_MARK = "__HERMES_RF_"
|
||
|
||
|
||
@pytest.fixture(scope="module")
|
||
def _local_env(tmp_path_factory):
|
||
"""One real LocalEnvironment per module; constructing one costs ~0.8 s."""
|
||
return LocalEnvironment(cwd=str(tmp_path_factory.mktemp("file-ops")))
|
||
|
||
|
||
@pytest.fixture
|
||
def _ops(_local_env, tmp_path):
|
||
"""(ops, calls): file ops over the real local shell, every execute recorded."""
|
||
env = _local_env
|
||
env.cwd = str(tmp_path)
|
||
calls = []
|
||
real_execute = type(env).execute.__get__(env, type(env))
|
||
|
||
def spy(command, *args, **kwargs):
|
||
calls.append(command)
|
||
return real_execute(command, *args, **kwargs)
|
||
|
||
env.execute = spy
|
||
try:
|
||
yield ShellFileOperations(env, cwd=str(tmp_path)), calls
|
||
finally:
|
||
env.__dict__.pop("execute", None)
|
||
|
||
|
||
@pytest.fixture
|
||
def shell(_ops, monkeypatch):
|
||
"""Pin the shell path even where a native fast path exists."""
|
||
monkeypatch.setenv("HERMES_NATIVE_FILE_READ", "0")
|
||
return _ops
|
||
|
||
|
||
@pytest.fixture
|
||
def native(_ops, monkeypatch):
|
||
"""Same wiring with the native fast path on."""
|
||
monkeypatch.delenv("HERMES_NATIVE_FILE_READ", raising=False)
|
||
return _ops
|
||
|
||
|
||
def _write(tmp_path, name, data: bytes):
|
||
p = tmp_path / name
|
||
p.write_bytes(data)
|
||
return str(p)
|
||
|
||
|
||
class TestReadFileOneRoundTrip:
|
||
def test_text_read_is_one_round_trip(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
p = _write(tmp_path, "a.txt", b"one\ntwo\nthree\n")
|
||
r = ops.read_file(p)
|
||
assert len(calls) == 1 and READ_PROBE_MARK in calls[0]
|
||
assert r.error is None
|
||
# ``_add_line_numbers`` numbers the empty tail after the final
|
||
# newline: long-standing behaviour, preserved byte for byte.
|
||
assert r.content == "1|one\n2|two\n3|three\n4|"
|
||
assert (r.total_lines, r.file_size, r.truncated) == (3, 14, False)
|
||
|
||
def test_no_trailing_newline_needs_no_extra_probe(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
p = _write(tmp_path, "b.txt", b"a\nb")
|
||
r = ops.read_file(p)
|
||
assert len(calls) == 1
|
||
# ``cut`` newline-terminates the last line; the artifact is stripped
|
||
# from the same reply that used to need a fifth ``tail -c 1`` call.
|
||
assert r.content == "1|a\n2|b"
|
||
assert r.total_lines == 1 # wc -l semantics, unchanged
|
||
|
||
def test_pagination_window_and_hint(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
p = _write(tmp_path, "c.txt", b"".join(b"l%d\n" % i for i in range(1, 11)))
|
||
r = ops.read_file(p, offset=3, limit=2)
|
||
assert len(calls) == 1
|
||
assert r.content == "3|l3\n4|l4\n5|"
|
||
assert r.truncated is True and r.total_lines == 10
|
||
assert "offset=5" in r.hint
|
||
|
||
def test_offset_past_eof_note(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
p = _write(tmp_path, "c.txt", b"".join(b"l%d\n" % i for i in range(1, 6)))
|
||
r = ops.read_file(p, offset=50)
|
||
assert len(calls) == 1
|
||
assert r.content == "" and r.error is None
|
||
assert "beyond the end" in r.hint and "5" in r.hint
|
||
|
||
def test_empty_file(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
r = ops.read_file(_write(tmp_path, "e.txt", b""))
|
||
assert len(calls) == 1
|
||
assert r.error is None and r.content == "" and r.total_lines == 0
|
||
assert "empty" in r.hint
|
||
|
||
def test_bom_stripped_on_first_page(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
r = ops.read_file(_write(tmp_path, "f.txt", "hello\n".encode("utf-8")))
|
||
assert len(calls) == 1
|
||
assert r.content == "1|hello\n2|"
|
||
|
||
def test_crlf_bytes_survive(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
r = ops.read_file(_write(tmp_path, "g.txt", b"x\r\ny\r\n"))
|
||
assert r.content == "1|x\r\n2|y\r\n3|"
|
||
|
||
def test_long_line_clamped_and_marked(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
r = ops.read_file(_write(tmp_path, "L.txt", b"a" * 9000 + b"\nshort\n"))
|
||
assert len(calls) == 1
|
||
first, second, tail = r.content.split("\n")
|
||
assert first.endswith("... [truncated]") and len(first) < 9000
|
||
assert second == "2|short" and tail == "3|"
|
||
|
||
def test_relative_path_resolves_against_env_cwd(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
_write(tmp_path, "rel.txt", b"here\n")
|
||
r = ops.read_file("rel.txt")
|
||
assert r.error is None and r.content == "1|here\n2|"
|
||
|
||
def test_sentinel_lookalike_in_content_reads_intact(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
lookalike = "__HERMES_RF_" + "ab" * 16 + "__"
|
||
p = _write(tmp_path, "s.txt", f"x\n{lookalike}\ny\n".encode("utf-8"))
|
||
r = ops.read_file(p)
|
||
assert r.error is None and r.total_lines == 3
|
||
assert r.content == f"1|x\n2|{lookalike}\n3|y\n4|"
|
||
|
||
|
||
class TestReadFileNonTextPaths:
|
||
def test_missing_file_probes_once_then_suggests(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
_write(tmp_path, "notes.txt", b"x\n")
|
||
r = ops.read_file(str(tmp_path / "note.txt"))
|
||
assert READ_PROBE_MARK in calls[0]
|
||
assert r.error and "File not found" in r.error
|
||
assert any(s.endswith("notes.txt") for s in r.similar_files)
|
||
|
||
def test_unicode_variant_retry_still_works(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
# A curly apostrophe vs the ASCII one: visually identical in a
|
||
# terminal, and — unlike NFC/NFD — never aliased by the filesystem
|
||
# (APFS resolves NFD lookups to NFC files directly, which would skip
|
||
# the retry path this test exists to exercise).
|
||
on_disk = "it\u2019s.txt"
|
||
typed = "it's.txt"
|
||
assert on_disk != typed
|
||
_write(tmp_path, on_disk, b"accent\n")
|
||
r = ops.read_file(str(tmp_path / typed))
|
||
assert r.error is None and r.content == "1|accent\n2|"
|
||
assert r.hint is not None and "unicode-equivalent" in r.hint
|
||
|
||
def test_directory_is_not_regular(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
r = ops.read_file(str(tmp_path))
|
||
assert len(calls) == 1
|
||
assert r.error and "not a regular file" in r.error
|
||
|
||
def test_binary_sample_detected_in_same_reply(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
p = _write(tmp_path, "blob", b"\x00\x01\x02" + b"\x00" * 50)
|
||
r = ops.read_file(p)
|
||
assert READ_PROBE_MARK in calls[0]
|
||
assert r.is_binary is True and r.error
|
||
# Only the UTF-16 rescue may add round-trips, never a second sample.
|
||
assert not any("head -c 1000" in c for c in calls[1:])
|
||
|
||
def test_image_extension_stops_at_size_probe(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
r = ops.read_file(_write(tmp_path, "p.png", b"\x89PNG\r\n"))
|
||
assert len(calls) == 1 and READ_PROBE_MARK not in calls[0]
|
||
assert r.is_image is True and r.file_size == 6
|
||
|
||
@pytest.mark.linux_only
|
||
def test_fifo_returns_not_regular_without_blocking(self, shell, tmp_path):
|
||
if not hasattr(os, "mkfifo"):
|
||
pytest.skip("no mkfifo")
|
||
ops, calls = shell
|
||
fifo = tmp_path / "pipe"
|
||
os.mkfifo(fifo)
|
||
box = {}
|
||
|
||
def run():
|
||
box["r"] = ops.read_file(str(fifo))
|
||
|
||
t = threading.Thread(target=run, daemon=True)
|
||
t.start()
|
||
t.join(20)
|
||
assert not t.is_alive(), "read_file blocked on a writer-less FIFO"
|
||
assert "not a regular file" in box["r"].error
|
||
assert len(calls) == 1
|
||
|
||
|
||
class TestWriteFileRoundTrips:
|
||
"""write_file: one probe, one atomic write, one hash check (three calls)."""
|
||
|
||
@staticmethod
|
||
def _execs(calls):
|
||
return [c for c in calls]
|
||
|
||
def test_new_text_file_is_three_round_trips(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
p = str(tmp_path / "new.txt")
|
||
r = ops.write_file(p, "line one\nline two\n")
|
||
assert r.error is None and r.verified is True
|
||
assert len(calls) == 3
|
||
assert "__HERMES_WF_" in calls[0] # probe
|
||
assert "mv -f" in calls[1] # atomic write
|
||
assert calls[2].startswith("sha256sum ") # verify
|
||
assert (tmp_path / "new.txt").read_bytes() == b"line one\nline two\n"
|
||
|
||
def test_crlf_file_keeps_crlf_from_the_probe(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
p = tmp_path / "crlf.txt"
|
||
p.write_bytes(b"a\r\nb\r\n")
|
||
r = ops.write_file(str(p), "x\ny\n")
|
||
assert r.error is None and len(calls) == 3
|
||
assert p.read_bytes() == b"x\r\ny\r\n"
|
||
|
||
def test_bom_is_read_from_disk_and_preserved(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
p = tmp_path / "bom.txt"
|
||
p.write_bytes("old\n".encode("utf-8"))
|
||
r = ops.write_file(str(p), "new\n")
|
||
assert r.error is None and len(calls) == 3
|
||
assert p.read_bytes() == "new\n".encode("utf-8")
|
||
|
||
def test_pre_content_read_rides_the_same_probe(self, shell, tmp_path):
|
||
"""A lintable extension wants the old text (lint delta); it comes
|
||
back in the probe reply instead of a separate ``cat``."""
|
||
ops, calls = shell
|
||
p = tmp_path / "code.py"
|
||
p.write_bytes(b"x = 1\r\ny = 2\r\n")
|
||
r = ops.write_file(str(p), "x = 1\ny = 3\n")
|
||
assert r.error is None
|
||
probes = [c for c in calls if "__HERMES_WF_" in c]
|
||
assert len(probes) == 1 and "cat " in probes[0]
|
||
assert not any(c.startswith("cat ") for c in calls)
|
||
assert p.read_bytes() == b"x = 1\r\ny = 3\r\n"
|
||
|
||
def test_missing_file_probe_does_not_block_the_write(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
p = tmp_path / "deep" / "er" / "new.md"
|
||
r = ops.write_file(str(p), "hi\n")
|
||
assert r.error is None and r.dirs_created is True
|
||
assert len(calls) == 3
|
||
assert p.read_bytes() == b"hi\n"
|
||
|
||
def test_unparseable_probe_reply_falls_back_to_separate_probes(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
p = tmp_path / "crlf.txt"
|
||
p.write_bytes(b"a\r\nb\r\n")
|
||
real_exec = ops._exec
|
||
|
||
def garbled(command, *args, **kwargs):
|
||
if "__HERMES_WF_" in command:
|
||
return ExecuteResult(stdout="[Command timed out after 1s]\n", exit_code=124)
|
||
return real_exec(command, *args, **kwargs)
|
||
|
||
with patch.object(ops, "_exec", side_effect=garbled):
|
||
r = ops.write_file(str(p), "x\ny\n")
|
||
assert r.error is None
|
||
assert p.read_bytes() == b"x\r\ny\r\n"
|
||
|
||
|
||
class TestNativeRead:
|
||
def test_native_read_makes_no_shell_call(self, native, tmp_path):
|
||
ops, calls = native
|
||
r = ops.read_file(_write(tmp_path, "a.txt", b"one\ntwo\n"))
|
||
assert calls == []
|
||
assert r.error is None and r.content == "1|one\n2|two\n3|"
|
||
assert (r.total_lines, r.file_size) == (2, 8)
|
||
|
||
def test_kill_switch_routes_to_the_shell(self, native, tmp_path, monkeypatch):
|
||
ops, calls = native
|
||
p = _write(tmp_path, "a.txt", b"one\n")
|
||
monkeypatch.setenv("HERMES_NATIVE_FILE_READ", "0")
|
||
ops.read_file(p)
|
||
assert len(calls) == 1 and READ_PROBE_MARK in calls[0]
|
||
|
||
def test_non_local_environment_keeps_the_shell_path(self):
|
||
from unittest.mock import MagicMock
|
||
|
||
env = MagicMock()
|
||
env.cwd = "/tmp"
|
||
assert ShellFileOperations(env)._native_read_enabled() is False
|
||
|
||
def test_tilde_still_expands_through_the_shell(self, native):
|
||
ops, calls = native
|
||
r = ops.read_file("~/hermes-no-such-file-7f3a.txt")
|
||
assert calls[0] == "echo $HOME"
|
||
assert r.error and "File not found" in r.error
|
||
|
||
def test_injection_lookalike_path_is_never_expanded(self, native, tmp_path):
|
||
ops, calls = native
|
||
marker = tmp_path / "pwned"
|
||
r = ops.read_file(f"~; echo PWNED > {marker}")
|
||
assert r.error and not marker.exists()
|
||
# The text reaches the shell only single-quoted, inside the missing-
|
||
# file recovery's directory listing; the tilde probe is a fixed
|
||
# ``echo $HOME`` that never embeds the path. Nothing else runs.
|
||
for c in calls:
|
||
assert c == "echo $HOME" or c.startswith("ls -1 '~; echo PWNED"), c
|
||
|
||
@pytest.mark.linux_only
|
||
def test_fifo_refused_without_a_shell_and_without_blocking(self, native, tmp_path):
|
||
if not hasattr(os, "mkfifo"):
|
||
pytest.skip("no mkfifo")
|
||
ops, calls = native
|
||
fifo = tmp_path / "pipe"
|
||
os.mkfifo(fifo)
|
||
box = {}
|
||
|
||
def run():
|
||
box["r"] = ops.read_file(str(fifo))
|
||
|
||
t = threading.Thread(target=run, daemon=True)
|
||
t.start()
|
||
t.join(20)
|
||
assert not t.is_alive(), "native read_file blocked on a writer-less FIFO"
|
||
assert "not a regular file" in box["r"].error
|
||
assert calls == []
|
||
|
||
|
||
# The native reader scans 1 MiB chunks and clamps each page line to
|
||
# ``4 * get_max_line_length() + 1`` bytes (8001 by default), exactly as
|
||
# ``sed | cut -b1-N`` does. These shapes put a newline, a line, the clamp
|
||
# point, a CRLF pair and EOF precisely on those chunk boundaries.
|
||
_CHUNK = 1 << 20
|
||
_CLAMP = 8001
|
||
|
||
PARITY_CASES = [
|
||
("plain", b"one\ntwo\nthree\n", {}),
|
||
("no_trailing_newline", b"a\nb", {}),
|
||
("blank_tail", b"a\n\n", {}),
|
||
("crlf", b"x\r\ny\r\n", {}),
|
||
("lone_cr", b"a\rb\n", {}),
|
||
("bom", "hello\n".encode("utf-8"), {}),
|
||
("empty", b"", {}),
|
||
("single_no_newline", b"solo", {}),
|
||
("only_newline", b"\n", {}),
|
||
("unicode", "héllo wörld\n汉字\n".encode("utf-8"), {}),
|
||
("long_line", b"a" * 9000 + b"\nshort\n", {}),
|
||
("multibyte_long_line", ("汉" * 4000 + "\nx\n").encode("utf-8"), {}),
|
||
("multi_chunk_line", b"b" * 3_000_000 + b"\nz\n", {}),
|
||
("newline_last_byte_of_chunk", b"a" * (_CHUNK - 1) + b"\nsecond\n", {}),
|
||
("newline_first_byte_of_next_chunk", b"a" * _CHUNK + b"\nsecond\n", {}),
|
||
("line_spans_three_boundaries", b"p\n" + b"b" * (3 * _CHUNK + 5) + b"\nz\n", {}),
|
||
(
|
||
"clamp_fills_on_boundary",
|
||
b"x" * (_CHUNK - _CLAMP - 1) + b"\n" + b"c" * 20000 + b"\ntail\n",
|
||
{"offset": 2, "limit": 3},
|
||
),
|
||
(
|
||
"clamp_fills_after_boundary",
|
||
b"x" * (_CHUNK - _CLAMP) + b"\n" + b"c" * 20000 + b"\ntail\n",
|
||
{"offset": 2, "limit": 3},
|
||
),
|
||
("eof_midline_on_boundary", b"l1\n" + b"d" * (2 * _CHUNK - 3), {}),
|
||
("blank_lines_on_boundary", b"e" * (_CHUNK - 2) + b"\n\n\n" + b"f\n", {}),
|
||
("crlf_split_on_boundary", b"r" * (_CHUNK - 1) + b"\r\n" + b"s\r\n", {}),
|
||
(
|
||
"page_starts_in_second_chunk",
|
||
b"q\n" * (_CHUNK // 2 + 3) + b"target1\ntarget2\n",
|
||
{"offset": _CHUNK // 2 + 3, "limit": 4},
|
||
),
|
||
("window", b"".join(b"l%d\n" % i for i in range(1, 11)), {"offset": 3, "limit": 2}),
|
||
("window_reaches_eof", b"".join(b"l%d\n" % i for i in range(1, 11)), {"offset": 9, "limit": 5}),
|
||
("past_eof", b"".join(b"l%d\n" % i for i in range(1, 6)), {"offset": 50}),
|
||
("nul_binary", b"\x00\x01\x02" * 20, {}),
|
||
("latin1_tail", b"caf\xe9\n", {}),
|
||
("sentinel_lookalike", b"x\n__HERMES_RF_" + b"ab" * 16 + b"__\ny\n", {}),
|
||
]
|
||
|
||
|
||
class TestNativeReadParity:
|
||
"""The native path must be indistinguishable from the shell path."""
|
||
|
||
@pytest.mark.parametrize("name,data,kwargs", PARITY_CASES, ids=[c[0] for c in PARITY_CASES])
|
||
def test_shell_and_native_agree(self, native, tmp_path, monkeypatch, name, data, kwargs):
|
||
ops, calls = native
|
||
p = _write(tmp_path, f"{name}.txt", data)
|
||
monkeypatch.setenv("HERMES_NATIVE_FILE_READ", "0")
|
||
via_shell = ops.read_file(p, **kwargs).to_dict()
|
||
assert calls and READ_PROBE_MARK in calls[0]
|
||
calls.clear()
|
||
monkeypatch.delenv("HERMES_NATIVE_FILE_READ")
|
||
via_native = ops.read_file(p, **kwargs).to_dict()
|
||
assert via_native == via_shell
|
||
# The native path touches the shell only for the UTF-16 rescue of binaries.
|
||
assert not any(READ_PROBE_MARK in c for c in calls)
|
||
|
||
def test_special_paths_agree(self, native, tmp_path, monkeypatch):
|
||
ops, calls = native
|
||
_write(tmp_path, "real.txt", b"target\n")
|
||
os.symlink(tmp_path / "real.txt", tmp_path / "link.txt")
|
||
os.symlink(tmp_path / "gone", tmp_path / "dangling.txt")
|
||
(tmp_path / "sub").mkdir()
|
||
_write(tmp_path, "pic.png", b"\x89PNG\r\n")
|
||
_write(tmp_path, "notes.txt", b"n\n")
|
||
for p in (
|
||
str(tmp_path / "link.txt"),
|
||
str(tmp_path / "dangling.txt"),
|
||
str(tmp_path / "sub"),
|
||
str(tmp_path / "pic.png"),
|
||
str(tmp_path / "note.txt"), # missing → similar-file suggestions
|
||
"real.txt", # relative to env.cwd
|
||
):
|
||
monkeypatch.setenv("HERMES_NATIVE_FILE_READ", "0")
|
||
via_shell = ops.read_file(p).to_dict()
|
||
monkeypatch.delenv("HERMES_NATIVE_FILE_READ")
|
||
calls.clear()
|
||
via_native = ops.read_file(p).to_dict()
|
||
assert via_native == via_shell, p
|
||
assert not any(READ_PROBE_MARK in c for c in calls), p
|
||
|
||
|
||
class TestCompoundFallback:
|
||
def test_unparseable_reply_falls_back_to_sequential_probes(self, shell, tmp_path):
|
||
ops, calls = shell
|
||
p = _write(tmp_path, "a.txt", b"one\ntwo\n")
|
||
real_exec = ops._exec
|
||
|
||
def garbled(command, *args, **kwargs):
|
||
if READ_PROBE_MARK in command:
|
||
return ExecuteResult(stdout="[Command timed out after 1s]\n", exit_code=124)
|
||
return real_exec(command, *args, **kwargs)
|
||
|
||
with patch.object(ops, "_exec", side_effect=garbled):
|
||
r = ops.read_file(p)
|
||
assert r.error is None and r.content == "1|one\n2|two\n3|"
|
||
assert r.total_lines == 2
|
||
|
||
def test_fallback_is_logged_at_debug(self, shell, tmp_path, caplog):
|
||
"""A backend that keeps falling back shows up in debug logs."""
|
||
ops, calls = shell
|
||
p = _write(tmp_path, "a.txt", b"one\n")
|
||
real_exec = ops._exec
|
||
|
||
def garbled(command, *args, **kwargs):
|
||
if READ_PROBE_MARK in command:
|
||
return ExecuteResult(stdout="garbage\n", exit_code=0)
|
||
return real_exec(command, *args, **kwargs)
|
||
|
||
with caplog.at_level(logging.DEBUG, logger="tools.file_operations"), \
|
||
patch.object(ops, "_exec", side_effect=garbled):
|
||
r = ops.read_file(p)
|
||
assert r.error is None and r.content == "1|one\n2|"
|
||
assert any(
|
||
"falling back to sequential probes" in rec.getMessage()
|
||
and str(p) in rec.getMessage()
|
||
for rec in caplog.records
|
||
)
|