diff --git a/tests/tools/test_process_registry.py b/tests/tools/test_process_registry.py index 6b4f18ee4e..589d4e2775 100644 --- a/tests/tools/test_process_registry.py +++ b/tests/tools/test_process_registry.py @@ -2,6 +2,8 @@ import json import os +import shlex +import shutil import signal import subprocess import sys @@ -849,6 +851,33 @@ class TestEnvPollerIncrementalRead: cmd = ProcessRegistry._log_delta_command("'/tmp/bg.log'", 0) assert "O=0" in cmd + @pytest.mark.skipif(not shutil.which("sh"), reason="needs a POSIX sh") + def test_read_command_holds_back_a_split_utf8_sequence(self, tmp_path): + """A multibyte character straddling two polls must not be split. + + The backend decodes each execute() result on its own, so returning + the first byte of an 'é' in one poll and the rest in the next would + yield replacement characters in the transcript (and break watch + patterns at the seam). Every prefix of a mixed ASCII/2/3/4-byte + string must come back decodable, with at most 3 bytes held back and + nothing held back once the trailing character is complete. + """ + full = "hé😀中a\n€bz🚀".encode() + log = tmp_path / "bg.log" + quoted = shlex.quote(str(log)) + for n in range(1, len(full) + 1): + log.write_bytes(full[:n]) + out = subprocess.run( + ["sh", "-c", ProcessRegistry._log_delta_command(quoted, 0)], + capture_output=True, timeout=30, + ).stdout + header, _, delta = out.partition(b"\n") + size, _offset = map(int, header.split()) + delta.decode("utf-8") # must not raise + assert delta == full[:size] + complete = full[:n].decode("utf-8", "ignore").encode() == full[:n] + assert (n - size) == 0 if complete else 0 < (n - size) <= 3 + def test_first_poll_reads_from_the_start(self, registry): session = _make_session(sid="proc_delta") session.exited = False diff --git a/tools/process_registry.py b/tools/process_registry.py index 70495b05b6..8937c4cce8 100644 --- a/tools/process_registry.py +++ b/tools/process_registry.py @@ -1537,12 +1537,31 @@ class ProcessRegistry: a file that grows while the command runs never sends a byte twice. A file that shrank was rotated or truncated, so the offset drops back to 0 and the reader starts over. + + The end of the window is pulled back to a UTF-8 character boundary: + the backend decodes each ``execute()`` result on its own, so a + multibyte character straddling two polls would otherwise come back + as replacement characters (and break watch patterns near the seam). + Up to 3 trailing continuation bytes are held for the next poll; the + header reports the trimmed size so the offset stays consistent. """ return ( f"O={offset}; " f"S=$({{ wc -c < {quoted_log_path}; }} 2>/dev/null | tr -dc '0-9'); " f"S=${{S:-0}}; " f'if [ "$S" -lt "$O" ]; then O=0; fi; ' + # Hold back an INCOMPLETE trailing UTF-8 sequence for the next + # poll. Scan back up to 3 continuation bytes (octal 200-277) to + # the lead byte; if the lead byte's declared length (3xx=2, 34x-35x + # =3, 36x-37x=4) exceeds the bytes present, trim to before it. + # Complete sequences and ASCII tails are left untouched. + f'N=0; P=$S; while [ "$P" -gt "$O" ] && [ "$N" -lt 3 ]; do ' + f"B=$(tail -c +$P {quoted_log_path} 2>/dev/null | head -c 1 | od -An -to1 | tr -dc '0-9'); " + f'case "$B" in 2[0-7][0-7]) P=$((P-1)); N=$((N+1));; *) break;; esac; done; ' + f'if [ "$N" -gt 0 ] || [ "$P" -eq "$S" ]; then ' + f"B=$(tail -c +$P {quoted_log_path} 2>/dev/null | head -c 1 | od -An -to1 | tr -dc '0-9'); " + f'case "$B" in 3[0-3][0-7]) L=2;; 3[4-5][0-7]) L=3;; 3[6-7][0-7]) L=4;; *) L=1;; esac; ' + f'if [ "$L" -gt $((N+1)) ]; then S=$((P-1)); fi; fi; ' f'echo "$S $O"; ' f'if [ "$S" -gt "$O" ]; then ' f"tail -c +$((O+1)) {quoted_log_path} 2>/dev/null | head -c $((S-O)); fi"