fix(process): keep the sandbox log poller from splitting a UTF-8 character across polls

Follow-up to #92164: the delta window now ends on a character boundary
(up to 3 trailing continuation bytes are held for the next poll), so
multibyte output no longer decodes to U+FFFD at the seam. Verified on
bash, dash and busybox sh; exhaustive-prefix regression test added.
This commit is contained in:
kshitijk4poor
2026-09-03 02:20:46 +05:30
parent a2b5d4d490
commit 75bcd86687
2 changed files with 48 additions and 0 deletions
+29
View File
@@ -2,6 +2,8 @@
import json
import os
import shlex
import shutil
import signal
import subprocess
import sys
@@ -849,6 +851,33 @@ class TestEnvPollerIncrementalRead:
cmd = ProcessRegistry._log_delta_command("'/tmp/bg.log'", 0)
assert "O=0" in cmd
@pytest.mark.skipif(not shutil.which("sh"), reason="needs a POSIX sh")
def test_read_command_holds_back_a_split_utf8_sequence(self, tmp_path):
"""A multibyte character straddling two polls must not be split.
The backend decodes each execute() result on its own, so returning
the first byte of an 'é' in one poll and the rest in the next would
yield replacement characters in the transcript (and break watch
patterns at the seam). Every prefix of a mixed ASCII/2/3/4-byte
string must come back decodable, with at most 3 bytes held back and
nothing held back once the trailing character is complete.
"""
full = "hé😀中a\n€bz🚀".encode()
log = tmp_path / "bg.log"
quoted = shlex.quote(str(log))
for n in range(1, len(full) + 1):
log.write_bytes(full[:n])
out = subprocess.run(
["sh", "-c", ProcessRegistry._log_delta_command(quoted, 0)],
capture_output=True, timeout=30,
).stdout
header, _, delta = out.partition(b"\n")
size, _offset = map(int, header.split())
delta.decode("utf-8") # must not raise
assert delta == full[:size]
complete = full[:n].decode("utf-8", "ignore").encode() == full[:n]
assert (n - size) == 0 if complete else 0 < (n - size) <= 3
def test_first_poll_reads_from_the_start(self, registry):
session = _make_session(sid="proc_delta")
session.exited = False
+19
View File
@@ -1537,12 +1537,31 @@ class ProcessRegistry:
a file that grows while the command runs never sends a byte twice.
A file that shrank was rotated or truncated, so the offset drops back
to 0 and the reader starts over.
The end of the window is pulled back to a UTF-8 character boundary:
the backend decodes each ``execute()`` result on its own, so a
multibyte character straddling two polls would otherwise come back
as replacement characters (and break watch patterns near the seam).
Up to 3 trailing continuation bytes are held for the next poll; the
header reports the trimmed size so the offset stays consistent.
"""
return (
f"O={offset}; "
f"S=$({{ wc -c < {quoted_log_path}; }} 2>/dev/null | tr -dc '0-9'); "
f"S=${{S:-0}}; "
f'if [ "$S" -lt "$O" ]; then O=0; fi; '
# Hold back an INCOMPLETE trailing UTF-8 sequence for the next
# poll. Scan back up to 3 continuation bytes (octal 200-277) to
# the lead byte; if the lead byte's declared length (3xx=2, 34x-35x
# =3, 36x-37x=4) exceeds the bytes present, trim to before it.
# Complete sequences and ASCII tails are left untouched.
f'N=0; P=$S; while [ "$P" -gt "$O" ] && [ "$N" -lt 3 ]; do '
f"B=$(tail -c +$P {quoted_log_path} 2>/dev/null | head -c 1 | od -An -to1 | tr -dc '0-9'); "
f'case "$B" in 2[0-7][0-7]) P=$((P-1)); N=$((N+1));; *) break;; esac; done; '
f'if [ "$N" -gt 0 ] || [ "$P" -eq "$S" ]; then '
f"B=$(tail -c +$P {quoted_log_path} 2>/dev/null | head -c 1 | od -An -to1 | tr -dc '0-9'); "
f'case "$B" in 3[0-3][0-7]) L=2;; 3[4-5][0-7]) L=3;; 3[6-7][0-7]) L=4;; *) L=1;; esac; '
f'if [ "$L" -gt $((N+1)) ]; then S=$((P-1)); fi; fi; '
f'echo "$S $O"; '
f'if [ "$S" -gt "$O" ]; then '
f"tail -c +$((O+1)) {quoted_log_path} 2>/dev/null | head -c $((S-O)); fi"