fix(process): keep the sandbox log poller from splitting a UTF-8 character across polls
Follow-up to #92164: the delta window now ends on a character boundary (up to 3 trailing continuation bytes are held for the next poll), so multibyte output no longer decodes to U+FFFD at the seam. Verified on bash, dash and busybox sh; exhaustive-prefix regression test added.
This commit is contained in:
@@ -2,6 +2,8 @@
|
||||
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -849,6 +851,33 @@ class TestEnvPollerIncrementalRead:
|
||||
cmd = ProcessRegistry._log_delta_command("'/tmp/bg.log'", 0)
|
||||
assert "O=0" in cmd
|
||||
|
||||
@pytest.mark.skipif(not shutil.which("sh"), reason="needs a POSIX sh")
|
||||
def test_read_command_holds_back_a_split_utf8_sequence(self, tmp_path):
|
||||
"""A multibyte character straddling two polls must not be split.
|
||||
|
||||
The backend decodes each execute() result on its own, so returning
|
||||
the first byte of an 'é' in one poll and the rest in the next would
|
||||
yield replacement characters in the transcript (and break watch
|
||||
patterns at the seam). Every prefix of a mixed ASCII/2/3/4-byte
|
||||
string must come back decodable, with at most 3 bytes held back and
|
||||
nothing held back once the trailing character is complete.
|
||||
"""
|
||||
full = "hé😀中a\n€bz🚀".encode()
|
||||
log = tmp_path / "bg.log"
|
||||
quoted = shlex.quote(str(log))
|
||||
for n in range(1, len(full) + 1):
|
||||
log.write_bytes(full[:n])
|
||||
out = subprocess.run(
|
||||
["sh", "-c", ProcessRegistry._log_delta_command(quoted, 0)],
|
||||
capture_output=True, timeout=30,
|
||||
).stdout
|
||||
header, _, delta = out.partition(b"\n")
|
||||
size, _offset = map(int, header.split())
|
||||
delta.decode("utf-8") # must not raise
|
||||
assert delta == full[:size]
|
||||
complete = full[:n].decode("utf-8", "ignore").encode() == full[:n]
|
||||
assert (n - size) == 0 if complete else 0 < (n - size) <= 3
|
||||
|
||||
def test_first_poll_reads_from_the_start(self, registry):
|
||||
session = _make_session(sid="proc_delta")
|
||||
session.exited = False
|
||||
|
||||
@@ -1537,12 +1537,31 @@ class ProcessRegistry:
|
||||
a file that grows while the command runs never sends a byte twice.
|
||||
A file that shrank was rotated or truncated, so the offset drops back
|
||||
to 0 and the reader starts over.
|
||||
|
||||
The end of the window is pulled back to a UTF-8 character boundary:
|
||||
the backend decodes each ``execute()`` result on its own, so a
|
||||
multibyte character straddling two polls would otherwise come back
|
||||
as replacement characters (and break watch patterns near the seam).
|
||||
Up to 3 trailing continuation bytes are held for the next poll; the
|
||||
header reports the trimmed size so the offset stays consistent.
|
||||
"""
|
||||
return (
|
||||
f"O={offset}; "
|
||||
f"S=$({{ wc -c < {quoted_log_path}; }} 2>/dev/null | tr -dc '0-9'); "
|
||||
f"S=${{S:-0}}; "
|
||||
f'if [ "$S" -lt "$O" ]; then O=0; fi; '
|
||||
# Hold back an INCOMPLETE trailing UTF-8 sequence for the next
|
||||
# poll. Scan back up to 3 continuation bytes (octal 200-277) to
|
||||
# the lead byte; if the lead byte's declared length (3xx=2, 34x-35x
|
||||
# =3, 36x-37x=4) exceeds the bytes present, trim to before it.
|
||||
# Complete sequences and ASCII tails are left untouched.
|
||||
f'N=0; P=$S; while [ "$P" -gt "$O" ] && [ "$N" -lt 3 ]; do '
|
||||
f"B=$(tail -c +$P {quoted_log_path} 2>/dev/null | head -c 1 | od -An -to1 | tr -dc '0-9'); "
|
||||
f'case "$B" in 2[0-7][0-7]) P=$((P-1)); N=$((N+1));; *) break;; esac; done; '
|
||||
f'if [ "$N" -gt 0 ] || [ "$P" -eq "$S" ]; then '
|
||||
f"B=$(tail -c +$P {quoted_log_path} 2>/dev/null | head -c 1 | od -An -to1 | tr -dc '0-9'); "
|
||||
f'case "$B" in 3[0-3][0-7]) L=2;; 3[4-5][0-7]) L=3;; 3[6-7][0-7]) L=4;; *) L=1;; esac; '
|
||||
f'if [ "$L" -gt $((N+1)) ]; then S=$((P-1)); fi; fi; '
|
||||
f'echo "$S $O"; '
|
||||
f'if [ "$S" -gt "$O" ]; then '
|
||||
f"tail -c +$((O+1)) {quoted_log_path} 2>/dev/null | head -c $((S-O)); fi"
|
||||
|
||||
Reference in New Issue
Block a user