From 58dca1be5148c5cd2bac6341d5eb7b4171745073 Mon Sep 17 00:00:00 2001 From: Teknium <127238744+teknium1@users.noreply.github.com> Date: Thu, 3 Sep 2026 09:32:38 -0700 Subject: [PATCH] review-fix(gateway): pin split_markdown_atoms empty-fence atoms to main main's 'and len(current_lines) > 1' fence-close guard is dead (the opening fence is always already in current_lines), so dropping it is byte-identical for every input (A/B: 11 empty/nested fence shapes). Add a parity test that asserts main's atoms for empty ``` blocks so the boundary stays pinned. --- tests/gateway/test_fence_chunker.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/gateway/test_fence_chunker.py b/tests/gateway/test_fence_chunker.py index 5bd5ba41d1..9c39b01aa9 100644 --- a/tests/gateway/test_fence_chunker.py +++ b/tests/gateway/test_fence_chunker.py @@ -174,3 +174,15 @@ def test_mixed_emoji_newline_text_utf16_limit(): ) assert all(utf16_len(c) <= 500 for c in chunks) assert all(c for c in chunks) + + +# ── split_markdown_atoms: empty ``` ``` blocks (main parity) ───────────────── + + +def test_empty_fence_block_atoms_match_main(): + """An empty fenced block (``` immediately followed by ```) is its own + atom and the following text starts a fresh one — main's chunk boundaries.""" + assert split_markdown_atoms("```\n```\ntext") == ["```\n```", "text"] + assert split_markdown_atoms("```\n```\ncode\n```\ntail") == ["```\n```", "code", "```\ntail"] + assert split_markdown_atoms("```py\n```\n\n```\n```") == ["```py\n```", "```\n```"] + assert split_markdown_atoms("a\n```\ncode\n```\nb") == ["a", "```\ncode\n```", "b"]