fix(gateway): accept CJK full-width punctuation as MEDIA path terminators

MEDIA_TAG_CLEANUP_RE (and MEDIA_EXTENSIONLESS_TAG_RE) only recognized
ASCII terminators after a MEDIA:<path> tag. Chinese-language agent
output naturally writes MEDIA:D:\...\zhibao.pdf(782.6 KB)or ...pdf:内容 —
the full-width punctuation failed the trailing lookahead and the
attachment was silently dropped (cron even reported 'delivered') (#88038).

Both lookaheads now accept a CJK full-width terminator set (()〈〉《》:,。;
!?、curly quotes【】) alongside the ASCII set. The #68773 adjacent-tag
splitting guard is covered by a regression test.
This commit is contained in:
liuhao1024
2026-08-17 09:06:09 +08:00
committed by Teknium
parent a40324bb32
commit 8dc427949f
2 changed files with 70 additions and 2 deletions
+8 -2
View File
@@ -1939,11 +1939,17 @@ _MEDIA_EXT_ALTERNATION = "|".join(
# guard keeps multi-part extensions intact — for ``archive.tar.gz`` the
# ``.`` after ``tar`` is followed by ``g``, so the match must extend to
# ``.gz`` instead of stopping early at ``.tar``.
# CJK full-width punctuation accepted as MEDIA path terminators, mirroring the
# ASCII set in the looka below. Chinese-language agent output naturally writes
# ``MEDIA:D:\path\早报.pdf(782.6 KB)`` or ``MEDIA:...pdf:内容`` — without
# these, the lookahead fails and the attachment is silently dropped (#88038).
_MEDIA_CJK_TERMINATORS = "()〈〉《》:,。;!?、\u201c\u201d\u2018\u2019【】"
MEDIA_TAG_CLEANUP_RE = re.compile(
r'''[`"'*_]{0,3}MEDIA:\s*'''
r'''(?P<path>`[^`\n]+?`|"[^"\n]+?"|'[^'\n]+?'|'''
r'''(?:~/|/|[A-Za-z]:[/\\])\S+?(?:[^\S\n]+\S+?)*?\.(?:''' + _MEDIA_EXT_ALTERNATION + r'''))'''
r'''(?=[\s`"'*_,;:)\]}\[]|MEDIA:|\.(?:\s|$)|$)[`"'*_]{0,3}\.?''',
r'''(?=[\s`"'*_,;:)\]}\[''' + _MEDIA_CJK_TERMINATORS + r''']|MEDIA:|\.(?:\s|$)|$)[`"'*_]{0,3}\.?''',
re.IGNORECASE,
)
@@ -1979,7 +1985,7 @@ MEDIA_EXTENSIONLESS_TAG_RE = re.compile(
r'''[`"'*_]{0,3}MEDIA:\s*'''
r'''(?P<path>`[^`\n]+`|"[^"\n]+"|'[^'\n]+'|'''
r'''(?:~/|/|[A-Za-z]:[/\\])[^\s\n`"']+?)'''
r'''(?=[`"'\s,;:)\]}]|MEDIA:|$)'''
r'''(?=[`"'\s,;:)\]}''' + _MEDIA_CJK_TERMINATORS + r''']|MEDIA:|$)'''
r'''[`"'*_]{0,3}\s*''',
re.IGNORECASE,
)
+62
View File
@@ -34,3 +34,65 @@ class TestMediaTagCleanup:
assert "/tmp/chart.png" not in stripped
class TestMediaTagCjkTerminators:
"""#88038: CJK full-width punctuation after a MEDIA path is a valid
terminator, exactly like its ASCII counterparts. Chinese-language agent
output naturally writes ``MEDIA:D:\\...\\zhibao.pdf(782.6 KB)`` — the
ASCII-only lookahead silently dropped the attachment."""
def test_full_width_suffixes_terminate_the_path(self):
from gateway.platforms.base import MEDIA_TAG_CLEANUP_RE
cases = [
"MEDIA:D:/workspace/out/report.pdf(782.6 KB)",
"MEDIA:D:/workspace/out/report.pdf:内容",
"MEDIA:D:/workspace/out/report.pdf。",
"MEDIA:D:/workspace/out/report.pdf,下一条",
"MEDIA:D:/workspace/out/report.pdf;",
"MEDIA:D:/workspace/out/report.pdf!",
"MEDIA:D:/workspace/out/report.pdf?",
"MEDIA:D:/workspace/out/report.pdf、",
"MEDIA:D:/workspace/out/report.pdf”",
"MEDIA:D:/workspace/out/report.pdf’",
]
for text in cases:
m = MEDIA_TAG_CLEANUP_RE.search(text)
assert m is not None, text
assert m.group("path").endswith(".pdf"), (text, m.group("path"))
def test_real_chinese_cron_delivery_line(self):
from gateway.platforms.base import MEDIA_TAG_CLEANUP_RE
text = (
"## 交付物\n\n- **PDF 早报**:"
"MEDIA:D:/workspace/zaobao/output/早报_2026-08-16.pdf(782.6 KB)"
)
m = MEDIA_TAG_CLEANUP_RE.search(text)
assert m is not None
assert m.group("path") == "D:/workspace/zaobao/output/早报_2026-08-16.pdf"
def test_ascii_terminators_still_work(self):
from gateway.platforms.base import MEDIA_TAG_CLEANUP_RE
for text in (
"MEDIA:/tmp/report.pdf",
"MEDIA:/tmp/report.pdf 782 KB",
"MEDIA:/tmp/report.pdf, next",
):
m = MEDIA_TAG_CLEANUP_RE.search(text)
assert m is not None and m.group("path").endswith(".pdf"), text
def test_adjacent_tags_still_split(self):
"""#68773 guard: the widened class must not let tags glue together."""
from gateway.platforms.base import MEDIA_TAG_CLEANUP_RE
found = MEDIA_TAG_CLEANUP_RE.findall("MEDIA:/a.pngMEDIA:/b.png")
assert list(found) == ["/a.png", "/b.png"]
def test_extensionless_variant_accepts_full_width_colon(self):
from gateway.platforms.base import MEDIA_EXTENSIONLESS_TAG_RE
m = MEDIA_EXTENSIONLESS_TAG_RE.search("MEDIA:/tmp/Caddyfile:内容")
assert m is not None and m.group("path") == "/tmp/Caddyfile"