fix(gateway): accept CJK full-width punctuation as MEDIA path terminators
MEDIA_TAG_CLEANUP_RE (and MEDIA_EXTENSIONLESS_TAG_RE) only recognized ASCII terminators after a MEDIA:<path> tag. Chinese-language agent output naturally writes MEDIA:D:\...\zhibao.pdf(782.6 KB)or ...pdf:内容 — the full-width punctuation failed the trailing lookahead and the attachment was silently dropped (cron even reported 'delivered') (#88038). Both lookaheads now accept a CJK full-width terminator set (()〈〉《》:,。; !?、curly quotes【】) alongside the ASCII set. The #68773 adjacent-tag splitting guard is covered by a regression test.
This commit is contained in:
@@ -1939,11 +1939,17 @@ _MEDIA_EXT_ALTERNATION = "|".join(
|
||||
# guard keeps multi-part extensions intact — for ``archive.tar.gz`` the
|
||||
# ``.`` after ``tar`` is followed by ``g``, so the match must extend to
|
||||
# ``.gz`` instead of stopping early at ``.tar``.
|
||||
# CJK full-width punctuation accepted as MEDIA path terminators, mirroring the
|
||||
# ASCII set in the looka below. Chinese-language agent output naturally writes
|
||||
# ``MEDIA:D:\path\早报.pdf(782.6 KB)`` or ``MEDIA:...pdf:内容`` — without
|
||||
# these, the lookahead fails and the attachment is silently dropped (#88038).
|
||||
_MEDIA_CJK_TERMINATORS = "()〈〉《》:,。;!?、\u201c\u201d\u2018\u2019【】"
|
||||
|
||||
MEDIA_TAG_CLEANUP_RE = re.compile(
|
||||
r'''[`"'*_]{0,3}MEDIA:\s*'''
|
||||
r'''(?P<path>`[^`\n]+?`|"[^"\n]+?"|'[^'\n]+?'|'''
|
||||
r'''(?:~/|/|[A-Za-z]:[/\\])\S+?(?:[^\S\n]+\S+?)*?\.(?:''' + _MEDIA_EXT_ALTERNATION + r'''))'''
|
||||
r'''(?=[\s`"'*_,;:)\]}\[]|MEDIA:|\.(?:\s|$)|$)[`"'*_]{0,3}\.?''',
|
||||
r'''(?=[\s`"'*_,;:)\]}\[''' + _MEDIA_CJK_TERMINATORS + r''']|MEDIA:|\.(?:\s|$)|$)[`"'*_]{0,3}\.?''',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
@@ -1979,7 +1985,7 @@ MEDIA_EXTENSIONLESS_TAG_RE = re.compile(
|
||||
r'''[`"'*_]{0,3}MEDIA:\s*'''
|
||||
r'''(?P<path>`[^`\n]+`|"[^"\n]+"|'[^'\n]+'|'''
|
||||
r'''(?:~/|/|[A-Za-z]:[/\\])[^\s\n`"']+?)'''
|
||||
r'''(?=[`"'\s,;:)\]}]|MEDIA:|$)'''
|
||||
r'''(?=[`"'\s,;:)\]}''' + _MEDIA_CJK_TERMINATORS + r''']|MEDIA:|$)'''
|
||||
r'''[`"'*_]{0,3}\s*''',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
@@ -34,3 +34,65 @@ class TestMediaTagCleanup:
|
||||
assert "/tmp/chart.png" not in stripped
|
||||
|
||||
|
||||
class TestMediaTagCjkTerminators:
|
||||
"""#88038: CJK full-width punctuation after a MEDIA path is a valid
|
||||
terminator, exactly like its ASCII counterparts. Chinese-language agent
|
||||
output naturally writes ``MEDIA:D:\\...\\zhibao.pdf(782.6 KB)`` — the
|
||||
ASCII-only lookahead silently dropped the attachment."""
|
||||
|
||||
def test_full_width_suffixes_terminate_the_path(self):
|
||||
from gateway.platforms.base import MEDIA_TAG_CLEANUP_RE
|
||||
|
||||
cases = [
|
||||
"MEDIA:D:/workspace/out/report.pdf(782.6 KB)",
|
||||
"MEDIA:D:/workspace/out/report.pdf:内容",
|
||||
"MEDIA:D:/workspace/out/report.pdf。",
|
||||
"MEDIA:D:/workspace/out/report.pdf,下一条",
|
||||
"MEDIA:D:/workspace/out/report.pdf;",
|
||||
"MEDIA:D:/workspace/out/report.pdf!",
|
||||
"MEDIA:D:/workspace/out/report.pdf?",
|
||||
"MEDIA:D:/workspace/out/report.pdf、",
|
||||
"MEDIA:D:/workspace/out/report.pdf”",
|
||||
"MEDIA:D:/workspace/out/report.pdf’",
|
||||
]
|
||||
for text in cases:
|
||||
m = MEDIA_TAG_CLEANUP_RE.search(text)
|
||||
assert m is not None, text
|
||||
assert m.group("path").endswith(".pdf"), (text, m.group("path"))
|
||||
|
||||
def test_real_chinese_cron_delivery_line(self):
|
||||
from gateway.platforms.base import MEDIA_TAG_CLEANUP_RE
|
||||
|
||||
text = (
|
||||
"## 交付物\n\n- **PDF 早报**:"
|
||||
"MEDIA:D:/workspace/zaobao/output/早报_2026-08-16.pdf(782.6 KB)"
|
||||
)
|
||||
m = MEDIA_TAG_CLEANUP_RE.search(text)
|
||||
assert m is not None
|
||||
assert m.group("path") == "D:/workspace/zaobao/output/早报_2026-08-16.pdf"
|
||||
|
||||
def test_ascii_terminators_still_work(self):
|
||||
from gateway.platforms.base import MEDIA_TAG_CLEANUP_RE
|
||||
|
||||
for text in (
|
||||
"MEDIA:/tmp/report.pdf",
|
||||
"MEDIA:/tmp/report.pdf 782 KB",
|
||||
"MEDIA:/tmp/report.pdf, next",
|
||||
):
|
||||
m = MEDIA_TAG_CLEANUP_RE.search(text)
|
||||
assert m is not None and m.group("path").endswith(".pdf"), text
|
||||
|
||||
def test_adjacent_tags_still_split(self):
|
||||
"""#68773 guard: the widened class must not let tags glue together."""
|
||||
from gateway.platforms.base import MEDIA_TAG_CLEANUP_RE
|
||||
|
||||
found = MEDIA_TAG_CLEANUP_RE.findall("MEDIA:/a.pngMEDIA:/b.png")
|
||||
assert list(found) == ["/a.png", "/b.png"]
|
||||
|
||||
def test_extensionless_variant_accepts_full_width_colon(self):
|
||||
from gateway.platforms.base import MEDIA_EXTENSIONLESS_TAG_RE
|
||||
|
||||
m = MEDIA_EXTENSIONLESS_TAG_RE.search("MEDIA:/tmp/Caddyfile:内容")
|
||||
assert m is not None and m.group("path") == "/tmp/Caddyfile"
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user