diff --git a/plugins/platforms/whatsapp/adapter.py b/plugins/platforms/whatsapp/adapter.py index d001c0215f..91f6cbb22b 100644 --- a/plugins/platforms/whatsapp/adapter.py +++ b/plugins/platforms/whatsapp/adapter.py @@ -802,6 +802,34 @@ class WhatsAppAdapter(WhatsAppBehaviorMixin, BasePlatformAdapter): # Quoted message stays in structured fields only — GatewayRunner renders the "[Replying to: ...]" pointer. quoted = bool(data.get("hasQuotedMessage")) raw_reply_id = data.get("quotedMessageId") if quoted else None + if quoted: + # contextInfo.quotedMessage only ever carries a thumbnail-sized + # stub for media (or nothing for an uncaptioned attachment) — + # never a way to fetch the original file. The bridge resolves + # the quoted message's already-downloaded media via its own + # cache and hands back real local paths in quotedMediaUrls. + # Append them to this event's own media so the existing + # vision/audio pipeline (which reads media_urls/media_types) + # picks up the quoted attachment exactly like a direct one — + # no separate reply-media code path needed. + quoted_media_urls = data.get("quotedMediaUrls") or [] + quoted_media_type = str(data.get("quotedMediaType") or "").strip() + _quoted_mime_map = { + "image": "image/jpeg", + "video": "video/mp4", + "gif": "video/mp4", + "audio": "audio/ogg", + "ptt": "audio/ogg", + "document": "application/octet-stream", + "sticker": "image/webp", + } + for _qurl in quoted_media_urls: + if not (isinstance(_qurl, str) and os.path.isabs(_qurl) and _is_allowed_bridge_path(_qurl)): + print(f"[{self.name}] Rejected quoted-media path outside cache dir: {_qurl}", flush=True) + continue + cached_urls.append(_qurl) + media_types.append(_quoted_mime_map.get(quoted_media_type, "application/octet-stream")) + print(f"[{self.name}] Attached quoted-reply media: {_qurl}", flush=True) if msg_type == MessageType.DOCUMENT and cached_urls: body = self._inject_document_text(cached_urls, body) native_metadata = data.get("nativeMetadata") diff --git a/scripts/whatsapp-bridge/bridge.js b/scripts/whatsapp-bridge/bridge.js index 7db3f96c48..dc8a18421f 100644 --- a/scripts/whatsapp-bridge/bridge.js +++ b/scripts/whatsapp-bridge/bridge.js @@ -40,6 +40,7 @@ import { buildLocationPayload, buildTextSendPayload, createBoundedMessageStore, + createQuotedMediaCache, extractBridgeEvent, getMessageContent, inboundReadReceiptKeys, @@ -260,6 +261,10 @@ const MAX_QUEUE_SIZE = 100; const recentlySentIds = createOutboundIdTracker(512); const recentlyProcessedPollUpdates = createOutboundIdTracker(512); const messageStore = createBoundedMessageStore(512); +// Bounded cache of already-downloaded inbound media, so a later reply to an +// uncaptioned photo/video/document/voice note can still surface the original +// file — see createQuotedMediaCache's doc comment in bridge_helpers.js. +const quotedMediaCache = createQuotedMediaCache(512); function normalizePollUpdateOptions(aggregation, pollUpdateMessage, meId) { const selected = []; @@ -711,6 +716,7 @@ async function startSocket() { document: DOCUMENT_CACHE_DIR, audio: AUDIO_CACHE_DIR, }, + lookupQuotedMedia: (quotedChatId, quotedMessageId) => quotedMediaCache.get(quotedChatId, quotedMessageId), }); event.fromOwner = fromOwner; @@ -739,6 +745,15 @@ async function startSocket() { } messageStore.remember(msg); + // Remember this message's already-downloaded media/text so a later + // reply to it (even uncaptioned media) can resolve the original + // content instead of seeing only a stripped-down quoted-message stub. + quotedMediaCache.remember(chatId, msg.key.id, { + body: event.body, + hasMedia: event.hasMedia, + mediaType: event.mediaType, + mediaUrls: event.mediaUrls, + }); messageQueue.push(event); emitDebugEvent({ stage: 'queued', diff --git a/scripts/whatsapp-bridge/bridge.native.test.mjs b/scripts/whatsapp-bridge/bridge.native.test.mjs index 8e8f026c73..affcbb481b 100644 --- a/scripts/whatsapp-bridge/bridge.native.test.mjs +++ b/scripts/whatsapp-bridge/bridge.native.test.mjs @@ -16,6 +16,7 @@ import { buildPollPayload, buildTextSendPayload, createBoundedMessageStore, + createQuotedMediaCache, appendMediaFailureNote, extractBridgeEvent, inboundReadReceiptKeys, @@ -129,6 +130,89 @@ import { console.log(' ✓ inbound quoted metadata includes quoted text'); } +// -- reply to uncaptioned quoted media resolves the cached original file -- +{ + // contextInfo.quotedMessage only ever carries a thumbnail-sized stub for + // media (or nothing for an uncaptioned attachment) — extractBridgeEvent + // must fall back to lookupQuotedMedia to find the original cached file. + const event = await extractBridgeEvent({ + msg: { + key: { + id: 'incoming-2', + remoteJid: '15551234567@s.whatsapp.net', + participant: '15550001111@s.whatsapp.net', + fromMe: false, + }, + pushName: 'Tester', + messageTimestamp: 123, + message: { + extendedTextMessage: { + text: 'did you save this?', + contextInfo: { + stanzaId: 'original-image-1', + participant: '15550001111@s.whatsapp.net', + remoteJid: '15551234567@s.whatsapp.net', + // Real WhatsApp traffic: an uncaptioned quoted image carries no + // usable text, just a thumbnail-only imageMessage stub. + quotedMessage: { imageMessage: {} }, + }, + }, + }, + }, + chatId: '15551234567@s.whatsapp.net', + senderId: '15550001111@s.whatsapp.net', + senderNumber: '15550001111', + botIds: [], + downloadMedia: async () => Buffer.from(''), + lookupQuotedMedia: (chatId, messageId) => { + assert.equal(chatId, '15551234567@s.whatsapp.net'); + assert.equal(messageId, 'original-image-1'); + return { hasMedia: true, mediaType: 'image', mediaUrls: ['/cache/image/img_original.jpg'] }; + }, + }); + + assert.deepEqual(event.quotedMediaUrls, ['/cache/image/img_original.jpg']); + assert.equal(event.quotedMediaType, 'image'); + assert.equal(event.quotedText, 'sent an image'); + console.log(' ✓ reply to uncaptioned quoted image resolves cached original file'); +} + +// -- quoted media lookup miss leaves quoted fields empty (no crash) ------- +{ + const event = await extractBridgeEvent({ + msg: { + key: { + id: 'incoming-3', + remoteJid: '15551234567@s.whatsapp.net', + participant: '15550001111@s.whatsapp.net', + fromMe: false, + }, + messageTimestamp: 123, + message: { + extendedTextMessage: { + text: 'thanks', + contextInfo: { + stanzaId: 'long-gone-message', + participant: '15550001111@s.whatsapp.net', + remoteJid: '15551234567@s.whatsapp.net', + quotedMessage: { imageMessage: {} }, + }, + }, + }, + }, + chatId: '15551234567@s.whatsapp.net', + senderId: '15550001111@s.whatsapp.net', + senderNumber: '15550001111', + botIds: [], + downloadMedia: async () => Buffer.from(''), + lookupQuotedMedia: () => null, + }); + + assert.deepEqual(event.quotedMediaUrls, []); + assert.equal(event.quotedMediaType, ''); + console.log(' ✓ quoted media cache miss leaves quoted media fields empty'); +} + { const event = await extractBridgeEvent({ msg: { @@ -507,4 +591,25 @@ import { console.log(' ✓ nested envelopes: quote text resolves through both layers'); } +// -- createQuotedMediaCache ------------------------------------------------ +{ + const cache = createQuotedMediaCache(2); + cache.remember('chat-a', 'msg-1', { hasMedia: true, mediaType: 'image', mediaUrls: ['/cache/img1.jpg'] }); + + assert.deepEqual(cache.get('chat-a', 'msg-1'), { hasMedia: true, mediaType: 'image', mediaUrls: ['/cache/img1.jpg'] }); + // Different chatId, same messageId — must not collide; message ids are + // only unique within their own chat/JID. + assert.equal(cache.get('chat-b', 'msg-1'), null); + assert.equal(cache.get('chat-a', 'unknown-msg'), null); + + cache.remember('chat-a', 'msg-2', { hasMedia: false }); + cache.remember('chat-a', 'msg-3', { hasMedia: false }); + // Capacity is 2: the oldest entry (msg-1) must have been evicted. + assert.equal(cache.get('chat-a', 'msg-1'), null); + assert.notEqual(cache.get('chat-a', 'msg-2'), null); + assert.notEqual(cache.get('chat-a', 'msg-3'), null); + + console.log(' ✓ createQuotedMediaCache resolves by chatId+messageId and evicts oldest past capacity'); +} + console.log('\n✅ All WhatsApp native bridge helper tests passed.'); diff --git a/scripts/whatsapp-bridge/bridge_helpers.js b/scripts/whatsapp-bridge/bridge_helpers.js index 9c007ce24d..b50c70e654 100644 --- a/scripts/whatsapp-bridge/bridge_helpers.js +++ b/scripts/whatsapp-bridge/bridge_helpers.js @@ -81,6 +81,45 @@ export function createBoundedMessageStore(limit = 512) { return { remember, get }; } +/** + * Bounded cache of the already-downloaded media (and plain text) for + * recently seen inbound messages, keyed by "chatId:messageId". + * + * When a user replies to an earlier message, Baileys' contextInfo.quotedMessage + * only carries a thumbnail-sized stub for media (or nothing at all for an + * uncaptioned attachment) — never a way to re-fetch the full original file. + * Without this cache, replying to a photo/video/document/voice note with no + * caption gives the agent no text and no media reference: it looks like the + * message never had an attachment. Since extractBridgeEvent already downloads + * and caches the media for every inbound message as it arrives, remembering + * that outcome here lets a later reply resolve the original file path. + */ +export function createQuotedMediaCache(limit = 512) { + const byKey = new Map(); + + function key(chatId, messageId) { + return `${chatId || ''}:${messageId || ''}`; + } + + function remember(chatId, messageId, payload) { + if (!messageId) return; + const k = key(chatId, messageId); + byKey.delete(k); + byKey.set(k, payload); + while (byKey.size > limit) { + const oldest = byKey.keys().next().value; + byKey.delete(oldest); + } + } + + function get(chatId, messageId) { + if (!messageId) return null; + return byKey.get(key(chatId, messageId)) || null; + } + + return { remember, get }; +} + export function pollCreationMessageSecret(pollCreation) { return pollCreation?.message?.messageContextInfo?.messageSecret || pollCreation?.messageContextInfo?.messageSecret @@ -323,6 +362,7 @@ export async function extractBridgeEvent({ downloadMedia, writeMediaFile, cacheDirs = {}, + lookupQuotedMedia, }) { const messageContent = getMessageContent(msg); const contextInfo = getContextInfo(messageContent); @@ -331,7 +371,38 @@ export async function extractBridgeEvent({ const quotedParticipant = normalizeWhatsAppId(contextInfo?.participant || '') || null; const quotedRemoteJid = normalizeWhatsAppId(contextInfo?.remoteJid || '') || null; const hasQuotedMessage = !!contextInfo?.quotedMessage; - const quotedText = textFromQuotedMessage(contextInfo?.quotedMessage); + let quotedText = textFromQuotedMessage(contextInfo?.quotedMessage); + let quotedMediaUrls = []; + let quotedMediaType = ''; + + // contextInfo.quotedMessage only ever carries a thumbnail-sized stub for + // media (or nothing for an uncaptioned attachment) — never a way to + // re-fetch the original file. Resolve the quoted message's already-cached + // media (downloaded when it first arrived) via lookupQuotedMedia so a + // reply to an uncaptioned photo/video/document/voice note still gives the + // agent the original file, not just silence. + if (quotedMessageId && typeof lookupQuotedMedia === 'function') { + const original = lookupQuotedMedia(quotedRemoteJid || chatId, quotedMessageId); + if (original) { + if (original.hasMedia && original.mediaUrls?.length) { + quotedMediaUrls = original.mediaUrls; + quotedMediaType = original.mediaType || ''; + if (!quotedText) { + quotedText = { + image: 'sent an image', + video: 'sent a video', + gif: 'sent a GIF', + audio: 'sent an audio message', + ptt: 'sent a voice message', + document: 'sent a document', + sticker: 'sent a sticker', + }[original.mediaType] || 'sent media'; + } + } else if (original.body && !quotedText) { + quotedText = original.body; + } + } + } let body = ''; let hasMedia = false; @@ -498,6 +569,8 @@ export async function extractBridgeEvent({ quotedParticipant, quotedRemoteJid, quotedText, + quotedMediaUrls, + quotedMediaType, hasQuotedMessage, botIds, readReceiptKey: { diff --git a/tests/gateway/test_whatsapp_formatting.py b/tests/gateway/test_whatsapp_formatting.py index 79e1dacb78..914e385e1e 100644 --- a/tests/gateway/test_whatsapp_formatting.py +++ b/tests/gateway/test_whatsapp_formatting.py @@ -216,6 +216,92 @@ class TestBridgeEventMetadata: assert event.raw_message["quotedRemoteJid"] == "15551234567@s.whatsapp.net" assert event.raw_message["hasQuotedMessage"] is True + @pytest.mark.asyncio + async def test_reply_to_uncaptioned_image_attaches_quoted_media(self, tmp_path, monkeypatch): + # contextInfo.quotedMessage only ever carries a thumbnail-sized stub + # for media (or nothing for an uncaptioned attachment). The bridge + # resolves the quoted message's already-downloaded media via its own + # cache (createQuotedMediaCache) and hands back the real cached path + # in quotedMediaUrls. The adapter must fold that into this event's + # own media_urls/media_types so the existing vision pipeline picks it + # up — otherwise a reply like "save this" to an uncaptioned photo + # someone else sent looks to the agent like there is no image at all. + adapter = _make_adapter() + + cache_dir = tmp_path / "cache" / "image" + cache_dir.mkdir(parents=True) + quoted_image_path = cache_dir / "img_original.jpg" + quoted_image_path.write_bytes(b"fake-jpeg-bytes") + + from plugins.platforms.whatsapp import adapter as adapter_module + monkeypatch.setattr( + adapter_module, "_is_allowed_bridge_path", lambda url: True, + ) + + data = { + "messageId": "reply-msg", + "chatId": "15551234567@s.whatsapp.net", + "senderId": "15551234567@s.whatsapp.net", + "senderName": "Ananya", + "chatName": "Family", + "isGroup": True, + "body": "did you save this wedding invite?", + "hasMedia": False, + "mediaUrls": [], + "mediaType": "", + "quotedMessageId": "original-image-msg", + "quotedParticipant": "99999999999@s.whatsapp.net", + "quotedRemoteJid": "15551234567@s.whatsapp.net", + "hasQuotedMessage": True, + "quotedText": "", + "quotedMediaUrls": [str(quoted_image_path)], + "quotedMediaType": "image", + } + + event = await adapter._build_message_event(data) + + assert event is not None + assert str(quoted_image_path) in event.media_urls + idx = event.media_urls.index(str(quoted_image_path)) + assert event.media_types[idx] == "image/jpeg" + + @pytest.mark.asyncio + async def test_quoted_media_path_outside_cache_dir_is_rejected(self, monkeypatch): + # _is_allowed_bridge_path guards against a compromised/buggy bridge + # handing back an arbitrary absolute path; quoted-media handling must + # respect the same guard as direct media, not bypass it. + adapter = _make_adapter() + + from plugins.platforms.whatsapp import adapter as adapter_module + monkeypatch.setattr( + adapter_module, "_is_allowed_bridge_path", lambda url: False, + ) + + data = { + "messageId": "reply-msg-2", + "chatId": "15551234567@s.whatsapp.net", + "senderId": "15551234567@s.whatsapp.net", + "senderName": "Ananya", + "chatName": "Family", + "isGroup": True, + "body": "did you save this?", + "hasMedia": False, + "mediaUrls": [], + "mediaType": "", + "quotedMessageId": "original-image-msg", + "quotedParticipant": "99999999999@s.whatsapp.net", + "quotedRemoteJid": "15551234567@s.whatsapp.net", + "hasQuotedMessage": True, + "quotedText": "", + "quotedMediaUrls": ["/etc/passwd"], + "quotedMediaType": "image", + } + + event = await adapter._build_message_event(data) + + assert event is not None + assert "/etc/passwd" not in event.media_urls + # --------------------------------------------------------------------------- # display_config tier classification