fix(web): cache extracts by returned URL

plugins/web/tavily/provider.py

@@ -57,13 +57,13 @@ def _normalize_tavily_search_results(response: Dict[str, Any]) -> Dict[str, Any]
     ])
 
 
-def _normalize_tavily_documents(response: Dict[str, Any], fallback_url: str = "") -> List[Dict[str, Any]]:
-    """Map ``/extract`` to documents; ``failed_results`` / ``failed_urls`` become ``error`` entries."""
+def _normalize_tavily_documents(response: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Map ``/extract`` to documents without attributing missing URLs to a request."""
     documents = [
-        document(r.get("url", fallback_url), r.get("title", ""), r.get("raw_content", "") or r.get("content", ""))
+        document(r.get("url", ""), r.get("title", ""), r.get("raw_content", "") or r.get("content", ""))
         for r in response.get("results", [])
     ]
-    documents += [_failed_document(f.get("url", fallback_url), f.get("error", "extraction failed")) for f in response.get("failed_results", [])]
+    documents += [_failed_document(f.get("url", ""), f.get("error", "extraction failed")) for f in response.get("failed_results", [])]
     documents += [_failed_document(str(u), "extraction failed") for u in response.get("failed_urls", [])]
     return documents
 
@@ -113,7 +113,7 @@ class TavilyWebSearchProvider(BaseWebSearchProvider):
                 return extract_fail(urls, missing)
             logger.info("Tavily %sextract: %d URL(s)", prefix, len(urls))
             raw = _tavily_request("extract", {"urls": urls, "include_images": False}, api_key=key)
-            return _normalize_tavily_documents(raw, fallback_url=urls[0] if urls else "")
+            return _normalize_tavily_documents(raw)
 
         return run_extract("Tavily", logger, urls, _body)
 

tests/tools/test_web_tools_tavily.py

@@ -142,11 +142,35 @@ class TestNormalizeTavilyDocuments:
         assert docs[0]["metadata"]["sourceURL"] == "https://example.com"
 
 
-    def test_fallback_url(self):
+    def test_missing_result_url_is_not_attributed_to_requested_url(self):
         from plugins.web.tavily.provider import _normalize_tavily_documents
         raw = {"results": [{"content": "data"}]}
-        docs = _normalize_tavily_documents(raw, fallback_url="https://fallback.com")
-        assert docs[0]["url"] == "https://fallback.com"
+        docs = _normalize_tavily_documents(raw)
+        assert docs[0]["url"] == ""
+
+
+class TestWebExtractCacheAttribution:
+    """Only cache content under the URL reported by the extract provider."""
+
+    def test_partial_result_caches_under_its_own_requested_url(self):
+        from tools import web_tools_extract as wte
+
+        class _PartialProvider:
+            name = "tavily"
+
+            async def extract(self, urls, format=None):
+                return [
+                    {"url": "https://unrequested.example/page", "raw_content": "foreign page"},
+                    {"url": urls[1], "raw_content": "second page", "title": "Second"},
+                ]
+
+        urls = ["https://example.com/first", "https://example.com/second"]
+        with patch("tools.web_result_cache.extract_cache_put") as cache_put:
+            asyncio.run(wte._dispatch_extract(_PartialProvider(), urls, None))
+
+        cache_put.assert_called_once_with(
+            "https://example.com/second", "second page", "Second", format=None, provider="tavily"
+        )
 
 
 # ─── availability / auto-detect ───────────────────────────────────────────────

tools/web_tools_extract.py

@@ -180,10 +180,12 @@ async def _dispatch_extract(provider, fetch_urls: List[str], format: Optional[st
     if results and all(r.get("error") for r in results) and _rescue_eligible(provider):
         return await asyncio.to_thread(_rescue_extract, provider.name, fetch_urls, results)
 
-    # Cache each successful fetch's full clean text (best-effort; oversized skipped).
-    for url, fetched in zip(fetch_urls, results):
+    # Cache each successful fetch under the URL that the provider actually returned.
+    # Providers may omit failed URLs or return successful results out of request order.
+    for fetched in results:
+        url = fetched.get("url")
         _content = fetched.get("raw_content", "") or fetched.get("content", "")
-        if _content and not fetched.get("error"):
+        if url in fetch_urls and _content and not fetched.get("error"):
             extract_cache_put(url, _content, fetched.get("title", ""), format=format, provider=provider.name)
     return results