fix(distributions): keep browser+web_search co-occurrence at 97% via grouped roll

Independent 97% rolls for browser and search co-occur only ~94% of the
time, versus 97% when web_search was bundled inside the browser toolset
(hermes-sweeper review). Grouped "+"-compound entries roll once and
select every member together; browser_tasks now uses "browser+search".
Deterministic tests cover the hit/miss/fallback/invalid-member contracts
and a 500-sample co-occurrence invariant.
This commit is contained in:
falkoro
2026-07-16 05:24:16 +02:00
committed by kshitij
parent decacbac35
commit 6ba09c8cc3
2 changed files with 78 additions and 12 deletions
+51
View File
@@ -64,3 +64,54 @@ class TestDistributionStructure:
for ts_name, prob in dist["toolsets"].items():
assert 0 < prob <= 100, f"{name}.{ts_name} has invalid probability {prob}"
class TestGroupedCompoundEntries:
""""+"-grouped entries roll once and select all members together.
browser_tasks uses "browser+search" so web_search availability stays
coupled to browser at the original 97% co-occurrence after #64503 removed
web_search from the browser toolset (independent 97% rolls would co-occur
only ~94% of the time).
"""
def test_browser_tasks_uses_grouped_entry(self):
assert "browser+search" in DISTRIBUTIONS["browser_tasks"]["toolsets"]
def test_hit_roll_selects_all_members_together(self, monkeypatch):
import toolset_distributions as td
monkeypatch.setattr(td.random, "random", lambda: 0.0) # every roll hits
result = sample_toolsets_from_distribution("browser_tasks")
assert "browser" in result
assert "search" in result
assert "browser+search" not in result # members, not the raw key
def test_members_always_co_occur(self, monkeypatch):
import random as _random
_random.seed(64525)
for _ in range(500):
result = sample_toolsets_from_distribution("browser_tasks")
assert ("browser" in result) == ("search" in result), result
def test_fallback_expands_compound_members(self, monkeypatch):
import toolset_distributions as td
monkeypatch.setattr(td.random, "random", lambda: 1.0) # every roll misses
# browser+search is browser_tasks' highest-probability entry (97%).
result = sample_toolsets_from_distribution("browser_tasks")
assert result == ["browser", "search"]
def test_invalid_member_skips_whole_entry(self, monkeypatch, capsys):
import toolset_distributions as td
monkeypatch.setitem(
td.DISTRIBUTIONS,
"_test_compound_invalid",
{"description": "t", "toolsets": {"browser+__not_a_toolset__": 100, "web": 100}},
)
monkeypatch.setattr(td.random, "random", lambda: 0.0)
result = sample_toolsets_from_distribution("_test_compound_invalid")
assert "browser" not in result # invalid member disqualifies the group
assert "web" in result
assert "not valid" in capsys.readouterr().out
+27 -12
View File
@@ -30,10 +30,13 @@ DISTRIBUTIONS = {
"reasoning": _dist("Heavy research/reasoning distribution with minimal other tools", web=90, file=60, terminal=20),
"browser_use": _dist("Full browser-based web interaction with search, vision, and page control", browser=100, web=80, vision=70),
"browser_only": _dist("Only browser automation tools for pure web interaction tasks", browser=100),
# browser-use-tasks.jsonl: web_search for finding URLs since Google blocks direct browser searches
# browser-use-tasks.jsonl: web_search for finding URLs since Google blocks direct browser searches.
# One grouped roll keeps web_search coupled to browser at the original 97%
# co-occurrence (browser no longer bundles it since #64503; independent
# rolls would drop co-occurrence to ~94%).
"browser_tasks": _dist(
"Browser-focused distribution with web_search for finding URLs (Google blocks direct browser searches)",
browser=97, search=97, vision=12, terminal=15,
**{"browser+search": 97}, vision=12, terminal=15,
),
# nous-terminal-tasks.jsonl
"terminal_tasks": _dist("Terminal-focused distribution with high terminal/file availability, occasional other tools",
@@ -57,25 +60,37 @@ def validate_distribution(distribution_name: str) -> bool:
return distribution_name in DISTRIBUTIONS
def sample_toolsets_from_distribution(distribution_name: str) -> List[str]:
"""Sample toolset names, each included independently with its % probability.
def _entry_members(entry: str) -> List[str]:
"""Toolsets named by a distribution entry: a bare name or a "+"-grouped compound."""
return [name.strip() for name in entry.split("+")]
Falls back to the highest-probability toolset when nothing was rolled.
def sample_toolsets_from_distribution(distribution_name: str) -> List[str]:
"""Sample toolset names, each entry included independently with its % probability.
An entry may be a single toolset or a "+"-grouped compound like
"browser+search": one roll selects (or skips) every member together, so
co-occurrence guarantees survive that independent rolls would break
(two independent 97% rolls co-occur only ~94% of the time).
Falls back to the highest-probability entry when nothing was rolled.
Raises ValueError for an unknown distribution.
"""
dist = get_distribution(distribution_name)
if not dist:
raise ValueError(f"Unknown distribution: {distribution_name}")
selected_toolsets = []
for toolset_name, probability in dist["toolsets"].items():
if not validate_toolset(toolset_name):
print(f"⚠️ Warning: Toolset '{toolset_name}' in distribution '{distribution_name}' is not valid")
for entry, probability in dist["toolsets"].items():
members = _entry_members(entry)
invalid = [name for name in members if not validate_toolset(name)]
if invalid:
print(f"⚠️ Warning: Toolset '{'+'.join(invalid)}' in distribution '{distribution_name}' is not valid")
elif random.random() * 100 < probability:
selected_toolsets.append(toolset_name)
selected_toolsets.extend(members)
if not selected_toolsets and dist["toolsets"]:
highest_prob_toolset = max(dist["toolsets"].items(), key=lambda x: x[1])[0]
if validate_toolset(highest_prob_toolset):
selected_toolsets.append(highest_prob_toolset)
highest_prob_entry = max(dist["toolsets"].items(), key=lambda x: x[1])[0]
members = _entry_members(highest_prob_entry)
if all(validate_toolset(name) for name in members):
selected_toolsets.extend(members)
return selected_toolsets