Teknium
39975613b1
Second, deeper pass over tools/gateway/hermes_cli plus first pass over the trees wave 1 missed (acp, acp_adapter, skills, computer_use, docker, dashboard, conformance, monitoring, secret_sources, hermes_state, providers). Same rubric as wave 1 (AGENTS.md test policy); security, alternation/caching invariants, issue-number regressions, and E2E kept. Real test-quality fixes found and rooted out along the way: - tests/tools/test_command_guards.py made real auxiliary-LLM HTTPS calls (DEFAULT_CONFIG smart-approval leaked in) — pinned approval mode=manual via autouse fixture: 17.4s → 0.4s. - test_model_switch_custom_providers.py / test_user_providers_model_switch.py silently probed live provider catalogs (~2s/test) — stubbed cached_provider_model_ids/provider_model_ids/fetch_api_models. - test_telegram_noise_filter.py: 15-platform copy-paste matrix over shared gateway.run logic → 3 representative platforms (55s → 3.9s). - test_gateway_shutdown.py: stop()'s 5s interrupt-deadline loop spun on MagicMock agents — interrupt.side_effect now clears _running_agents (22s → 1.0s). - test_gateway_inactivity_timeout.py poll-harness timings shrunk 3-5x (24s → 1.1s); test_mcp_stability.py backoff/SIGTERM-grace sleeps patched (15.4s → 2.5s); test_async_delegation.py negative-drain wait 5s → 0.5s. - test_telegram_init_deadline.py: loop-block margin restored to 1.0s with rationale comment — the watchdog-dump assertion needs the loop blocked well past deadline+grace under parallel load (flaked once in the 40-worker verification run at a 0.2s margin). Verification: full hermetic suite via scripts/run_tests.sh — 2,438 files, 21,718 tests passed, 0 failed, 293.9s wall. Suite totals vs original baseline: 46,820 → 19,757 test functions (−57.8%), wall 583.5s → 293.9s (−50%), subprocess CPU 13,564s → 11,623s.
166 lines
6.4 KiB
Python
166 lines
6.4 KiB
Python
"""Regression tests for private-page browser interaction guards."""
|
|
|
|
import json
|
|
|
|
import pytest
|
|
|
|
from tools import browser_tool
|
|
|
|
|
|
PRIVATE_URL = "http://169.254.169.254/latest/meta-data/"
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _browser_mode(monkeypatch):
|
|
monkeypatch.setattr(browser_tool, "_is_camofox_mode", lambda: False)
|
|
monkeypatch.setattr(browser_tool, "_last_session_key", lambda task_id: task_id)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("tool_call", "args"),
|
|
[
|
|
(browser_tool.browser_click, ("@e1",)),
|
|
(browser_tool.browser_type, ("@e1", "do-not-send-this")),
|
|
(browser_tool.browser_press, ("Enter",)),
|
|
],
|
|
)
|
|
def test_private_page_blocks_state_changing_actions(monkeypatch, tool_call, args):
|
|
monkeypatch.setattr(browser_tool, "_eval_ssrf_guard_active", lambda task_id: True)
|
|
monkeypatch.setattr(browser_tool, "_current_page_private_url", lambda task_id: PRIVATE_URL)
|
|
|
|
def fail_run(*_args, **_kwargs):
|
|
raise AssertionError("browser command should not run on a private page")
|
|
|
|
monkeypatch.setattr(browser_tool, "_run_browser_command", fail_run)
|
|
|
|
out = json.loads(tool_call(*args, task_id="task-1"))
|
|
|
|
assert out["success"] is False
|
|
assert PRIVATE_URL in out["error"]
|
|
assert "private or internal address" in out["error"]
|
|
assert "do-not-send-this" not in json.dumps(out)
|
|
|
|
|
|
def test_click_still_runs_when_current_page_is_public(monkeypatch):
|
|
calls = []
|
|
|
|
monkeypatch.setattr(browser_tool, "_eval_ssrf_guard_active", lambda task_id: True)
|
|
monkeypatch.setattr(browser_tool, "_current_page_private_url", lambda task_id: None)
|
|
|
|
def fake_run(task_id, command, args):
|
|
calls.append((task_id, command, args))
|
|
return {"success": True}
|
|
|
|
monkeypatch.setattr(browser_tool, "_run_browser_command", fake_run)
|
|
|
|
out = json.loads(browser_tool.browser_click("e1", task_id="task-1"))
|
|
|
|
assert out == {"success": True, "clicked": "@e1"}
|
|
assert calls == [("task-1", "click", ["@e1"])]
|
|
|
|
|
|
def test_guard_inactive_does_not_block_or_probe(monkeypatch):
|
|
"""When the SSRF guard is inactive (local backend / allow_private_urls),
|
|
the action must proceed WITHOUT even probing the page URL — a private-looking
|
|
current URL is irrelevant. This is the branch most likely to silently regress
|
|
if the guard condition is ever inverted, so it is exercised explicitly."""
|
|
calls = []
|
|
|
|
monkeypatch.setattr(browser_tool, "_eval_ssrf_guard_active", lambda task_id: False)
|
|
|
|
def fail_probe(task_id):
|
|
raise AssertionError("_current_page_private_url must not be probed when guard inactive")
|
|
|
|
monkeypatch.setattr(browser_tool, "_current_page_private_url", fail_probe)
|
|
|
|
def fake_run(task_id, command, args):
|
|
calls.append((task_id, command, args))
|
|
return {"success": True}
|
|
|
|
monkeypatch.setattr(browser_tool, "_run_browser_command", fake_run)
|
|
|
|
out = json.loads(browser_tool.browser_click("@e1", task_id="task-1"))
|
|
|
|
assert out == {"success": True, "clicked": "@e1"}
|
|
assert calls == [("task-1", "click", ["@e1"])]
|
|
|
|
|
|
def test_camofox_short_circuits_before_guard(monkeypatch):
|
|
"""Camofox mode returns from the dedicated camofox_* path BEFORE reaching the
|
|
private-page guard, so the guard's helpers must never be consulted. Guards the
|
|
ordering invariant (camofox early-return precedes _last_session_key + guard)."""
|
|
monkeypatch.setattr(browser_tool, "_is_camofox_mode", lambda: True)
|
|
|
|
def fail_guard(task_id):
|
|
raise AssertionError("guard must not run in camofox mode")
|
|
|
|
monkeypatch.setattr(browser_tool, "_eval_ssrf_guard_active", fail_guard)
|
|
monkeypatch.setattr(browser_tool, "_current_page_private_url", fail_guard)
|
|
|
|
import tools.browser_camofox as camofox
|
|
|
|
monkeypatch.setattr(camofox, "camofox_click", lambda ref, task_id: '{"success": true, "camofox": true}')
|
|
|
|
out = json.loads(browser_tool.browser_click("@e1", task_id="task-1"))
|
|
|
|
assert out == {"success": True, "camofox": True}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# browser_back — unlike click/type/press (check current page BEFORE acting),
|
|
# going back IS the navigation: the guard must fire AFTER _run_browser_command
|
|
# reports success, checking the page it just landed on, not the page it left.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_browser_back_blocks_when_landed_page_is_private(monkeypatch):
|
|
"""Browser history can land on a private/internal address the initial
|
|
browser_navigate preflight never saw — the same class of gap already
|
|
closed for browser_snapshot/vision/console/eval and click/type/press."""
|
|
monkeypatch.setattr(browser_tool, "_eval_ssrf_guard_active", lambda task_id: True)
|
|
monkeypatch.setattr(browser_tool, "_current_page_private_url", lambda task_id: PRIVATE_URL)
|
|
monkeypatch.setattr(
|
|
browser_tool, "_run_browser_command",
|
|
lambda task_id, command, args: {"success": True, "data": {"url": PRIVATE_URL}},
|
|
)
|
|
|
|
out = json.loads(browser_tool.browser_back(task_id="task-1"))
|
|
|
|
assert out["success"] is False
|
|
assert PRIVATE_URL in out["error"]
|
|
assert "private or internal address" in out["error"]
|
|
# The blocked payload must not itself leak the raw URL as a "url" field
|
|
# the way the success payload does.
|
|
assert "url" not in out
|
|
|
|
|
|
def test_browser_back_returns_url_when_landed_page_is_public(monkeypatch):
|
|
monkeypatch.setattr(browser_tool, "_eval_ssrf_guard_active", lambda task_id: True)
|
|
monkeypatch.setattr(browser_tool, "_current_page_private_url", lambda task_id: None)
|
|
monkeypatch.setattr(
|
|
browser_tool, "_run_browser_command",
|
|
lambda task_id, command, args: {"success": True, "data": {"url": "https://example.com/"}},
|
|
)
|
|
|
|
out = json.loads(browser_tool.browser_back(task_id="task-1"))
|
|
|
|
assert out == {"success": True, "url": "https://example.com/"}
|
|
|
|
|
|
def test_browser_back_camofox_short_circuits_before_guard(monkeypatch):
|
|
monkeypatch.setattr(browser_tool, "_is_camofox_mode", lambda: True)
|
|
|
|
def fail_guard(task_id):
|
|
raise AssertionError("guard must not run in camofox mode")
|
|
|
|
monkeypatch.setattr(browser_tool, "_eval_ssrf_guard_active", fail_guard)
|
|
monkeypatch.setattr(browser_tool, "_current_page_private_url", fail_guard)
|
|
|
|
import tools.browser_camofox as camofox
|
|
|
|
monkeypatch.setattr(camofox, "camofox_back", lambda task_id: '{"success": true, "camofox": true}')
|
|
|
|
out = json.loads(browser_tool.browser_back(task_id="task-1"))
|
|
|
|
assert out == {"success": True, "camofox": True}
|