Drexuxux
5b5b5e8da0
A gate runs whatever command the operator configured, so its output is arbitrary bytes. run_gate captured it with text=True and no encoding, which decodes with locale.getpreferredencoding() under errors="strict". One byte the decoder rejects — a test runner's checkmarks or CJK on a non-UTF-8 Windows console, a stray binary byte anywhere in the stream — kills subprocess's reader thread. proc.stdout comes back None, the `or ""` fallback turns that into an empty tail, and an unhandled traceback is dumped to stderr. The gate's pass/fail verdict still lands on the exit code, but the output tail is exactly what the retry prompt feeds back so the agent can fix the failure. With it empty the agent is told a gate failed and given nothing to act on, so it burns every retry and the goal auto-pauses. workspace_fingerprint has the same two calls; there a non-ASCII path in `git status --porcelain` empties the fingerprint, silently disabling the unchanged-gate skip that exists to stop a stalled agent re-running the same red suite. Decode as UTF-8 with errors="replace" — what git and modern toolchains emit, and what 262 of the repo's 299 text-mode subprocess calls already do.
255 lines
9.6 KiB
Python
255 lines
9.6 KiB
Python
"""Tests for /goal quality gates (GoalGate, run_gate, GoalManager gate flow)."""
|
|
|
|
import json
|
|
import sys
|
|
import time
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from hermes_cli.goals import (
|
|
DEFAULT_GATE_MAX_RETRIES,
|
|
DEFAULT_GATE_TIMEOUT_SECONDS,
|
|
GoalGate,
|
|
GoalManager,
|
|
GoalState,
|
|
run_gate,
|
|
save_goal,
|
|
load_goal,
|
|
)
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# GoalGate serialization
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_gate_roundtrip_through_goalstate_json():
|
|
state = GoalState(goal="ship it", status="active")
|
|
state.gates.append(GoalGate(command="echo ok", timeout_seconds=42, max_retries=7))
|
|
raw = state.to_json()
|
|
loaded = GoalState.from_json(raw)
|
|
assert len(loaded.gates) == 1
|
|
g = loaded.gates[0]
|
|
assert g.command == "echo ok"
|
|
assert g.timeout_seconds == 42
|
|
assert g.max_retries == 7
|
|
assert g.attempts == 0
|
|
|
|
|
|
def test_gate_from_dict_defaults_and_garbage():
|
|
g = GoalGate.from_dict({"command": "true"})
|
|
assert g.timeout_seconds == DEFAULT_GATE_TIMEOUT_SECONDS
|
|
assert g.max_retries == DEFAULT_GATE_MAX_RETRIES
|
|
assert GoalGate.from_dict(None).command == ""
|
|
assert GoalGate.from_dict("nonsense").command == ""
|
|
|
|
|
|
def test_old_state_rows_without_gates_load_clean():
|
|
"""Backwards compatibility: pre-gates state_meta rows load with no gates."""
|
|
old = {"goal": "legacy", "status": "active"}
|
|
state = GoalState.from_json(json.dumps(old))
|
|
assert state.gates == []
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# run_gate
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_run_gate_pass():
|
|
passed, code, out = run_gate(GoalGate(command="echo hello"))
|
|
assert passed is True
|
|
assert code == 0
|
|
assert "hello" in out
|
|
|
|
|
|
def test_run_gate_fail_captures_output():
|
|
passed, code, out = run_gate(GoalGate(command="echo broken >&2; exit 3"))
|
|
assert passed is False
|
|
assert code == 3
|
|
assert "broken" in out
|
|
|
|
|
|
def test_run_gate_timeout():
|
|
passed, code, out = run_gate(GoalGate(command="sleep 5", timeout_seconds=1))
|
|
assert passed is False
|
|
assert code == -1
|
|
assert "timed out" in out
|
|
|
|
|
|
def test_run_gate_keeps_diagnostics_when_a_byte_will_not_decode(tmp_path):
|
|
"""A gate's output tail must survive bytes the decoder rejects.
|
|
|
|
A gate runs whatever the operator configured, so its output is arbitrary
|
|
bytes — a test runner's checkmarks or CJK on a non-UTF-8 Windows console,
|
|
or stray binary. Decoding strictly means one bad byte kills subprocess's
|
|
reader thread, stdout comes back None, and the tail lands empty: the agent
|
|
is told the gate failed with nothing to act on, so it burns every retry and
|
|
the goal auto-pauses.
|
|
"""
|
|
script = tmp_path / "gate.py"
|
|
script.write_text(
|
|
"import os, sys\n"
|
|
"os.write(1, b'FAILED: 3 tests broken \\x90\\x8d rerun me\\n')\n"
|
|
"sys.exit(1)\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
passed, code, out = run_gate(
|
|
GoalGate(command=f'"{sys.executable}" "{script}"'),
|
|
)
|
|
|
|
assert passed is False
|
|
assert code == 1
|
|
assert "FAILED: 3 tests broken" in out, (
|
|
f"gate diagnostics were lost to a decode failure (tail={out!r})"
|
|
)
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# GoalManager gate management
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def _mgr_with_goal(session_id="gate-test-sid"):
|
|
mgr = GoalManager(session_id=session_id)
|
|
mgr.set("test goal")
|
|
return mgr
|
|
|
|
|
|
def test_add_remove_clear_gates():
|
|
mgr = _mgr_with_goal("gate-mgmt-sid")
|
|
mgr.add_gate("echo one")
|
|
mgr.add_gate("echo two")
|
|
assert len(mgr.state.gates) == 2
|
|
assert "echo one" in mgr.render_gates()
|
|
|
|
removed = mgr.remove_gate(1)
|
|
assert removed == "echo one"
|
|
assert len(mgr.state.gates) == 1
|
|
|
|
assert mgr.clear_gates() == 1
|
|
assert mgr.state.gates == []
|
|
|
|
|
|
def test_add_gate_requires_active_goal():
|
|
mgr = GoalManager(session_id="gate-nogoal-sid")
|
|
with pytest.raises(RuntimeError):
|
|
mgr.add_gate("echo nope")
|
|
|
|
|
|
def test_gates_persist_and_reload():
|
|
mgr = _mgr_with_goal("gate-persist-sid")
|
|
mgr.add_gate("echo persisted")
|
|
reloaded = GoalManager(session_id="gate-persist-sid")
|
|
assert len(reloaded.state.gates) == 1
|
|
assert reloaded.state.gates[0].command == "echo persisted"
|
|
|
|
|
|
def test_status_line_mentions_gates():
|
|
mgr = _mgr_with_goal("gate-status-sid")
|
|
mgr.add_gate("echo g")
|
|
assert "1 gate" in mgr.status_line()
|
|
|
|
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
# evaluate_after_turn integration
|
|
# ──────────────────────────────────────────────────────────────────────
|
|
|
|
|
|
def test_failing_gate_short_circuits_judge():
|
|
mgr = _mgr_with_goal("gate-fail-sid")
|
|
mgr.add_gate("exit 5")
|
|
with patch("hermes_cli.goals.judge_goal") as mock_judge, \
|
|
patch("hermes_cli.goals.workspace_fingerprint", return_value=""):
|
|
decision = mgr.evaluate_after_turn("I think it's done!")
|
|
mock_judge.assert_not_called()
|
|
assert decision["verdict"] == "gate_failed"
|
|
assert decision["should_continue"] is True
|
|
assert "exit 5" in decision["continuation_prompt"]
|
|
assert "quality gate" in decision["continuation_prompt"].lower()
|
|
|
|
|
|
def test_passing_gates_fall_through_to_judge():
|
|
mgr = _mgr_with_goal("gate-pass-sid")
|
|
mgr.add_gate("true")
|
|
with patch(
|
|
"hermes_cli.goals.judge_goal",
|
|
return_value=("done", "all good", False, None, False),
|
|
) as mock_judge:
|
|
decision = mgr.evaluate_after_turn("finished")
|
|
mock_judge.assert_called_once()
|
|
assert decision["verdict"] == "done"
|
|
# Passing run resets attempt bookkeeping.
|
|
assert mgr.state.gates[0].attempts == 0
|
|
assert mgr.state.gates[0].last_exit_code == 0
|
|
|
|
|
|
def test_gate_retry_exhaustion_pauses_goal():
|
|
mgr = _mgr_with_goal("gate-exhaust-sid")
|
|
mgr.add_gate("exit 1")
|
|
mgr.state.gates[0].max_retries = 2
|
|
with patch("hermes_cli.goals.judge_goal") as mock_judge, \
|
|
patch("hermes_cli.goals.workspace_fingerprint", return_value=""):
|
|
d1 = mgr.evaluate_after_turn("attempt one")
|
|
d2 = mgr.evaluate_after_turn("attempt two")
|
|
d3 = mgr.evaluate_after_turn("attempt three")
|
|
mock_judge.assert_not_called()
|
|
assert d1["should_continue"] is True
|
|
assert d2["should_continue"] is True
|
|
assert d3["status"] == "paused"
|
|
assert d3["should_continue"] is False
|
|
assert mgr.state.status == "paused"
|
|
assert "gate" in (mgr.state.paused_reason or "")
|
|
|
|
|
|
def test_unchanged_workspace_skips_rerun():
|
|
mgr = _mgr_with_goal("gate-unchanged-sid")
|
|
mgr.add_gate("exit 1")
|
|
with patch("hermes_cli.goals.workspace_fingerprint", return_value="fp-1"), \
|
|
patch("hermes_cli.goals.judge_goal"):
|
|
mgr.evaluate_after_turn("turn 1")
|
|
# Second turn, same fingerprint — run_gate must NOT run again.
|
|
with patch("hermes_cli.goals.run_gate") as mock_run:
|
|
d2 = mgr.evaluate_after_turn("turn 2")
|
|
mock_run.assert_not_called()
|
|
assert d2["verdict"] == "gate_failed"
|
|
assert "unchanged" in d2["message"]
|
|
|
|
|
|
def test_changed_workspace_reruns_gate():
|
|
mgr = _mgr_with_goal("gate-changed-sid")
|
|
mgr.add_gate("exit 1")
|
|
with patch("hermes_cli.goals.judge_goal"):
|
|
with patch("hermes_cli.goals.workspace_fingerprint", return_value="fp-1"):
|
|
mgr.evaluate_after_turn("turn 1")
|
|
with patch("hermes_cli.goals.workspace_fingerprint", return_value="fp-2"), \
|
|
patch("hermes_cli.goals.run_gate", return_value=(False, 1, "still red")) as mock_run:
|
|
mgr.evaluate_after_turn("turn 2")
|
|
mock_run.assert_called_once()
|
|
|
|
|
|
def test_gate_continuation_respects_turn_budget():
|
|
mgr = GoalManager(session_id="gate-budget-sid", default_max_turns=1)
|
|
mgr.set("budget goal")
|
|
mgr.add_gate("exit 1")
|
|
with patch("hermes_cli.goals.judge_goal"), \
|
|
patch("hermes_cli.goals.workspace_fingerprint", return_value=""):
|
|
decision = mgr.evaluate_after_turn("only turn")
|
|
assert decision["status"] == "paused"
|
|
assert decision["should_continue"] is False
|
|
assert "turns used" in decision["message"]
|
|
|
|
|
|
def test_no_gates_behaves_exactly_as_before():
|
|
mgr = _mgr_with_goal("gate-none-sid")
|
|
with patch(
|
|
"hermes_cli.goals.judge_goal",
|
|
return_value=("continue", "keep going", False, None, False),
|
|
) as mock_judge:
|
|
decision = mgr.evaluate_after_turn("wip")
|
|
mock_judge.assert_called_once()
|
|
assert decision["verdict"] == "continue"
|
|
assert decision["should_continue"] is True
|