ethernet
3ee1bb323e
The publisher read the PR number from the CI run's pull_requests payload. GitHub keeps that payload empty for fork runs, so the job printed 'No pull request is associated' and stopped on every fork PR. Resolve the PR from the run's head owner, branch, and SHA instead. The SHA match skips runs that a newer push superseded. A fork PR also has no CI review comment, because the live poller skips forks. The publisher now logs this and exits clean instead of raising; the evidence stays in the workflow artifact.
218 lines
6.9 KiB
Python
218 lines
6.9 KiB
Python
"""Tests for scripts/ci/publish_e2e_evidence.py."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
_PATH = Path(__file__).resolve().parents[2] / "scripts" / "ci" / "publish_e2e_evidence.py"
|
|
_spec = importlib.util.spec_from_file_location("publish_e2e_evidence", _PATH)
|
|
if _spec is None or _spec.loader is None:
|
|
raise ImportError("Failed to load publish_e2e_evidence.py")
|
|
_mod = importlib.util.module_from_spec(_spec)
|
|
sys.modules["publish_e2e_evidence"] = _mod
|
|
_spec.loader.exec_module(_mod)
|
|
|
|
|
|
def _png(width: int = 4, height: int = 3) -> bytes:
|
|
return _mod.PNG_SIGNATURE + b"\x00\x00\x00\rIHDR" + width.to_bytes(4, "big") + height.to_bytes(4, "big")
|
|
|
|
|
|
def test_load_evidence_validates_manifest_and_pngs(tmp_path):
|
|
(tmp_path / "shot.png").write_bytes(_png())
|
|
(tmp_path / "diff.png").write_bytes(_png())
|
|
(tmp_path / "actual.png").write_bytes(_png())
|
|
(tmp_path / "expected.png").write_bytes(_png())
|
|
(tmp_path / "e2e-evidence.json").write_text(
|
|
"""{
|
|
"version": 1,
|
|
"screenshots": [{"name": "main-view.png", "file": "shot.png"}],
|
|
"diffs": [{"name": "main-view", "diff": "diff.png", "actual": "actual.png", "expected": "expected.png"}]
|
|
}""",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
files, payloads = _mod.load_evidence(tmp_path)
|
|
|
|
assert [item.label for item in files] == [
|
|
"new screenshot: main-view.png",
|
|
"visual diff: main-view",
|
|
"visual actual: main-view",
|
|
"visual expected: main-view",
|
|
]
|
|
assert set(payloads) == {"shot.png", "diff.png", "actual.png", "expected.png"}
|
|
|
|
|
|
def test_load_evidence_rejects_path_escape_and_non_png(tmp_path):
|
|
(tmp_path / "e2e-evidence.json").write_text(
|
|
'{"version":1,"screenshots":[{"name":"bad","file":"../secret.png"}],"diffs":[]}',
|
|
encoding="utf-8",
|
|
)
|
|
|
|
with pytest.raises(ValueError, match="unsafe filename"):
|
|
_mod.load_evidence(tmp_path)
|
|
|
|
(tmp_path / "e2e-evidence.json").write_text(
|
|
'{"version":1,"screenshots":[{"name":"bad","file":"not-png.png"}],"diffs":[]}',
|
|
encoding="utf-8",
|
|
)
|
|
(tmp_path / "not-png.png").write_bytes(b"not a png")
|
|
|
|
with pytest.raises(ValueError, match="not a PNG"):
|
|
_mod.load_evidence(tmp_path)
|
|
|
|
|
|
|
|
|
|
def test_upload_evidence_accepts_only_attachment_urls(tmp_path, monkeypatch):
|
|
shot = tmp_path / "shot.png"
|
|
shot.write_bytes(_png())
|
|
calls = []
|
|
|
|
def fake_run(args, **kwargs):
|
|
calls.append((args, kwargs))
|
|
return _mod.subprocess.CompletedProcess(
|
|
args,
|
|
0,
|
|
stdout="\n",
|
|
)
|
|
|
|
monkeypatch.setattr(_mod.subprocess, "run", fake_run)
|
|
|
|
result = _mod.upload_evidence(
|
|
[_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
|
|
tmp_path,
|
|
"NousResearch/hermes-agent",
|
|
"bot-session-token",
|
|
)
|
|
|
|
assert result == {"shot.png": "https://github.com/user-attachments/assets/12345678-1234-1234-1234-123456789abc"}
|
|
assert calls[0][0] == ["gh", "image", "--repo", "NousResearch/hermes-agent", str(shot)]
|
|
assert calls[0][1]["env"]["GH_SESSION_TOKEN"] == "bot-session-token"
|
|
|
|
|
|
|
|
|
|
def test_upload_evidence_reports_gh_image_error(tmp_path, monkeypatch, capsys):
|
|
shot = tmp_path / "shot.png"
|
|
shot.write_bytes(_png())
|
|
|
|
def fake_run(args, **kwargs):
|
|
raise _mod.subprocess.CalledProcessError(
|
|
1,
|
|
args,
|
|
output="upload output",
|
|
stderr="upload error",
|
|
)
|
|
|
|
monkeypatch.setattr(_mod.subprocess, "run", fake_run)
|
|
|
|
with pytest.raises(RuntimeError, match="Failed to upload shot.png.*upload error"):
|
|
_mod.upload_evidence(
|
|
[_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
|
|
tmp_path,
|
|
"NousResearch/hermes-agent",
|
|
"bot-session-token",
|
|
)
|
|
|
|
captured = capsys.readouterr()
|
|
assert "Failed to upload shot.png" in captured.err
|
|
assert "upload output" in captured.err
|
|
assert "upload error" in captured.err
|
|
|
|
|
|
def test_publish_marks_evidence_upload_failure_in_pr_comment(tmp_path, monkeypatch):
|
|
comment = {
|
|
"id": 123,
|
|
"body": "before\n<!-- hermes-e2e-evidence:start -->\npending\n<!-- hermes-e2e-evidence:end -->\nafter",
|
|
}
|
|
updates = []
|
|
|
|
monkeypatch.setattr(
|
|
_mod,
|
|
"load_evidence",
|
|
lambda evidence_dir: (
|
|
[_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
|
|
{},
|
|
),
|
|
)
|
|
monkeypatch.setattr(_mod, "_wait_for_review_comment", lambda *args: comment)
|
|
monkeypatch.setattr(
|
|
_mod,
|
|
"upload_evidence",
|
|
lambda *args: (_ for _ in ()).throw(
|
|
RuntimeError("Failed to upload shot.png: bad <response>")
|
|
),
|
|
)
|
|
monkeypatch.setattr(
|
|
_mod,
|
|
"_api_request",
|
|
lambda url, token, method, payload: updates.append((
|
|
url,
|
|
token,
|
|
method,
|
|
payload,
|
|
)),
|
|
)
|
|
|
|
with pytest.raises(RuntimeError, match="Failed to upload shot.png"):
|
|
_mod.publish(
|
|
"github-token",
|
|
"NousResearch/hermes-agent",
|
|
tmp_path,
|
|
"69868",
|
|
"image-token",
|
|
)
|
|
|
|
assert updates == [
|
|
(
|
|
"https://api.github.com/repos/NousResearch/hermes-agent/issues/comments/123",
|
|
"github-token",
|
|
"PATCH",
|
|
{
|
|
"body": "before\n<!-- hermes-e2e-evidence:start -->\n<sub>inline evidence upload failed.</sub>\n\n<pre>Failed to upload shot.png: bad <response></pre>\n<!-- hermes-e2e-evidence:end -->\nafter"
|
|
},
|
|
)
|
|
]
|
|
|
|
|
|
def test_publish_skips_when_no_review_comment_exists(tmp_path, monkeypatch, capsys):
|
|
monkeypatch.setattr(
|
|
_mod,
|
|
"load_evidence",
|
|
lambda evidence_dir: (
|
|
[_mod.EvidenceFile("shot.png", "new screenshot: shot.png")],
|
|
{},
|
|
),
|
|
)
|
|
monkeypatch.setattr(_mod, "_wait_for_review_comment", lambda *args: None)
|
|
monkeypatch.setattr(
|
|
_mod,
|
|
"upload_evidence",
|
|
lambda *args: (_ for _ in ()).throw(AssertionError("must not upload")),
|
|
)
|
|
|
|
assert _mod.publish(
|
|
"github-token",
|
|
"NousResearch/hermes-agent",
|
|
tmp_path,
|
|
"83202",
|
|
"image-token",
|
|
) is False
|
|
assert "no CI review comment" in capsys.readouterr().out
|
|
|
|
|
|
def test_find_review_comment_requires_the_evidence_marker():
|
|
pending = "<!-- hermes-ci-review-bot -->\n<!-- hermes-e2e-evidence:start -->\npending\n<!-- hermes-e2e-evidence:end -->"
|
|
|
|
assert _mod._find_review_comment([{"body": "<!-- hermes-ci-review-bot --> no evidence"}]) is None
|
|
assert _mod._find_review_comment([{"body": pending, "id": 123}]) == {"body": pending, "id": 123}
|
|
|
|
|
|
def test_replace_evidence_marker_requires_exactly_one_marker():
|
|
with pytest.raises(ValueError, match="does not contain one"):
|
|
_mod.replace_evidence_marker("no marker", "evidence")
|