Ben Barclay
8ddff33e4c
Third independent review. Both blockers reproduced against a real store before and after the fix. BLOCKER 1 — head-of-line starvation. The claim query is LIMIT 1, and a package already handled this pass was rejected AFTER the fetch, so _claim_next returned None and send_pending read that as 'queue empty'. Any row that sorts first and becomes eligible again mid-pass therefore terminated the pass. This is reachable normally: a 429 with a short Retry-After, or a pass outliving the 15-minute failure backoff (a legal pass runs ~1900s). Measured: 10 of 19 healthy packages silently dropped. The seen-set is now excluded IN SQL, so None genuinely means no eligible work. Same scenario now delivers 19 of 19. BLOCKER 2 — revoking consent leaked once it was re-granted. opt_in_period was write-once, so packages collected while the user had send: false still had period_start >= the ORIGINAL opt-in day; re-enabling released the whole refused window. Reproduced: 5 packages from a 5-day opted-out window transmitted on re-enable. Turning sending off now closes the consent window, and the next enabled pass opens a new one from that day. Recorded both in the setup wizard and in the sender itself, because config.yaml can be hand-edited where the wizard never sees it. Also: a send_attempts ceiling (a poisoned head row burned ~160 requests over 30 days, unbounded), _defer clamps to >= 1s so it cannot write a past deadline, and the dead skipped_not_due field is removed. Test-quality fixes, since vacuous tests have been the recurring problem: - the lease test asserted only 'in the future', passing for a 1s lease; it now requires the lease to outlast one package's worst legal case - test_shutdown_joins_the_send_thread grepped getsource for a method name — a change-detector AGENTS.md rejects — and is now behavioural - gzip determinism was unguarded: both retries in one pass compress in the same second, so removing mtime=0 was caught by nothing. Now compares output across a real second boundary. All five new regressions are mutation-verified: reintroducing each bug fails its test. The first attempt-ceiling test SURVIVED its mutation (the seeded row was excluded by another predicate) and was rewritten to drive the real loop. 251 tests pass. Staging E2E re-run: both packages 202.
278 lines
8.8 KiB
Python
278 lines
8.8 KiB
Python
"""Tests for wiring the sender into the shared-metrics export hook.
|
|
|
|
The properties that matter here are negative ones: the interactive path must
|
|
not block, and nothing must leave the machine unless the user opted in.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
import time
|
|
|
|
import pytest
|
|
|
|
from hermes_cli.observability import relay_shared_metrics as mod
|
|
|
|
|
|
class FakeStore:
|
|
def __init__(self):
|
|
self.exported = 0
|
|
|
|
def create_and_export_package_if_due(self):
|
|
self.exported += 1
|
|
return []
|
|
|
|
|
|
class FakeSubscriber:
|
|
def __init__(self):
|
|
self.store = FakeStore()
|
|
|
|
|
|
class Runtime(mod._Runtime):
|
|
"""A _Runtime with the relay host stubbed out."""
|
|
|
|
def __init__(self):
|
|
self._sessions_lock = threading.RLock()
|
|
self._sessions = {}
|
|
self._task_creation_lock = threading.RLock()
|
|
self._task_sessions_lock = threading.RLock()
|
|
self._send_lock = threading.RLock()
|
|
self._send_thread = None
|
|
self._task_sessions = {}
|
|
self._turn_sessions = {}
|
|
self.subscriber = FakeSubscriber()
|
|
|
|
|
|
@pytest.fixture
|
|
def runtime():
|
|
return Runtime()
|
|
|
|
|
|
def _config(**shared):
|
|
return {"telemetry": {"shared_metrics": shared}}
|
|
|
|
|
|
@pytest.fixture
|
|
def capture_sender(monkeypatch):
|
|
"""Replace the sender with a recorder and return the record."""
|
|
record = {"passes": [], "endpoints": []}
|
|
|
|
class FakeSender:
|
|
def __init__(self, store, endpoint, **kwargs):
|
|
record["endpoints"].append(endpoint)
|
|
|
|
def send_pending(self):
|
|
record["passes"].append(time.time())
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.observability.shared_metrics_sender.SharedMetricsSender",
|
|
FakeSender,
|
|
)
|
|
return record
|
|
|
|
|
|
def _set_config(monkeypatch, config):
|
|
monkeypatch.setattr(
|
|
"hermes_cli.config.read_raw_config_readonly", lambda: config, raising=False
|
|
)
|
|
|
|
|
|
class TestOptIn:
|
|
def test_no_send_when_nothing_is_configured(self, runtime, monkeypatch, capture_sender):
|
|
_set_config(monkeypatch, {})
|
|
runtime._export()
|
|
runtime._join_send_thread(timeout=1)
|
|
assert capture_sender["passes"] == []
|
|
|
|
def test_no_send_when_only_collection_is_on(self, runtime, monkeypatch, capture_sender):
|
|
_set_config(monkeypatch, _config(enabled=True))
|
|
runtime._export()
|
|
runtime._join_send_thread(timeout=1)
|
|
assert capture_sender["passes"] == []
|
|
|
|
def test_no_send_when_send_is_on_without_collection(
|
|
self, runtime, monkeypatch, capture_sender
|
|
):
|
|
_set_config(monkeypatch, _config(enabled=False, send=True))
|
|
runtime._export()
|
|
runtime._join_send_thread(timeout=1)
|
|
assert capture_sender["passes"] == []
|
|
|
|
def test_sends_when_both_are_on(self, runtime, monkeypatch, capture_sender):
|
|
_set_config(monkeypatch, _config(enabled=True, send=True))
|
|
runtime._export()
|
|
runtime._join_send_thread(timeout=2)
|
|
assert len(capture_sender["passes"]) == 1
|
|
|
|
def test_uses_the_resolved_endpoint(self, runtime, monkeypatch, capture_sender):
|
|
_set_config(
|
|
monkeypatch,
|
|
_config(enabled=True, send=True, endpoint="https://staging.test/v1"),
|
|
)
|
|
runtime._export()
|
|
runtime._join_send_thread(timeout=2)
|
|
assert capture_sender["endpoints"] == ["https://staging.test/v1"]
|
|
|
|
def test_export_still_runs_when_sending_is_off(self, runtime, monkeypatch, capture_sender):
|
|
_set_config(monkeypatch, _config(enabled=True))
|
|
runtime._export()
|
|
assert runtime.subscriber.store.exported == 1
|
|
|
|
|
|
class TestInteractivePathIsNotBlocked:
|
|
def test_export_returns_before_the_send_finishes(
|
|
self, runtime, monkeypatch
|
|
):
|
|
started = threading.Event()
|
|
release = threading.Event()
|
|
|
|
class SlowSender:
|
|
def __init__(self, store, endpoint, **kwargs):
|
|
pass
|
|
|
|
def send_pending(self):
|
|
started.set()
|
|
release.wait(5)
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.observability.shared_metrics_sender.SharedMetricsSender",
|
|
SlowSender,
|
|
)
|
|
_set_config(monkeypatch, _config(enabled=True, send=True))
|
|
|
|
began = time.monotonic()
|
|
runtime._export()
|
|
elapsed = time.monotonic() - began
|
|
|
|
assert started.wait(2), "the send should have started"
|
|
assert elapsed < 1.0, "finish_task must not wait on the network"
|
|
release.set()
|
|
runtime._join_send_thread(timeout=5)
|
|
|
|
def test_the_send_thread_is_a_daemon(self, runtime, monkeypatch, capture_sender):
|
|
_set_config(monkeypatch, _config(enabled=True, send=True))
|
|
runtime._export()
|
|
with runtime._send_lock:
|
|
thread = runtime._send_thread
|
|
assert thread is not None
|
|
assert thread.daemon, "an unfinished send must not hold the process open"
|
|
runtime._join_send_thread(timeout=2)
|
|
|
|
def test_only_one_pass_runs_at_a_time(self, runtime, monkeypatch):
|
|
release = threading.Event()
|
|
starts = []
|
|
|
|
class SlowSender:
|
|
def __init__(self, store, endpoint, **kwargs):
|
|
pass
|
|
|
|
def send_pending(self):
|
|
starts.append(1)
|
|
release.wait(5)
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.observability.shared_metrics_sender.SharedMetricsSender",
|
|
SlowSender,
|
|
)
|
|
_set_config(monkeypatch, _config(enabled=True, send=True))
|
|
|
|
for _ in range(5):
|
|
runtime._export()
|
|
time.sleep(0.2)
|
|
assert len(starts) == 1, "hook fires must not pile up send passes"
|
|
release.set()
|
|
runtime._join_send_thread(timeout=5)
|
|
|
|
|
|
class TestFailureIsolation:
|
|
def test_a_sender_crash_does_not_propagate(self, runtime, monkeypatch):
|
|
class Exploding:
|
|
def __init__(self, store, endpoint, **kwargs):
|
|
pass
|
|
|
|
def send_pending(self):
|
|
raise RuntimeError("boom")
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.observability.shared_metrics_sender.SharedMetricsSender",
|
|
Exploding,
|
|
)
|
|
_set_config(monkeypatch, _config(enabled=True, send=True))
|
|
runtime._export() # must not raise
|
|
runtime._join_send_thread(timeout=2)
|
|
|
|
def test_an_unreadable_config_does_not_break_export(self, runtime, monkeypatch, capture_sender):
|
|
def explode():
|
|
raise OSError("config unreadable")
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.config.read_raw_config_readonly", explode, raising=False
|
|
)
|
|
runtime._export()
|
|
assert runtime.subscriber.store.exported == 1
|
|
assert capture_sender["passes"] == []
|
|
|
|
def test_join_is_safe_with_no_thread(self, runtime):
|
|
runtime._join_send_thread(timeout=0.1)
|
|
|
|
def test_join_waits_for_an_in_flight_send(self, runtime, monkeypatch):
|
|
"""shutdown() must give a started send a chance to finish.
|
|
|
|
A short-lived CLI exits straight after its final export; without the
|
|
join the daemon thread is killed mid-request, and the hook path is the
|
|
only delivery cadence this feature has.
|
|
"""
|
|
finished = []
|
|
release = threading.Event()
|
|
|
|
class SlowSender:
|
|
def __init__(self, store, endpoint, **kwargs):
|
|
pass
|
|
|
|
def send_pending(self):
|
|
release.wait(3)
|
|
finished.append(True)
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.observability.shared_metrics_sender.SharedMetricsSender",
|
|
SlowSender,
|
|
)
|
|
_set_config(monkeypatch, _config(enabled=True, send=True))
|
|
|
|
runtime._export()
|
|
release.set()
|
|
runtime._join_send_thread(timeout=3)
|
|
assert finished == [True]
|
|
|
|
def test_shutdown_joins_the_send_thread(self, monkeypatch):
|
|
"""shutdown() must actually wait, not merely mention the join.
|
|
|
|
Behavioural, not a source grep: an earlier version of this test
|
|
inspected getsource for a method name, which AGENTS.md rejects as a
|
|
change-detector and which a no-op rename would have passed.
|
|
"""
|
|
runtime = Runtime()
|
|
released = threading.Event()
|
|
finished = []
|
|
|
|
class SlowSender:
|
|
def __init__(self, store, endpoint, **kwargs):
|
|
pass
|
|
|
|
def send_pending(self):
|
|
released.wait(3)
|
|
finished.append(True)
|
|
|
|
monkeypatch.setattr(
|
|
"hermes_cli.observability.shared_metrics_sender.SharedMetricsSender",
|
|
SlowSender,
|
|
)
|
|
_set_config(monkeypatch, _config(enabled=True, send=True))
|
|
|
|
# Stand in for the parts of shutdown() that need a live relay.
|
|
runtime._export()
|
|
assert runtime._send_thread is not None
|
|
released.set()
|
|
runtime._join_send_thread()
|
|
assert finished == [True], "shutdown returned while a send was in flight"
|